stdio-common: Convert vfprintf and related functions to buffers

vfprintf is entangled with vfwprintf (of course), __printf_fp,
__printf_fphex, __vstrfmon_l_internal, and the strfrom family of
functions.  The latter use the internal snprintf functionality,
so vsnprintf is converted as well.

The simples conversion is __printf_fphex, followed by
__vstrfmon_l_internal and __printf_fp, and finally
__vfprintf_internal and __vfwprintf_internal.  __vsnprintf_internal
and strfrom* are mostly consuming the new interfaces, so they
are comparatively simple.

__printf_fp is a public symbol, so the FILE *-based interface
had to preserved.

The __printf_fp rewrite does not change the actual binary-to-decimal
conversion algorithm, and digits are still not emitted directly to
the target buffer.  However, the staging buffer now uses bytes
instead of wide characters, and one buffer copy is eliminated.

The changes are at least performance-neutral in my testing.
Floating point printing and snprintf improved measurably, so that
this Lua script

  for i=1,5000000 do
      print(i, i * math.pi)
  end

runs about 5% faster for me.  To preserve fprintf performance for
a simple "%d" format, this commit has some logic changes under
LABEL (unsigned_number) to avoid additional function calls.  There
are certainly some very easy performance improvements here: binary,
octal and hexadecimal formatting can easily avoid the temporary work
buffer (the number of digits can be computed ahead-of-time using one
of the __builtin_clz* built-ins). Decimal formatting can use a
specialized version of _itoa_word for base 10.

The existing (inconsistent) width handling between strfmon and printf
is preserved here.  __print_fp_buffer_1 would have to use
__translated_number_width to achieve ISO conformance for printf.

Test expectations in libio/tst-vtables-common.c are adjusted because
the internal staging buffer merges all virtual function calls into
one.

In general, stack buffer usage is greatly reduced, particularly for
unbuffered input streams.  __printf_fp can still use a large buffer
in binary128 mode for %g, though.

Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>

18 files changed, 950 insertions, 1480 deletions

diff --git a/elf/Makefile b/elf/Makefile
index 0bfaffbd42..ea64d8a470 100644
--- a/elf/Makefile
+++ b/elf/Makefile
@@ -599,14 +599,12 @@ $(objpfx)tst-relro-libc.out: tst-relro-symbols.py $(..)/scripts/glibcelf.py \
 	    --required=_IO_file_jumps \
 	    --required=_IO_file_jumps_maybe_mmap \
 	    --required=_IO_file_jumps_mmap \
-	    --required=_IO_helper_jumps \
 	    --required=_IO_mem_jumps \
 	    --required=_IO_obstack_jumps \
 	    --required=_IO_printf_buffer_as_file_jumps \
 	    --required=_IO_proc_jumps \
 	    --required=_IO_str_chk_jumps \
 	    --required=_IO_str_jumps \
-	    --required=_IO_strn_jumps \
 	    --required=_IO_wfile_jumps \
 	    --required=_IO_wfile_jumps_maybe_mmap \
 	    --required=_IO_wfile_jumps_mmap \
diff --git a/include/printf.h b/include/printf.h
index 5127a45f9b..2c998059d4 100644
--- a/include/printf.h
+++ b/include/printf.h
@@ -65,18 +65,31 @@ int __translated_number_width (locale_t loc,
 			       const char *first, const char *last)
   attribute_hidden;
 
-extern int __printf_fphex (FILE *, const struct printf_info *,
-			   const void *const *) attribute_hidden;
+
+struct __printf_buffer;
+void __printf_buffer (struct __printf_buffer *buf, const char *format,
+		      va_list ap, unsigned int mode_flags);
+struct __wprintf_buffer;
+void __wprintf_buffer (struct __wprintf_buffer *buf, const wchar_t *format,
+		       va_list ap, unsigned int mode_flags);
+
 extern int __printf_fp (FILE *, const struct printf_info *,
 			const void *const *);
 libc_hidden_proto (__printf_fp)
-extern int __printf_fp_l (FILE *, locale_t, const struct printf_info *,
-			  const void *const *);
-libc_hidden_proto (__printf_fp_l)
 
-extern unsigned int __guess_grouping (unsigned int intdig_max,
-				      const char *grouping)
-     attribute_hidden;
+void __printf_fphex_l_buffer (struct __printf_buffer *, locale_t,
+			      const struct printf_info *,
+			      const void *const *) attribute_hidden;
+void __printf_fp_l_buffer (struct __printf_buffer *, locale_t,
+			   const struct printf_info *,
+			   const void *const *) attribute_hidden;
+struct __wprintf_buffer;
+void __wprintf_fphex_l_buffer (struct __wprintf_buffer *, locale_t,
+			       const struct printf_info *,
+			       const void *const *) attribute_hidden;
+void __wprintf_fp_l_buffer (struct __wprintf_buffer *, locale_t,
+			    const struct printf_info *,
+			    const void *const *) attribute_hidden;
 
 # endif /* !_ISOMAC */
 #endif
diff --git a/include/printf_buffer.h b/include/printf_buffer.h
index e27f2a899c..39ef232587 100644
--- a/include/printf_buffer.h
+++ b/include/printf_buffer.h
@@ -45,7 +45,12 @@
 enum __printf_buffer_mode
   {
     __printf_buffer_mode_failed,
+    __printf_buffer_mode_snprintf,
     __printf_buffer_mode_to_file,
+    __printf_buffer_mode_strfmon,
+    __printf_buffer_mode_fp,         /* For __printf_fp_l_buffer.  */
+    __printf_buffer_mode_fp_to_wide, /* For __wprintf_fp_l_buffer.  */
+    __printf_buffer_mode_fphex_to_wide, /* For __wprintf_fphex_l_buffer.  */
   };
 
 /* Buffer for fast character writing with overflow handling.
@@ -268,13 +273,45 @@ bool __wprintf_buffer_flush (struct __wprintf_buffer *buf) attribute_hidden;
 #define Xprintf_buffer_puts Xprintf (buffer_puts)
 #define Xprintf_buffer_write Xprintf (buffer_write)
 
+/* Commonly used buffers.  */
+
+struct __printf_buffer_snprintf
+{
+  struct __printf_buffer base;
+#define PRINTF_BUFFER_SIZE_DISCARD 128
+  char discard[PRINTF_BUFFER_SIZE_DISCARD]; /* Used in counting mode.  */
+};
+
+/* Sets up [BUFFER, BUFFER + LENGTH) as the write target.  If LENGTH
+   is positive, also writes a NUL byte to *BUFFER.  */
+void __printf_buffer_snprintf_init (struct __printf_buffer_snprintf *,
+                                    char *buffer, size_t length)
+  attribute_hidden;
+
+/* Add the null terminator after everything has been written.  The
+   return value is the one expected by printf (see __printf_buffer_done).  */
+int __printf_buffer_snprintf_done (struct __printf_buffer_snprintf *)
+  attribute_hidden;
+
 /* Flush function implementations follow.  They are called from
    __printf_buffer_flush.  Generic code should not call these flush
    functions directly.  Some modes have inline implementations.  */
 
+void __printf_buffer_flush_snprintf (struct __printf_buffer_snprintf *)
+  attribute_hidden;
 struct __printf_buffer_to_file;
 void __printf_buffer_flush_to_file (struct __printf_buffer_to_file *)
   attribute_hidden;
+struct __printf_buffer_fp;
+void __printf_buffer_flush_fp (struct __printf_buffer_fp *)
+  attribute_hidden;
+struct __printf_buffer_fp_to_wide;
+void __printf_buffer_flush_fp_to_wide (struct __printf_buffer_fp_to_wide *)
+  attribute_hidden;
+struct __printf_buffer_fphex_to_wide;
+void __printf_buffer_flush_fphex_to_wide (struct
+                                          __printf_buffer_fphex_to_wide *)
+  attribute_hidden;
 
 struct __wprintf_buffer_to_file;
 void __wprintf_buffer_flush_to_file (struct __wprintf_buffer_to_file *)
@@ -282,10 +319,15 @@ void __wprintf_buffer_flush_to_file (struct __wprintf_buffer_to_file *)
 
 /* Buffer sizes.  These can be tuned as necessary.  There is a tension
    here between stack consumption, cache usage, and additional system
-   calls or heap allocations (if the buffer is too small).  */
+   calls or heap allocations (if the buffer is too small).
+
+   Also see PRINTF_BUFFER_SIZE_DISCARD above for snprintf.  */
 
 /* Fallback buffer if the underlying FILE * stream does not provide
    buffer space.  */
 #define PRINTF_BUFFER_SIZE_TO_FILE_STAGE 128
 
+/* Temporary buffer used during floating point digit translation.  */
+#define PRINTF_BUFFER_SIZE_DIGITS 64
+
 #endif /* PRINTF_BUFFER_H */
diff --git a/libio/strfile.h b/libio/strfile.h
index fa44ebc7e7..b7a57317d0 100644
--- a/libio/strfile.h
+++ b/libio/strfile.h
@@ -70,9 +70,6 @@ typedef struct
   char overflow_buf[64];
 } _IO_strnfile;
 
-extern const struct _IO_jump_t _IO_strn_jumps attribute_hidden;
-
-
 typedef struct
 {
   _IO_strfile f;
diff --git a/libio/tst-vtables-common.c b/libio/tst-vtables-common.c
index d18df23e55..a310e516f2 100644
--- a/libio/tst-vtables-common.c
+++ b/libio/tst-vtables-common.c
@@ -409,11 +409,14 @@ void _IO_init (FILE *fp, int flags);
 static void
 with_compatibility_fprintf (void *closure)
 {
+  /* A temporary staging buffer is used in the current fprintf
+     implementation, which is why there is just one call to
+     xsputn.  */
   TEST_COMPARE (fprintf_ptr (shared->fp, "A%sCD", "B"), 4);
-  TEST_COMPARE (shared->calls, 3);
-  TEST_COMPARE (shared->calls_xsputn, 3);
+  TEST_COMPARE (shared->calls, 1);
+  TEST_COMPARE (shared->calls_xsputn, 1);
   TEST_COMPARE_BLOB (shared->buffer, shared->buffer_length,
-                     "CD", 2);
+                     "ABCD", 4);
 }
 
 static void
diff --git a/libio/vsnprintf.c b/libio/vsnprintf.c
index 8dae66761d..7a9667f966 100644
--- a/libio/vsnprintf.c
+++ b/libio/vsnprintf.c
@@ -25,97 +25,76 @@
    in files containing the exception.  */
 
 #include "libioP.h"
-#include "strfile.h"
 
-static int _IO_strn_overflow (FILE *fp, int c) __THROW;
+#include <array_length.h>
+#include <printf.h>
+#include <printf_buffer.h>
 
-static int
-_IO_strn_overflow (FILE *fp, int c)
+void
+__printf_buffer_flush_snprintf (struct __printf_buffer_snprintf *buf)
 {
-  /* When we come to here this means the user supplied buffer is
-     filled.  But since we must return the number of characters which
-     would have been written in total we must provide a buffer for
-     further use.  We can do this by writing on and on in the overflow
-     buffer in the _IO_strnfile structure.  */
-  _IO_strnfile *snf = (_IO_strnfile *) fp;
-
-  if (fp->_IO_buf_base != snf->overflow_buf)
+  /* Record the bytes written so far, before switching buffers.  */
+  buf->base.written += buf->base.write_ptr - buf->base.write_base;
+
+  if (buf->base.write_base != buf->discard)
     {
-      /* Terminate the string.  We know that there is room for at
-	 least one more character since we initialized the stream with
-	 a size to make this possible.  */
-      *fp->_IO_write_ptr = '\0';
-
-      _IO_setb (fp, snf->overflow_buf,
-		snf->overflow_buf + sizeof (snf->overflow_buf), 0);
-
-      fp->_IO_write_base = snf->overflow_buf;
-      fp->_IO_read_base = snf->overflow_buf;
-      fp->_IO_read_ptr = snf->overflow_buf;
-      fp->_IO_read_end = snf->overflow_buf + sizeof (snf->overflow_buf);
-    }
+      /* We just finished writing the caller-supplied buffer.  Force
+	 NUL termination if the string length is not zero.  */
+      if (buf->base.write_base != buf->base.write_end)
+	buf->base.write_end[-1] = '\0';
 
-  fp->_IO_write_ptr = snf->overflow_buf;
-  fp->_IO_write_end = snf->overflow_buf;
 
-  /* Since we are not really interested in storing the characters
-     which do not fit in the buffer we simply ignore it.  */
-  return c;
-}
+      /* Switch to the discard buffer.  */
+      buf->base.write_base = buf->discard;
+      buf->base.write_ptr = buf->discard;
+      buf->base.write_end = array_end (buf->discard);
+    }
 
+  buf->base.write_base = buf->discard;
+  buf->base.write_ptr = buf->discard;
+}
 
-const struct _IO_jump_t _IO_strn_jumps libio_vtable attribute_hidden =
+void
+__printf_buffer_snprintf_init (struct __printf_buffer_snprintf *buf,
+			       char *buffer, size_t length)
 {
-  JUMP_INIT_DUMMY,
-  JUMP_INIT(finish, _IO_str_finish),
-  JUMP_INIT(overflow, _IO_strn_overflow),
-  JUMP_INIT(underflow, _IO_str_underflow),
-  JUMP_INIT(uflow, _IO_default_uflow),
-  JUMP_INIT(pbackfail, _IO_str_pbackfail),
-  JUMP_INIT(xsputn, _IO_default_xsputn),
-  JUMP_INIT(xsgetn, _IO_default_xsgetn),
-  JUMP_INIT(seekoff, _IO_str_seekoff),
-  JUMP_INIT(seekpos, _IO_default_seekpos),
-  JUMP_INIT(setbuf, _IO_default_setbuf),
-  JUMP_INIT(sync, _IO_default_sync),
-  JUMP_INIT(doallocate, _IO_default_doallocate),
-  JUMP_INIT(read, _IO_default_read),
-  JUMP_INIT(write, _IO_default_write),
-  JUMP_INIT(seek, _IO_default_seek),
-  JUMP_INIT(close, _IO_default_close),
-  JUMP_INIT(stat, _IO_default_stat),
-  JUMP_INIT(showmanyc, _IO_default_showmanyc),
-  JUMP_INIT(imbue, _IO_default_imbue)
-};
+  __printf_buffer_init (&buf->base, buffer, length,
+			__printf_buffer_mode_snprintf);
+  if (length > 0)
+    /* Historic behavior for trivially overlapping buffers (checked by
+       the test suite).  */
+    *buffer = '\0';
+}
 
+int
+__printf_buffer_snprintf_done (struct __printf_buffer_snprintf *buf)
+{
+  /* NB: Do not check for buf->base.fail here.  Write the null
+     terminator even in case of errors. */
+
+  if (buf->base.write_ptr < buf->base.write_end)
+    *buf->base.write_ptr = '\0';
+  else if (buf->base.write_ptr > buf->base.write_base)
+    /* If write_ptr == write_base, nothing has been written.  No null
+       termination is needed because of the early truncation in
+       __printf_buffer_snprintf_init (the historic behavior).
+
+       We might also be at the start of the discard buffer, but in
+       this case __printf_buffer_flush_snprintf has already written
+       the NUL terminator.  */
+    buf->base.write_ptr[-1] = '\0';
+
+  return __printf_buffer_done (&buf->base);
+}
 
 int
 __vsnprintf_internal (char *string, size_t maxlen, const char *format,
 		      va_list args, unsigned int mode_flags)
 {
-  _IO_strnfile sf;
-  int ret;
-#ifdef _IO_MTSAFE_IO
-  sf.f._sbf._f._lock = NULL;
-#endif
-
-  /* We need to handle the special case where MAXLEN is 0.  Use the
-     overflow buffer right from the start.  */
-  if (maxlen == 0)
-    {
-      string = sf.overflow_buf;
-      maxlen = sizeof (sf.overflow_buf);
-    }
-
-  _IO_no_init (&sf.f._sbf._f, _IO_USER_LOCK, -1, NULL, NULL);
-  _IO_JUMPS (&sf.f._sbf) = &_IO_strn_jumps;
-  string[0] = '\0';
-  _IO_str_init_static_internal (&sf.f, string, maxlen - 1, string);
-  ret = __vfprintf_internal (&sf.f._sbf._f, format, args, mode_flags);
-
-  if (sf.f._sbf._f._IO_buf_base != sf.overflow_buf)
-    *sf.f._sbf._f._IO_write_ptr = '\0';
-  return ret;
+  struct __printf_buffer_snprintf buf;
+  __printf_buffer_snprintf_init (&buf, string, maxlen);
+  __printf_buffer (&buf.base, format, args, mode_flags);
+  return __printf_buffer_snprintf_done (&buf);
 }
 
 int
diff --git a/stdio-common/printf_buffer_flush.c b/stdio-common/printf_buffer_flush.c
index 9b25c0fde5..bfd1f9d733 100644
--- a/stdio-common/printf_buffer_flush.c
+++ b/stdio-common/printf_buffer_flush.c
@@ -16,6 +16,7 @@
    License along with the GNU C Library; if not, see
    <https://www.gnu.org/licenses/>.  */
 
+#include <errno.h>
 #include <printf_buffer.h>
 
 #include "printf_buffer-char.h"
@@ -24,7 +25,11 @@
 /* The __printf_buffer_flush_* functions are defined together with
    functions that are pulled in by strong references.  */
 #ifndef SHARED
+# pragma weak __printf_buffer_flush_snprintf
 # pragma weak __printf_buffer_flush_to_file
+# pragma weak __printf_buffer_flush_fp
+# pragma weak __printf_buffer_flush_fp_to_wide
+# pragma weak __printf_buffer_flush_fphex_to_wide
 #endif /* !SHARED */
 
 static void
@@ -34,9 +39,27 @@ __printf_buffer_do_flush (struct __printf_buffer *buf)
     {
     case __printf_buffer_mode_failed:
       return;
+    case __printf_buffer_mode_snprintf:
+      __printf_buffer_flush_snprintf ((struct __printf_buffer_snprintf *) buf);
+      return;
     case __printf_buffer_mode_to_file:
       __printf_buffer_flush_to_file ((struct __printf_buffer_to_file *) buf);
       return;
+    case __printf_buffer_mode_strfmon:
+      __set_errno (E2BIG);
+      __printf_buffer_mark_failed (buf);
+      return;
+    case __printf_buffer_mode_fp:
+      __printf_buffer_flush_fp ((struct __printf_buffer_fp *) buf);
+      return;
+    case __printf_buffer_mode_fp_to_wide:
+      __printf_buffer_flush_fp_to_wide
+        ((struct __printf_buffer_fp_to_wide *) buf);
+      return;
+    case __printf_buffer_mode_fphex_to_wide:
+      __printf_buffer_flush_fphex_to_wide
+        ((struct __printf_buffer_fphex_to_wide *) buf);
+      return;
     }
   __builtin_trap ();
 }
diff --git a/stdio-common/printf_fp.c b/stdio-common/printf_fp.c
index 3a5560fc16..e1abb1f240 100644
--- a/stdio-common/printf_fp.c
+++ b/stdio-common/printf_fp.c
@@ -41,90 +41,12 @@
 #include <wchar.h>
 #include <stdbool.h>
 #include <rounding-mode.h>
+#include <printf_buffer.h>
+#include <printf_buffer_to_file.h>
+#include <grouping_iterator.h>
 
-#ifdef COMPILE_WPRINTF
-# define CHAR_T        wchar_t
-#else
-# define CHAR_T        char
-#endif
-
-#include "_i18n_number.h"
-
-#ifndef NDEBUG
-# define NDEBUG			/* Undefine this for debugging assertions.  */
-#endif
 #include <assert.h>
 
-#define PUT(f, s, n) _IO_sputn (f, s, n)
-#define PAD(f, c, n) (wide ? _IO_wpadn (f, c, n) : _IO_padn (f, c, n))
-#undef putc
-#define putc(c, f) (wide \
-		    ? (int)_IO_putwc_unlocked (c, f) : _IO_putc_unlocked (c, f))
-
-
-/* Macros for doing the actual output.  */
-
-#define outchar(ch)							      \
-  do									      \
-    {									      \
-      const int outc = (ch);						      \
-      if (putc (outc, fp) == EOF)					      \
-	{								      \
-	  if (buffer_malloced)						      \
-	    {								      \
-	      free (buffer);						      \
-	      free (wbuffer);						      \
-	    }								      \
-	  return -1;							      \
-	}								      \
-      ++done;								      \
-    } while (0)
-
-#define PRINT(ptr, wptr, len)						      \
-  do									      \
-    {									      \
-      size_t outlen = (len);						      \
-      if (len > 20)							      \
-	{								      \
-	  if (PUT (fp, wide ? (const char *) wptr : ptr, outlen) != outlen)   \
-	    {								      \
-	      if (buffer_malloced)					      \
-		{							      \
-		  free (buffer);					      \
-		  free (wbuffer);					      \
-		}							      \
-	      return -1;						      \
-	    }								      \
-	  ptr += outlen;						      \
-	  done += outlen;						      \
-	}								      \
-      else								      \
-	{								      \
-	  if (wide)							      \
-	    while (outlen-- > 0)					      \
-	      outchar (*wptr++);					      \
-	  else								      \
-	    while (outlen-- > 0)					      \
-	      outchar (*ptr++);						      \
-	}								      \
-    } while (0)
-
-#define PADN(ch, len)							      \
-  do									      \
-    {									      \
-      if (PAD (fp, ch, len) != len)					      \
-	{								      \
-	  if (buffer_malloced)						      \
-	    {								      \
-	      free (buffer);						      \
-	      free (wbuffer);						      \
-	    }								      \
-	  return -1;							      \
-	}								      \
-      done += len;							      \
-    }									      \
-  while (0)
-
 /* We use the GNU MP library to handle large numbers.
 
    An MP variable occupies a varying number of entries in its array.  We keep
@@ -145,10 +67,6 @@ extern mp_size_t __mpn_extract_long_double (mp_ptr res_ptr, mp_size_t size,
 					    long double value);
 
 
-static wchar_t *group_number (wchar_t *buf, wchar_t *bufend,
-			      unsigned int intdig_no, const char *grouping,
-			      wchar_t thousands_sep, int ngroups);
-
 struct hack_digit_param
 {
   /* Sign of the exponent.  */
@@ -165,7 +83,7 @@ struct hack_digit_param
   MPN_VAR(tmp);
 };
 
-static wchar_t
+static char
 hack_digit (struct hack_digit_param *p)
 {
   mp_limb_t hi;
@@ -197,7 +115,7 @@ hack_digit (struct hack_digit_param *p)
 	      /* We're not prepared for an mpn variable with zero
 		 limbs.  */
 	      p->fracsize = 1;
-	      return L'0' + hi;
+	      return '0' + hi;
 	    }
 	}