Update.

2000-04-09  Ulrich Drepper  <drepper@redhat.com>

	Implement handling of restartable conversion functions according to
	ISO C.
	* iconv/gconv.h (__gconv_fct): Add additional parameter.
	* iconv/gconv_int.h (__BUILTIN_TRANS): Likewise.
	* iconv/gconv.c: Pass additional parameter to conversion function.
	* iconv/gconv_simple.c (internal_ucs4_loop_single): New function.
	(internal_ucs4le_loop_single): New function.
	(__gconv_transform_ascii_internal): Define ONE_DIRECTION.
	(__gconv_transform_internal_ascii): Likewise.
	(__gconv_transform_internal_utf8): Likewise.
	(__gconv_transform_utf8_internal): Likewise.
	(__gconv_transform_ucs2_internal): Likewise.
	(__gconv_transform_internal_ucs2): Likewise.
	(__gconv_transform_ucs2reverse_internal): Likewise.
	(__gconv_transform_internal_ucs2reverse): Likewise.
	(internal_ucs4le_loop_unaligned): Before return
	__GCONV_INCOMPLETE_INPUT check that the remaining bytes really form
	a valid character.  Otherwise return __GCONV_ILLEGAL_INPUT.
	(__gconv_transform_utf8_internal): Define STORE_REST and UNPACK_BYTES.
	* iconv/loop.c: Fit in definition of function to convert one character
	for processing of left-over bytes from the state object.
	* iconv/skeleton.c (gconv): Rename inbuf to inptrp and inbufend to
	inend to match names in loop functions.
	(RESET_INPUT_BUFFER): Change apprpriately.
	(gconv): If needed, call function to process bytes from the state
	object.  Similar at the end: store left over bytes if input is
	incomplete.
	Take extra argument and add new argument to all calls of the
	conversion function.
	* iconvdata/iso-2022-cn.c: Adjust numeric values used to store
	information in the state object to not conflict with length count.
	* iconvdata/iso-2022-jp.c: Likewise.
	* iconvdata/iso-2022-kr.c: Likewise.
	* iconvdata/unicode.c: Adjust for change change in parameters of
	skeleton function.
	* iconvdata/utf-16.c: Likewise.
	* libio/iofwide.c: Add new parameter to all calls of conversion
	function.
	* wcsmbs/btowc.c: Likewise.
	* wcsmbs/mbrtowc.c: Likewise.
	* wcsmbs/mbsnrtowcs.c: Likewise.
	* wcsmbs/mbsrtowcs.c: Likewise.
	* wcsmbs/wcrtomb.c: Likewise.
	* wcsmbs/wcsnrtombs.c: Likewise.
	* wcsmbs/wcsrtombs.c: Likewise.
	* wcsmbs/wctob.c: Likewise.

	* iconvdata/gbgbk.c: Always define MAX_NEEDED_OUTPUT and
	MAX_NEEDED_INPUT.

21 files changed, 487 insertions, 88 deletions

diff --git a/ChangeLog b/ChangeLog
index eb8940083e..7e022fd475 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,55 @@
+2000-04-09  Ulrich Drepper  <drepper@redhat.com>
+
+	Implement handling of restartable conversion functions according to
+	ISO C.
+	* iconv/gconv.h (__gconv_fct): Add additional parameter.
+	* iconv/gconv_int.h (__BUILTIN_TRANS): Likewise.
+	* iconv/gconv.c: Pass additional parameter to conversion function.
+	* iconv/gconv_simple.c (internal_ucs4_loop_single): New function.
+	(internal_ucs4le_loop_single): New function.
+	(__gconv_transform_ascii_internal): Define ONE_DIRECTION.
+	(__gconv_transform_internal_ascii): Likewise.
+	(__gconv_transform_internal_utf8): Likewise.
+	(__gconv_transform_utf8_internal): Likewise.
+	(__gconv_transform_ucs2_internal): Likewise.
+	(__gconv_transform_internal_ucs2): Likewise.
+	(__gconv_transform_ucs2reverse_internal): Likewise.
+	(__gconv_transform_internal_ucs2reverse): Likewise.
+	(internal_ucs4le_loop_unaligned): Before return
+	__GCONV_INCOMPLETE_INPUT check that the remaining bytes really form
+	a valid character.  Otherwise return __GCONV_ILLEGAL_INPUT.
+	(__gconv_transform_utf8_internal): Define STORE_REST and UNPACK_BYTES.
+	* iconv/loop.c: Fit in definition of function to convert one character
+	for processing of left-over bytes from the state object.
+	* iconv/skeleton.c (gconv): Rename inbuf to inptrp and inbufend to
+	inend to match names in loop functions.
+	(RESET_INPUT_BUFFER): Change apprpriately.
+	(gconv): If needed, call function to process bytes from the state
+	object.  Similar at the end: store left over bytes if input is
+	incomplete.
+	Take extra argument and add new argument to all calls of the
+	conversion function.
+	* iconvdata/iso-2022-cn.c: Adjust numeric values used to store
+	information in the state object to not conflict with length count.
+	* iconvdata/iso-2022-jp.c: Likewise.
+	* iconvdata/iso-2022-kr.c: Likewise.
+	* iconvdata/unicode.c: Adjust for change change in parameters of
+	skeleton function.
+	* iconvdata/utf-16.c: Likewise.
+	* libio/iofwide.c: Add new parameter to all calls of conversion
+	function.
+	* wcsmbs/btowc.c: Likewise.
+	* wcsmbs/mbrtowc.c: Likewise.
+	* wcsmbs/mbsnrtowcs.c: Likewise.
+	* wcsmbs/mbsrtowcs.c: Likewise.
+	* wcsmbs/wcrtomb.c: Likewise.
+	* wcsmbs/wcsnrtombs.c: Likewise.
+	* wcsmbs/wcsrtombs.c: Likewise.
+	* wcsmbs/wctob.c: Likewise.
+
+	* iconvdata/gbgbk.c: Always define MAX_NEEDED_OUTPUT and
+	MAX_NEEDED_INPUT.
+
 2000-04-07  Andreas Jaeger  <aj@suse.de>
 
 	* sysdeps/unix/sysv/linux/mips/kernel_sigaction.h: Updated for
diff --git a/iconv/gconv.h b/iconv/gconv.h
index 85b9a3a0e0..6d79b07ddd 100644
--- a/iconv/gconv.h
+++ b/iconv/gconv.h
@@ -59,7 +59,7 @@ struct __gconv_loaded_object;
 /* Type of a conversion function.  */
 typedef int (*__gconv_fct) (struct __gconv_step *, struct __gconv_step_data *,
 			    __const unsigned char **, __const unsigned char *,
-			    size_t *, int);
+			    size_t *, int, int);
 
 /* Constructor and destructor for local data for conversion step.  */
 typedef int (*__gconv_init_fct) (struct __gconv_step *);
diff --git a/iconv/gconv_int.h b/iconv/gconv_int.h
index 626e45e45e..bc5d003325 100644
--- a/iconv/gconv_int.h
+++ b/iconv/gconv_int.h
@@ -161,7 +161,7 @@ extern void __gconv_get_builtin_trans (const char *__name,
 		   struct __gconv_step_data *__data,			      \
 		   const unsigned char **__inbuf,			      \
 		   const unsigned char *__inbufend, size_t *__written,	      \
-		   int __do_flush)
+		   int __do_flush, int __consume_incomplete)
 
 __BUILTIN_TRANS (__gconv_transform_ascii_internal);
 __BUILTIN_TRANS (__gconv_transform_internal_ascii);
diff --git a/iconv/gconv_simple.c b/iconv/gconv_simple.c
index 9710eb1707..d06db5a316 100644
--- a/iconv/gconv_simple.c
+++ b/iconv/gconv_simple.c
@@ -145,6 +145,45 @@ internal_ucs4_loop_unaligned (const unsigned char **inptrp,
 }
 #endif
 
+
+static inline int
+internal_ucs4_loop_single (const unsigned char **inptrp,
+			   const unsigned char *inend,
+			   unsigned char **outptrp, unsigned char *outend,
+			   mbstate_t *state, void *data, size_t *converted)
+{
+  size_t cnt = state->__count & 7;
+
+  while (*inptrp < inend && cnt < 4)
+    state->__value.__wchb[cnt++] = *(*inptrp)++;
+
+  if (cnt < 4)
+    {
+      /* Still not enough bytes.  Store the ones in the input buffer.  */
+      state->__count &= ~7;
+      state->__count |= cnt;
+
+      return __GCONV_INCOMPLETE_INPUT;
+    }
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+  (*outptrp)[0] = state->__value.__wchb[3];
+  (*outptrp)[1] = state->__value.__wchb[2];
+  (*outptrp)[2] = state->__value.__wchb[1];
+  (*outptrp)[3] = state->__value.__wchb[0];
+#elif __BYTE_ORDER == __BIG_ENDIAN
+  /* XXX unaligned */
+  *(*((uint32_t **) outptrp)++) = state->__value.__wch;
+#else
+# error "This endianess is not supported."
+#endif
+
+  /* Clear the state buffer.  */
+  state->__count &= ~7;
+
+  return __GCONV_OK;
+}
+
 #include <iconv/skeleton.c>
 
 
@@ -244,6 +283,43 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
 }
 #endif
 
+
+static inline int
+internal_ucs4le_loop_single (const unsigned char **inptrp,
+			     const unsigned char *inend,
+			     unsigned char **outptrp, unsigned char *outend,
+			     mbstate_t *state, void *data, size_t *converted)
+{
+  size_t cnt = state->__count & 7;
+
+  while (*inptrp < inend && cnt < 4)
+    state->__value.__wchb[cnt++] = *(*inptrp)++;
+
+  if (cnt < 4)
+    {
+      /* Still not enough bytes.  Store the ones in the input buffer.  */
+      state->__count &= ~7;
+      state->__count |= cnt;
+
+      return __GCONV_INCOMPLETE_INPUT;
+    }
+
+#if __BYTE_ORDER == __BIG_ENDIAN
+  (*outptrp)[0] = state->__value.__wchb[3];
+  (*outptrp)[1] = state->__value.__wchb[2];
+  (*outptrp)[2] = state->__value.__wchb[1];
+  (*outptrp)[3] = state->__value.__wchb[0];
+#else
+  /* XXX unaligned */
+  *(*((uint32_t **) outptrp)++) = state->__value.__wch;
+#endif
+
+  /* Clear the state buffer.  */
+  state->__count &= ~7;
+
+  return __GCONV_OK;
+}
+
 #include <iconv/skeleton.c>
 
 
@@ -256,6 +332,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
 #define FROM_LOOP		ascii_internal_loop
 #define TO_LOOP			ascii_internal_loop /* This is not used.  */
 #define FUNCTION_NAME		__gconv_transform_ascii_internal
+#define ONE_DIRECTION		1
 
 #define MIN_NEEDED_INPUT	MIN_NEEDED_FROM
 #define MIN_NEEDED_OUTPUT	MIN_NEEDED_TO
@@ -270,6 +347,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
       }									      \
 									      \
     /* It's an one byte sequence.  */					      \
+    /* XXX unaligned.  */						      \
     *((uint32_t *) outptr)++ = *inptr++;				      \
   }
 #include <iconv/loop.c>
@@ -285,6 +363,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
 #define FROM_LOOP		internal_ascii_loop
 #define TO_LOOP			internal_ascii_loop /* This is not used.  */
 #define FUNCTION_NAME		__gconv_transform_internal_ascii
+#define ONE_DIRECTION		1
 
 #define MIN_NEEDED_INPUT	MIN_NEEDED_FROM
 #define MIN_NEEDED_OUTPUT	MIN_NEEDED_TO
@@ -315,6 +394,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
 #define FROM_LOOP		internal_utf8_loop
 #define TO_LOOP			internal_utf8_loop /* This is not used.  */
 #define FUNCTION_NAME		__gconv_transform_internal_utf8
+#define ONE_DIRECTION		1
 
 #define MIN_NEEDED_INPUT	MIN_NEEDED_FROM
 #define MIN_NEEDED_OUTPUT	MIN_NEEDED_TO
@@ -375,6 +455,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
 #define FROM_LOOP		utf8_internal_loop
 #define TO_LOOP			utf8_internal_loop /* This is not used.  */
 #define FUNCTION_NAME		__gconv_transform_utf8_internal
+#define ONE_DIRECTION		1
 
 #define MIN_NEEDED_INPUT	MIN_NEEDED_FROM
 #define MAX_NEEDED_INPUT	MAX_NEEDED_FROM
@@ -438,8 +519,13 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
 									      \
 	if (NEED_LENGTH_TEST && inptr + cnt > inend)			      \
 	  {								      \
-	    /* We don't have enough input.  */				      \
-	    result = __GCONV_INCOMPLETE_INPUT;				      \
+	    /* We don't have enough input.  But before we report that check   \
+	       that all the bytes are correct.  */			      \
+	    for (i = 1; inptr + i < inend; ++i)				      \
+	      if ((inptr[i] & 0xc0) != 0x80)				      \
+		break;							      \
+	    result = (inptr + i == inend				      \
+		      ? __GCONV_INCOMPLETE_INPUT : __GCONV_ILLEGAL_INPUT);    \
 	    break;							      \
 	  }								      \
 									      \
@@ -472,6 +558,89 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
     /* Now adjust the pointers and store the result.  */		      \
     *((uint32_t *) outptr)++ = ch;					      \
   }
+
+#define STORE_REST \
+  {									      \
+    /* We store the remaining bytes while converting them into the UCS4	      \
+       format.  We can assume that the first byte in the buffer is	      \
+       correct and that it requires a larger number of bytes than there	      \
+       are in the input buffer.  */					      \
+    wint_t ch = **inptrp;						      \
+    size_t cnt;								      \
+									      \
+    state->__count = inend - *inptrp;					      \
+									      \
+    if (ch >= 0xc2 && ch < 0xe0)					      \
+      {									      \
+	/* We expect two bytes.  The first byte cannot be 0xc0 or	      \
+	   0xc1, otherwise the wide character could have been		      \
+	   represented using a single byte.  */				      \
+	cnt = 2;							      \
+	ch &= 0x1f;							      \
+      }									      \
+    else if ((ch & 0xf0) == 0xe0)					      \
+      {									      \
+	/* We expect three bytes.  */					      \
+	cnt = 3;							      \
+	ch &= 0x0f;							      \
+      }									      \
+    else if ((ch & 0xf8) == 0xf0)					      \
+      {									      \
+	/* We expect four bytes.  */					      \
+	cnt = 4;							      \
+	ch &= 0x07;							      \
+      }									      \
+    else if ((ch & 0xfc) == 0xf8)					      \
+      {									      \
+	/* We expect five bytes.  */					      \
+	cnt = 5;							      \
+	ch &= 0x03;							      \
+      }									      \
+    else								      \
+      {									      \
+	/* We expect six bytes.  */					      \
+	cnt = 6;							      \
+	ch &= 0x01;							      \
+      }									      \
+									      \
+    /* The first byte is already consumed.  */				      \
+    --cnt;								      \
+    while (++(*inptrp) < inend)						      \
+      {									      \
+	ch <<= 6;							      \
+	ch |= **inptrp & 0x3f;						      \
+	--cnt;								      \
+      }									      \
+									      \
+    /* Shift for the so far missing bytes.  */				      \
+    ch <<= cnt * 6;							      \
+									      \
+    /* Store the value.  */						      \
+    state->__value.__wch = ch;						      \
+  }
+
+#define UNPACK_BYTES \
+  {									      \
+    wint_t wch = state->__value.__wch;					      \
+    inlen = state->__count;						      \
+									      \
+    if (state->__value.__wch <= 0x7ff)					      \
+      bytebuf[0] = 0xc0;						      \
+    else if (state->__value.__wch <= 0xffff)				      \
+      bytebuf[0] = 0xe0;						      \
+    else if (state->__value.__wch <= 0x1fffff)				      \
+      bytebuf[0] = 0xf0;						      \
+    else if (state->__value.__wch <= 0x3ffffff)				      \
+      bytebuf[0] = 0xf8;						      \
+    else								      \
+      bytebuf[0] = 0xfc;						      \
+									      \
+    while (inlen-- > 1)							      \
+      bytebuf[inlen] = 0x80 | (wch & 0x3f);				      \
+									      \
+    bytebuf[0] |= wch;							      \
+  }
+
 #include <iconv/loop.c>
 #include <iconv/skeleton.c>
 
@@ -485,6 +654,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
 #define FROM_LOOP		ucs2_internal_loop
 #define TO_LOOP			ucs2_internal_loop /* This is not used.  */
 #define FUNCTION_NAME		__gconv_transform_ucs2_internal
+#define ONE_DIRECTION		1
 
 #define MIN_NEEDED_INPUT	MIN_NEEDED_FROM
 #define MIN_NEEDED_OUTPUT	MIN_NEEDED_TO
@@ -504,6 +674,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
 #define FROM_LOOP		internal_ucs2_loop
 #define TO_LOOP			internal_ucs2_loop /* This is not used.  */
 #define FUNCTION_NAME		__gconv_transform_internal_ucs2
+#define ONE_DIRECTION		1
 
 #define MIN_NEEDED_INPUT	MIN_NEEDED_FROM
 #define MIN_NEEDED_OUTPUT	MIN_NEEDED_TO
@@ -530,6 +701,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
 #define FROM_LOOP		ucs2reverse_internal_loop
 #define TO_LOOP			ucs2reverse_internal_loop/* This is not used.*/
 #define FUNCTION_NAME		__gconv_transform_ucs2reverse_internal
+#define ONE_DIRECTION		1
 
 #define MIN_NEEDED_INPUT	MIN_NEEDED_FROM
 #define MIN_NEEDED_OUTPUT	MIN_NEEDED_TO
@@ -550,6 +722,7 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp,
 #define FROM_LOOP		internal_ucs2reverse_loop
 #define TO_LOOP			internal_ucs2reverse_loop/* This is not used.*/
 #define FUNCTION_NAME		__gconv_transform_internal_ucs2reverse
+#define ONE_DIRECTION		1
 
 #define MIN_NEEDED_INPUT	MIN_NEEDED_FROM
 #define MIN_NEEDED_OUTPUT	MIN_NEEDED_TO
diff --git a/iconv/loop.c b/iconv/loop.c
index 1e5403310e..ffdd24d7e3 100644
--- a/iconv/loop.c
+++ b/iconv/loop.c
@@ -45,6 +45,7 @@
      UPDATE_PARAMS	code to store result in params.
 */
 
+#include <assert.h>
 #include <endian.h>
 #include <gconv.h>
 #include <stdint.h>
@@ -261,23 +262,109 @@ FCTNAME (LOOPFCT) (const unsigned char **inptrp, const unsigned char *inend,
 }
 
 
-#undef get16
-#undef get32
-#undef put16
-#undef put32
-#undef unaligned
-
 /* Include the file a second time to define the function to define the
    function to handle unaligned access.  */
 #if !defined DEFINE_UNALIGNED && !defined _STRING_ARCH_unaligned \
     && MIN_NEEDED_FROM != 1 && MAX_NEEDED_FROM % MIN_NEEDED_FROM == 0 \
     && MIN_NEEDED_TO != 1 && MAX_NEEDED_TO % MIN_NEEDED_TO == 0
+# undef get16
+# undef get32
+# undef put16
+# undef put32
+# undef unaligned
+
 # define DEFINE_UNALIGNED
 # include "loop.c"
 # undef DEFINE_UNALIGNED
 #endif
 
 
+#if MAX_NEEDED_INPUT > 1
+# define SINGLE(fct) SINGLE2 (fct)
+# define SINGLE2(fct) fct##_single
+static inline int
+SINGLE(LOOPFCT) (const unsigned char **inptrp, const unsigned char *inend,
+		 unsigned char **outptrp, unsigned char *outend,
+		 mbstate_t *state, void *data, size_t *converted
+		 EXTRA_LOOP_DECLS)
+{
+  int result = __GCONV_OK;
+  unsigned char bytebuf[MAX_NEEDED_INPUT];
+  const unsigned char *inptr = *inptrp;
+  unsigned char *outptr = *outptrp;
+  size_t inlen;
+
+#ifdef INIT_PARAMS
+  INIT_PARAMS;
+#endif
+
+#ifdef UNPACK_BYTES
+  UNPACK_BYTES
+#else
+  /* Add the bytes from the state to the input buffer.  */
+  for (inlen = 0; inlen < (state->__count & 7); ++ inlen)
+    bytebuf[inlen] = state->__value.__wchb[inlen];
+#endif
+
+  /* Are there enough bytes in the input buffer?  */
+  if (inptr + (MAX_NEEDED_INPUT - inlen) > inend)
+    {
+#ifdef STORE_REST
+      *inptrp = inend;
+      inptr = bytebuf;
+      inptrp = &inptr;
+      inend = &bytebuf[inlen];
+
+      STORE_REST
+#else
+      /* We don't have enough input for another complete input
+	 character.  */
+      while (inptr < inend)
+	state->__value.__wchb[inlen++] = *inptr++;
+#endif
+
+      return __GCONV_INCOMPLETE_INPUT;
+    }
+
+  /* Enough space in output buffer.  */
+  if ((MIN_NEEDED_OUTPUT != 1 && outptr + MIN_NEEDED_OUTPUT > outend)
+      || (MIN_NEEDED_OUTPUT == 1 && outptr >= outend))
+    /* Overflow in the output buffer.  */
+    return __GCONV_FULL_OUTPUT;
+
+  /*  Now add characters from the normal input buffer.  */
+  do
+    bytebuf[inlen++] = *inptr++;
+  while (inlen < MAX_NEEDED_INPUT);
+
+  inptr = bytebuf;
+  inend = &inptr[MAX_NEEDED_INPUT];
+  do
+    {
+      BODY
+    }
+  while (0);
+
+  if (result == __GCONV_OK)
+    {
+      /* We successfully converted the character (maybe even more).
+	 Update the pointers passed in.  */
+      assert (inptr - bytebuf > (state->__count & 7));
+
+      *inptrp += inptr - bytebuf - (state->__count & 7);
+      *outptrp = outptr;
+
+      /* Clear the state buffer.  */
+      state->__count &= ~7;
+    }
+
+  return result;
+}
+# undef SINGLE
+# undef SINGLE2
+#endif
+
+
 /* We remove the macro definitions so that we can include this file again
    for the definition of another function.  */
 #undef MIN_NEEDED_INPUT
@@ -290,3 +377,9 @@ FCTNAME (LOOPFCT) (const unsigned char **inptrp, const unsigned char *inend,
 #undef EXTRA_LOOP_DECLS
 #undef INIT_PARAMS
 #undef UPDATE_PARAMS
+#undef get16
+#undef get32
+#undef put16
+#undef put32
+#undef unaligned
+#undef UNPACK_BYTES
diff --git a/iconv/skeleton.c b/iconv/skeleton.c
index 27b1cab234..50ee45d2c9 100644
--- a/iconv/skeleton.c
+++ b/iconv/skeleton.c
@@ -192,11 +192,11 @@ static int to_object;
    (outbuf - outerr) is always divisible by MIN_NEEDED_TO.  */
 #  define RESET_INPUT_BUFFER \
   if (MIN_NEEDED_FROM % MIN_NEEDED_TO == 0)				      \
-    *inbuf -= (outbuf - outerr) * (MIN_NEEDED_FROM / MIN_NEEDED_TO);	      \
+    *inptrp -= (outbuf - outerr) * (MIN_