diff options
| author | Ulrich Drepper <drepper@redhat.com> | 2000-06-12 19:47:50 +0000 |
|---|---|---|
| committer | Ulrich Drepper <drepper@redhat.com> | 2000-06-12 19:47:50 +0000 |
| commit | 55985355ade2a038b567dd9b58153a98384ae703 (patch) | |
| tree | b6252e23490e6d10d55ae926e30e32173f504190 /iconv | |
| parent | a5b97402f70a3cd43ffee9ccb71560457b4cd88c (diff) | |
| download | glibc-55985355ade2a038b567dd9b58153a98384ae703.tar.xz glibc-55985355ade2a038b567dd9b58153a98384ae703.zip | |
Update.
2000-06-12 Ulrich Drepper <drepper@redhat.com>
* Rules (%.out): Define GCONV_PATH in the environment.
* assert/Depend: New file.
* iconvdata/Depend: New file.
* intl/Depend: New file.
* timezone/Makefile (build-testdata): Add GCONV_PATH to environment.
* intl/tst-gettext.sh: Likewise.
* iconv/Makefile (routines): Add gconv_trans.
* iconv/gconv_trans.c: New file.
* iconv/gconv.h (struct __gconv_trans_data): New type.
(__gconv_fct): New parameter with starting position in output buffer.
(__gconv_trans_fct, __gconv_trans_context_fct, __gconv_trans_query_fct,
__gconv_trans_init_fct, __gconv_trans_end_fct): New types.
(struct __gconv_step): Add new member __trans.
* iconv/gconv_int.h: Pretty print prototypes.
(gconv_transliterate): New prototype.
(__BUILTIN_TRANS): Update for new conversion function interface.
* iconv/gconv.c (__gconv): Pass new parameter to conversion function.
* iconv/gconv_open.c (__gconv_open): Recognize error handling suffix
in names, find appropriate function, and install in the conversion
steps it can be used.
* iconv/skeleton.c: Add additional parameter for beginning of output
buffer. Change calls of downstream functions.
* iconv/loop.c: Change loop function interface completely. Pass in
step and step_data structure. Remove optimization for BODY with
NEED_LENGTH_TEST == 0.
* iconv/gconv_simple.c: Update interfaces of functions. Insert
appropriate error handling code to use transliteration steps. Remove
optimization for BODY with NEED_LENGTH_TEST == 0.
* iconvdata/8bit-gap.c: Likewise.
* iconvdata/8bit-generic.c: Likewise.
* iconvdata/ansi_x3.110.c: Likewise.
* iconvdata/big5.c: Likewise.
* iconvdata/big5hkscs.c: Likewise.
* iconvdata/euc-cn.c: Likewise.
* iconvdata/euc-jp.c: Likewise.
* iconvdata/euc-kr.c: Likewise.
* iconvdata/euc-tw.c: Likewise.
* iconvdata/gbgbk.c: Likewise.
* iconvdata/gbk.c: Likewise.
* iconvdata/iso-2022-cn.c: Likewise.
* iconvdata/iso-2022-jp.c: Likewise.
* iconvdata/iso-2022-kr.c: Likewise.
* iconvdata/iso646.c: Likewise.
* iconvdata/iso8859-1.c: Likewise.
* iconvdata/iso_6937-2.c: Likewise.
* iconvdata/iso_6937.c: Likewise.
* iconvdata/johab.c: Likewise.
* iconvdata/sjis.c: Likewise.
* iconvdata/t.61.c: Likewise.
* iconvdata/uhc.c: Likewise.
* iconvdata/unicode.c: Likewise.
* iconvdata/utf-16.c: Likewise.
* libio/iofwide.c: Adjust to new interface of gconv functions. Use
DL_CALL_FCT.
* wcsmbs/btowc.c: Likewise.
* wcsmbs/mbrtowc.c: Likewise.
* wcsmbs/mbsnrtowcs.c: Likewise.
* wcsmbs/mbsrtowcs.c: Likewise.
* wcsmbs/wcrtomb.c: Likewise.
* wcsmbs/wcsnrtombs.c: Likewise.
* wcsmbs/wcsrtombs.c: Likewise.
* wcsmbs/wctob.c: Likewise.
Diffstat (limited to 'iconv')
| -rw-r--r-- | iconv/Makefile | 2 | ||||
| -rw-r--r-- | iconv/gconv.c | 8 | ||||
| -rw-r--r-- | iconv/gconv.h | 39 | ||||
| -rw-r--r-- | iconv/gconv_int.h | 48 | ||||
| -rw-r--r-- | iconv/gconv_open.c | 102 | ||||
| -rw-r--r-- | iconv/gconv_simple.c | 177 | ||||
| -rw-r--r-- | iconv/gconv_trans.c | 50 | ||||
| -rw-r--r-- | iconv/loop.c | 129 | ||||
| -rw-r--r-- | iconv/skeleton.c | 60 |
9 files changed, 427 insertions, 188 deletions
diff --git a/iconv/Makefile b/iconv/Makefile index 14076e6f6e..6af661c964 100644 --- a/iconv/Makefile +++ b/iconv/Makefile @@ -26,7 +26,7 @@ include ../Makeconfig headers = iconv.h gconv.h routines = iconv_open iconv iconv_close \ gconv_open gconv gconv_close gconv_db gconv_conf \ - gconv_builtin gconv_simple + gconv_builtin gconv_simple gconv_trans ifeq ($(elf),yes) routines += gconv_dl else diff --git a/iconv/gconv.c b/iconv/gconv.c index 06e212b2cb..19f95622a2 100644 --- a/iconv/gconv.c +++ b/iconv/gconv.c @@ -46,8 +46,8 @@ __gconv (__gconv_t cd, const unsigned char **inbuf, if (inbuf == NULL || *inbuf == NULL) /* We just flush. */ result = DL_CALL_FCT (cd->__steps->__fct, - (cd->__steps, cd->__data, NULL, NULL, - irreversible, 1, 0)); + (cd->__steps, cd->__data, NULL, NULL, + cd->__data[0].__outbuf, irreversible, 1, 0)); else { const unsigned char *last_start; @@ -58,8 +58,8 @@ __gconv (__gconv_t cd, const unsigned char **inbuf, { last_start = *inbuf; result = DL_CALL_FCT (cd->__steps->__fct, - (cd->__steps, cd->__data, inbuf, inbufend, - irreversible, 0, 0)); + (cd->__steps, cd->__data, inbuf, inbufend, + cd->__data[0].__outbuf, irreversible, 0, 0)); } while (result == __GCONV_EMPTY_INPUT && last_start != *inbuf && *inbuf + cd->__steps->__min_needed_from <= inbufend); diff --git a/iconv/gconv.h b/iconv/gconv.h index 1821844952..5717ddbea0 100644 --- a/iconv/gconv.h +++ b/iconv/gconv.h @@ -62,18 +62,52 @@ enum struct __gconv_step; struct __gconv_step_data; struct __gconv_loaded_object; +struct __gconv_trans_data; /* Type of a conversion function. */ typedef int (*__gconv_fct) (struct __gconv_step *, struct __gconv_step_data *, __const unsigned char **, __const unsigned char *, - size_t *, int, int); + unsigned char *, size_t *, int, int); /* Constructor and destructor for local data for conversion step. */ typedef int (*__gconv_init_fct) (struct __gconv_step *); typedef void (*__gconv_end_fct) (struct __gconv_step *); +/* Type of a transliteration/transscription function. */ +typedef int (*__gconv_trans_fct) (struct __gconv_step *step, + struct __gconv_step_data *step_data, + __const unsigned char *, + __const unsigned char **, + __const unsigned char *, unsigned char *, + unsigned char **, unsigned char *, size_t *); + +/* Function to call to provide transliteration module with context. */ +typedef int (*__gconv_trans_context_fct) (struct __gconv_trans_data *data, + __const unsigned char *, + __const unsigned char *, + __const unsigned char *, + unsigned char *, unsigned char *, + unsigned char *); + +/* Function to query module about supported encoded character sets. */ +typedef int (*__gconv_trans_query_fct) (__const char **, size_t *); + +/* Constructor and destructor for local data for transliteration. */ +typedef int (*__gconv_trans_init_fct) (void **, const char *); +typedef void (*__gconv_trans_end_fct) (void *); + +struct __gconv_trans_data +{ + /* Transliteration/Transscription function. */ + __gconv_trans_fct __trans_fct; + __gconv_trans_context_fct __trans_context_fct; + __gconv_trans_end_fct __trans_end_fct; + void *__data; +}; + + /* Description of a conversion step. */ struct __gconv_step { @@ -124,6 +158,9 @@ struct __gconv_step_data __mbstate_t *__statep; __mbstate_t __state; /* This element must not be used directly by any module; always use STATEP! */ + + /* Transliteration information. */ + struct __gconv_trans_data __trans; }; diff --git a/iconv/gconv_int.h b/iconv/gconv_int.h index 01cebe72c4..87287d7b26 100644 --- a/iconv/gconv_int.h +++ b/iconv/gconv_int.h @@ -129,8 +129,8 @@ extern struct gconv_module *__gconv_modules_db; /* Return in *HANDLE decriptor for transformation from FROMSET to TOSET. */ -extern int __gconv_open (const char *__toset, const char *__fromset, - __gconv_t *__handle, int flags) +extern int __gconv_open (const char *toset, const char *fromset, + __gconv_t *handle, int flags) internal_function; /* Free resources associated with transformation descriptor CD. */ @@ -141,55 +141,65 @@ extern int __gconv_close (__gconv_t cd) according to rules described by CD and place up to *OUTBYTESLEFT bytes in buffer starting at *OUTBUF. Return number of non-identical conversions in *IRREVERSIBLE if this pointer is not null. */ -extern int __gconv (__gconv_t __cd, const unsigned char **__inbuf, - const unsigned char *inbufend, unsigned char **__outbuf, +extern int __gconv (__gconv_t cd, const unsigned char **inbuf, + const unsigned char *inbufend, unsigned char **outbuf, unsigned char *outbufend, size_t *irreversible) internal_function; /* Return in *HANDLE a pointer to an array with *NSTEPS elements describing the single steps necessary for transformation from FROMSET to TOSET. */ -extern int __gconv_find_transform (const char *__toset, const char *__fromset, - struct __gconv_step **__handle, - size_t *__nsteps, int flags) +extern int __gconv_find_transform (const char *toset, const char *fromset, + struct __gconv_step **handle, + size_t *nsteps, int flags) internal_function; /* Read all the configuration data and cache it. */ extern void __gconv_read_conf (void); /* Comparison function to search alias. */ -extern int __gconv_alias_compare (const void *__p1, const void *__p2); +extern int __gconv_alias_compare (const void *p1, const void *p2); /* Clear reference to transformation step implementations which might cause the code to be unloaded. */ -extern int __gconv_close_transform (struct __gconv_step *__steps, - size_t __nsteps) +extern int __gconv_close_transform (struct __gconv_step *steps, + size_t nsteps) internal_function; /* Load shared object named by NAME. If already loaded increment reference count. */ -extern struct __gconv_loaded_object *__gconv_find_shlib (const char *__name) +extern struct __gconv_loaded_object *__gconv_find_shlib (const char *name) internal_function; /* Release shared object. If no further reference is available unload the object. */ -extern int __gconv_release_shlib (struct __gconv_loaded_object *__handle) +extern int __gconv_release_shlib (struct __gconv_loaded_object *handle) internal_function; /* Fill STEP with information about builtin module with NAME. */ -extern void __gconv_get_builtin_trans (const char *__name, - struct __gconv_step *__step) +extern void __gconv_get_builtin_trans (const char *name, + struct __gconv_step *step) internal_function; +/* Transliteration using the locale's data. */ +extern int gconv_transliterate (struct __gconv_step *step, + struct __gconv_step_data *step_data, + __const unsigned char *inbufstart, + __const unsigned char **inbufp, + __const unsigned char *inbufend, + unsigned char *outbufstart, + unsigned char **outbufp, + unsigned char *outbufend, + size_t *irreversible); /* Builtin transformations. */ #ifdef _LIBC # define __BUILTIN_TRANS(Name) \ - extern int Name (struct __gconv_step *__step, \ - struct __gconv_step_data *__data, \ - const unsigned char **__inbuf, \ - const unsigned char *__inbufend, size_t *__written, \ - int __do_flush, int __consume_incomplete) + extern int Name (struct __gconv_step *step, \ + struct __gconv_step_data *data, \ + const unsigned char **inbuf, \ + const unsigned char *inbufend, unsigned char *outbufstart, \ + size_t *irreversible, int do_flush, int consume_incomplete) __BUILTIN_TRANS (__gconv_transform_ascii_internal); __BUILTIN_TRANS (__gconv_transform_internal_ascii); diff --git a/iconv/gconv_open.c b/iconv/gconv_open.c index da00b1abbd..984ca9dc5d 100644 --- a/iconv/gconv_open.c +++ b/iconv/gconv_open.c @@ -36,25 +36,65 @@ __gconv_open (const char *toset, const char *fromset, __gconv_t *handle, size_t cnt = 0; int res; int conv_flags = 0; - const char *runp; + const char *errhand; - /* Find out whether "IGNORE" is part of the options in the `toset' - name. If yes, remove the string and remember this in the flag. */ - runp = __strchrnul (__strchrnul (toset, '/'), '/'); - if (strcmp (runp, "IGNORE") == 0) + /* Find out whether any error handling method is specified. */ + errhand = strchr (toset, '/'); + if (errhand != NULL) + errhand = strchr (errhand + 1, '/'); + if (__builtin_expect (errhand != NULL, 1)) { - /* Found it. This means we should ignore conversion errors. */ - char *newtoset = (char *) alloca (runp - toset + 1); + if (errhand[1] == '\0') + errhand = NULL; + else + { + /* Make copy without the error handling description. */ + char *newtoset = (char *) alloca (errhand - toset + 1); - newtoset[runp - toset] = '\0'; - toset = memcpy (newtoset, toset, runp - toset); + newtoset[errhand - toset] = '\0'; + toset = memcpy (newtoset, toset, errhand - toset); - flags = __GCONV_IGNORE_ERRORS; + flags = __GCONV_IGNORE_ERRORS; + + if (strcasecmp (errhand, "IGNORE") == 0) + { + /* Found it. This means we should ignore conversion errors. */ + flags = __GCONV_IGNORE_ERRORS; + errhand = NULL; + } + } } res = __gconv_find_transform (toset, fromset, &steps, &nsteps, flags); if (res == __GCONV_OK) { + const char **csnames = NULL; + size_t ncsnames = 0; + __gconv_trans_fct trans_fct = NULL; + __gconv_trans_context_fct trans_context_fct = NULL; + __gconv_trans_init_fct trans_init_fct = NULL; + __gconv_trans_end_fct trans_end_fct = NULL; + + if (errhand != NULL) + { + /* Find the appropriate transliteration handling. */ + if (strcasecmp (errhand, "TRANSLIT") == 0) + { + /* It's the builtin transliteration handling. We only + suport for it working on the internal encoding. */ + static const char *internal_trans_names[1] = { "INTERNAL" }; + + csnames = internal_trans_names; + ncsnames = 1; + trans_fct = gconv_transliterate; + /* No context, init, or end function. */ + } + else if (strcasecmp (errhand, "WORK AROUND A GCC BUG") == 0) + { + trans_init_fct = (__gconv_trans_init_fct) 1; + } + } + /* Allocate room for handle. */ result = (__gconv_t) malloc (sizeof (struct __gconv_info) + (nsteps @@ -63,6 +103,8 @@ __gconv_open (const char *toset, const char *fromset, __gconv_t *handle, res = __GCONV_NOMEM; else { + size_t n; + /* Remember the list of steps. */ result->__steps = steps; result->__nsteps = nsteps; @@ -105,6 +147,26 @@ __gconv_open (const char *toset, const char *fromset, __gconv_t *handle, } result->__data[cnt].__outbufend = result->__data[cnt].__outbuf + size; + + /* Now see whether we can use the transliteration module + for this step. */ + for (n = 0; n < ncsnames; ++n) + if (strcasecmp (steps[cnt].__from_name, csnames[n]) == 0) + { + /* Match! Now try the initializer. */ + if (trans_init_fct == NULL + || (trans_init_fct (&result->__data[cnt].__trans.__data, + steps[cnt].__to_name) + == __GCONV_OK)) + { + result->__data[cnt].__trans.__trans_fct = trans_fct; + result->__data[cnt].__trans.__trans_context_fct = + trans_context_fct; + result->__data[cnt].__trans.__trans_end_fct = + trans_end_fct; + } + break; + } } /* Now handle the last entry. */ @@ -116,6 +178,26 @@ __gconv_open (const char *toset, const char *fromset, __gconv_t *handle, result->__data[cnt].__internal_use = 0; #endif result->__data[cnt].__statep = &result->__data[cnt].__state; + + /* Now see whether we can use the transliteration module + for this step. */ + for (n = 0; n < ncsnames; ++n) + if (strcasecmp (steps[cnt].__from_name, csnames[n]) == 0) + { + /* Match! Now try the initializer. */ + if (trans_init_fct == NULL + || trans_init_fct (&result->__data[cnt].__trans.__data, + steps[cnt].__to_name) + == __GCONV_OK) + { + result->__data[cnt].__trans.__trans_fct = trans_fct; + result->__data[cnt].__trans.__trans_context_fct = + trans_context_fct; + result->__data[cnt].__trans.__trans_end_fct = + trans_end_fct; + } + break; + } } if (res != __GCONV_OK) diff --git a/iconv/gconv_simple.c b/iconv/gconv_simple.c index 4b7004caa6..188fc04c0e 100644 --- a/iconv/gconv_simple.c +++ b/iconv/gconv_simple.c @@ -19,6 +19,7 @@ Boston, MA 02111-1307, USA. */ #include <byteswap.h> +#include <dlfcn.h> #include <endian.h> #include <errno.h> #include <gconv.h> @@ -62,9 +63,10 @@ static const unsigned char encoding_byte[] = static inline int -internal_ucs4_loop (const unsigned char **inptrp, const unsigned char *inend, +internal_ucs4_loop (struct __gconv_step *step, + struct __gconv_step_data *step_data, + const unsigned char **inptrp, const unsigned char *inend, unsigned char **outptrp, unsigned char *outend, - mbstate_t *state, int flags, void *data, size_t *irreversible) { const unsigned char *inptr = *inptrp; @@ -102,10 +104,11 @@ internal_ucs4_loop (const unsigned char **inptrp, const unsigned char *inend, #ifndef _STRING_ARCH_unaligned static inline int -internal_ucs4_loop_unaligned (const unsigned char **inptrp, +internal_ucs4_loop_unaligned (struct __gconv_step *step, + struct __gconv_step_data *step_data, + const unsigned char **inptrp, const unsigned char *inend, unsigned char **outptrp, unsigned char *outend, - mbstate_t *state, int flags, void *data, size_t *irreversible) { const unsigned char *inptr = *inptrp; @@ -149,12 +152,14 @@ internal_ucs4_loop_unaligned (const unsigned char **inptrp, static inline int -internal_ucs4_loop_single (const unsigned char **inptrp, +internal_ucs4_loop_single (struct __gconv_step *step, + struct __gconv_step_data *step_data, + const unsigned char **inptrp, const unsigned char *inend, unsigned char **outptrp, unsigned char *outend, - mbstate_t *state, int flags, void *data, size_t *irreversible) { + mbstate_t *state = step_data->__statep; size_t cnt = state->__count & 7; while (*inptrp < inend && cnt < 4) @@ -205,11 +210,13 @@ internal_ucs4_loop_single (const unsigned char **inptrp, static inline int -ucs4_internal_loop (const unsigned char **inptrp, const unsigned char *inend, +ucs4_internal_loop (struct __gconv_step *step, + struct __gconv_step_data *step_data, + const unsigned char **inptrp, const unsigned char *inend, unsigned char **outptrp, unsigned char *outend, - mbstate_t *state, int flags, void *data, size_t *irreversible) { + int flags = step_data->__flags; const unsigned char *inptr = *inptrp; unsigned char *outptr = *outptrp; size_t n_convert = MIN (inend - inptr, outend - outptr) / 4; @@ -228,6 +235,10 @@ ucs4_internal_loop (const unsigned char **inptrp, const unsigned char *inend, if (__builtin_expect (inval, 0) > 0x7fffffff) { + /* The value is too large. We don't try transliteration here since + this is not an error because of the lack of possibilities to + represent the result. This is a genuine bug in the input since + UCS4 does not allow such values. */ if (flags & __GCONV_IGNORE_ERRORS) { /* Just ignore this character. */ @@ -259,23 +270,28 @@ ucs4_internal_loop (const unsigned char **inptrp, const unsigned char *inend, #ifndef _STRING_ARCH_unaligned static inline int -ucs4_internal_loop_unaligned (const unsigned char **inptrp, +ucs4_internal_loop_unaligned (struct __gconv_step *step, + struct __gconv_step_data *step_data, + const unsigned char **inptrp, const unsigned char *inend, unsigned char **outptrp, unsigned char *outend, - mbstate_t *state, int flags, void *data, size_t *irreversible) { + int flags = step_data->__flags; const unsigned char *inptr = *inptrp; unsigned char *outptr = *outptrp; size_t n_convert = MIN (inend - inptr, outend - outptr) / 4; int result; size_t cnt; - for (cnt = 0; cnt < n_convert; ++cnt, inptr += 4, outptr += 4) + for (cnt = 0; cnt < n_convert; ++cnt, inptr += 4) { if (__builtin_expect (inptr[0], 0) > 0x80) { - /* The value is too large. */ + /* The value is too large. We don't try transliteration here since + this is not an error because of the lack of possibilities to + represent the result. This is a genuine bug in the input since + UCS4 does not allow such values. */ if (flags & __GCONV_IGNORE_ERRORS) { /* Just ignore this character. */ @@ -299,6 +315,7 @@ ucs4_internal_loop_unaligned (const unsigned char **inptrp, outptr[2] = inptr[2]; outptr[3] = inptr[3]; # endif + outptr += 4; } *inptrp = inptr; @@ -318,12 +335,15 @@ ucs4_internal_loop_unaligned (const unsigned char **inptrp, static inline int -ucs4_internal_loop_single (const unsigned char **inptrp, +ucs4_internal_loop_single (struct __gconv_step *step, + struct __gconv_step_data *step_data, + const unsigned char **inptrp, const unsigned char *inend, unsigned char **outptrp, unsigned char *outend, - mbstate_t *state, int flags, void *data, size_t *irreversible) { + mbstate_t *state = step_data->__statep; + int flags = step_data->__flags; size_t cnt = state->__count & 7; while (*inptrp < inend && cnt < 4) @@ -341,7 +361,10 @@ ucs4_internal_loop_single (const unsigned char **inptrp, if (__builtin_expect (((unsigned char *) state->__value.__wchb)[0], 0) > 0x80) { - /* The value is too large. */ + /* The value is too large. We don't try transliteration here since + this is not an error because of the lack of possibilities to + represent the result. This is a genuine bug in the input since + UCS4 does not allow such values. */ if (!(flags & __GCONV_IGNORE_ERRORS)) { *inptrp -= cnt - (state->__count & 7); @@ -386,9 +409,10 @@ ucs4_internal_loop_single (const unsigned char **inptrp, static inline int -internal_ucs4le_loop (const unsigned char **inptrp, const unsigned char *inend, +internal_ucs4le_loop (struct __gconv_step *step, + struct __gconv_step_data *step_data, + const unsigned char **inptrp, const unsigned char *inend, unsigned char **outptrp, unsigned char *outend, - mbstate_t *state, int flags, void *data, size_t *irreversible) { const unsigned char *inptr = *inptrp; @@ -426,10 +450,11 @@ internal_ucs4le_loop (const unsigned char **inptrp, const unsigned char *inend, #ifndef _STRING_ARCH_unaligned static inline int -internal_ucs4le_loop_unaligned (const unsigned char **inptrp, +internal_ucs4le_loop_unaligned (struct __gconv_step *step, + struct __gconv_step_data *step_data, + const unsigned char **inptrp, const unsigned char *inend, unsigned char **outptrp, unsigned char *outend, - mbstate_t *state, int flags, void *data, size_t *irreversible) { const unsigned char *inptr = *inptrp; @@ -473,12 +498,14 @@ internal_ucs4le_loop_unaligned (const unsigned char **inptrp, static inline int -internal_ucs4le_loop_single (const unsigned char **inptrp, +internal_ucs4le_loop_single (struct __gconv_step *step, + struct __gconv_step_data *step_data, + const unsigned char **inptrp, const unsigned char *inend, unsigned char **outptrp, unsigned char *outend, - mbstate_t *state, int flags, void *data, size_t *irreversible) { + mbstate_t *state = step_data->__statep; size_t cnt = state->__count & 7; while (*inptrp < inend && cnt < 4) @@ -526,11 +553,13 @@ internal_ucs4le_loop_single (const unsigned char **inptrp, s |
