diff options
Diffstat (limited to 'iconvdata')
| -rw-r--r-- | iconvdata/Makefile | 51 | ||||
| -rw-r--r-- | iconvdata/euckr.c | 15 | ||||
| -rw-r--r-- | iconvdata/gb2312.h | 15 | ||||
| -rw-r--r-- | iconvdata/iso-2022-jp.c | 693 | ||||
| -rw-r--r-- | iconvdata/iso646.c | 20 | ||||
| -rw-r--r-- | iconvdata/iso8859-10.c | 4 | ||||
| -rw-r--r-- | iconvdata/iso8859-5.c | 4 | ||||
| -rw-r--r-- | iconvdata/iso8859-7.c | 4 | ||||
| -rw-r--r-- | iconvdata/iso8859-8.c | 4 | ||||
| -rw-r--r-- | iconvdata/ksc5601.c | 3442 | ||||
| -rw-r--r-- | iconvdata/ksc5601.h | 96 | ||||
| -rw-r--r-- | iconvdata/uhc.c | 21 |
12 files changed, 2807 insertions, 1562 deletions
diff --git a/iconvdata/Makefile b/iconvdata/Makefile index 00ddd82c9d..8b83ade4fe 100644 --- a/iconvdata/Makefile +++ b/iconvdata/Makefile @@ -38,7 +38,7 @@ modules += KOI8-R LATIN-GREEK LATIN-GREEK-1 IBM256 IBM273 IBM277 IBM278 \ IBM850 IBM851 IBM852 IBM855 IBM857 IBM860 IBM861 IBM862 \ IBM863 IBM864 IBM865 IBM868 IBM869 IBM875 IBM880 IBM918 \ IBM1004 IBM1026 CP1250 CP1251 CP1252 CP1253 CP1254 CP1255 \ - CP1256 CP1257 + CP1256 CP1257 ISO-2022-JP endif modules.so := $(addsuffix .so, $(modules)) @@ -142,6 +142,7 @@ BIG5-routines := big5 EUC-JP-routines := eucjp EUC-CN-routines := euccn EUC-TW-routines := euctw +ISO-2022-JP-routines := iso-2022-jp libJIS-routines := jis0201 jis0208 jis0212 libKSC-routines := ksc5601 libGB-routines := gb2312 @@ -160,6 +161,10 @@ $(objpfx)EUC-CN.so: $(objpfx)libGB.so LDFLAGS-EUC-TW.so = -Wl,-rpath,$(gconvdir) $(objpfx)EUC-TW.so: $(objpfx)libCNS.so +LDFLAGS-ISO-2022-JP.so = -Wl,-rpath,$(gconvdir) +$(objpfx)ISO-2022-JP.so: $(objpfx)libJIS.so $(objpfx)libGB.so \ + $(objpfx)libCNS.so + LDFLAGS-libJIS.so = -Wl,-soname,$(@F) LDFLAGS-libKSC.so = -Wl,-soname,$(@F) LDFLAGS-libGB.so = -Wl,-soname,$(@F) @@ -188,7 +193,7 @@ distribute := 8bit-generic.c 8bit-gap.c gap.pl gaptab.pl gconv-modules \ ibm903.c ibm904.c ibm905.c ibm918.c ibm1004.c ibm1026.c \ ibm1047.c cp1250.c cp1251.c cp1252.c cp1253.c cp1254.c \ cp1255.c cp1256.c cp1257.c cp874.c cp874.h cp737.c cp737.h \ - cp775.c cp775.h + cp775.c cp775.h iso-2022-jp.c # We build the transformation modules only when we build shared libs. ifeq (yes,$(build-shared)) @@ -226,9 +231,8 @@ $(make-target-directory) echo "};" ) > $@.new && rm -f $@ && mv $@.new $@ endef -sed-generated-headers := iso8859-2.h iso8859-3.h iso8859-4.h iso8859-5.h \ - iso8859-6.h iso8859-7.h iso8859-8.h iso8859-9.h \ - iso8859-10.h koi-8.h hp-roman8.h ebcdic-at-de.h \ +sed-generated-headers := iso8859-2.h iso8859-3.h iso8859-4.h iso8859-6.h \ + iso8859-9.h koi-8.h hp-roman8.h ebcdic-at-de.h \ ebcdic-at-de-a.h ebcdic-ca-fr.h ebcdic-dk-no.h \ ebcdic-dk-no-a.h ebcdic-es.h ebcdic-es-a.h \ ebcdic-es-s.h ebcdic-fi-se.h ebcdic-fi-se-a.h \ @@ -236,7 +240,7 @@ sed-generated-headers := iso8859-2.h iso8859-3.h iso8859-4.h iso8859-5.h \ ebcdic-pt.h ebcdic-uk.h ebcdic-us.h ibm037.h \ ibm038.h ibm274.h ibm275.h ibm423.h ibm424.h \ ibm500.h ibm870.h ibm871.h ibm891.h ibm903.h \ - ibm904.h ibm905.h ibm1047.h + ibm904.h ibm905.h ibm1047.h iso8859-7jp.h define generate-8bit-gap-table $(make-target-directory) @@ -260,7 +264,9 @@ perl-generated-headers := koi8-r.h latin-greek.h latin-greek-1.h \ ibm863.h ibm864.h ibm865.h ibm868.h ibm869.h \ ibm875.h ibm880.h ibm918.h ibm1004.h ibm1026.h \ cp1250.h cp1251.h cp1252.h cp1253.h cp1254.h \ - cp1255.h cp1256.h cp1257.h + cp1255.h cp1256.h cp1257.h iso8859-5.h \ + iso8859-7.h iso8859-8.h iso8859-10.h \ + iso8859-7jp.h # The headers must be generated before the compilation. before-compile = $(addprefix $(objpfx),$(sed-generated-headers)) \ @@ -274,18 +280,10 @@ $(objpfx)iso8859-3.h: ../localedata/charmaps/ISO-8859-3 Makefile $(generate-8bit-table) $(objpfx)iso8859-4.h: ../localedata/charmaps/ISO-8859-4 Makefile $(generate-8bit-table) -$(objpfx)iso8859-5.h: ../localedata/charmaps/ISO-8859-5 Makefile - $(generate-8bit-table) $(objpfx)iso8859-6.h: ../localedata/charmaps/ISO-8859-6 Makefile $(generate-8bit-table) -$(objpfx)iso8859-7.h: ../localedata/charmaps/ISO-8859-7 Makefile - $(generate-8bit-table) -$(objpfx)iso8859-8.h: ../localedata/charmaps/ISO-8859-8 Makefile - $(generate-8bit-table) $(objpfx)iso8859-9.h: ../localedata/charmaps/ISO-8859-9 Makefile $(generate-8bit-table) -$(objpfx)iso8859-10.h: ../localedata/charmaps/ISO-8859-10 Makefile - $(generate-8bit-table) $(objpfx)koi-8.h: ../localedata/charmaps/KOI-8 Makefile $(generate-8bit-table) @@ -352,7 +350,30 @@ $(objpfx)ibm905.h: ../localedata/charmaps/IBM905 Makefile $(objpfx)ibm1047.h: ../localedata/charmaps/IBM1047 Makefile $(generate-8bit-table) +$(objpfx)iso8859-7jp.h: ../localedata/charmaps/ISO-8859-7 Makefile + $(make-target-directory) + ( echo "static const uint32_t iso88597_to_ucs4[96] = {"; \ + sed -e '/^[^[:space:]]*[[:space:]]*.x00/d' -e 's/^[^[:space:]]*[[:space:]]*.x\([A-F].\)[[:space:]]*<U\(....\)>.*/ [0x\1-0xA0] = 0x\2,/p' -e d $^ | sort -u; \ + echo "};"; \ + echo "static struct gap from_idx[] = {"; \ + sed -e '/^[^[:space:]]*[[:space:]]*.x00/d' -e 's/^[^[:space:]]*[[:space:]]*.x\([A-F].\)[[:space:]]*<U\(....\)>.*/0x\2 0x\1/p' -e d $^ | sort -u | $(PERL) gap.pl; \ + echo " { start: 0xffff, end: 0xffff, idx: 0 }"; \ + echo "};"; \ + echo "static const char iso88597_from_ucs4[] = {"; \ + sed -e '/^[^[:space:]]*[[:space:]]*.x00/d' -e 's/^[^[:space:]]*[[:space:]]*.x\([A-F].\)[[:space:]]*<U\(....\)>.*/0x\2 0x\1/p' -e d $^ | sort -u | $(PERL) gaptab.pl; \ + echo "};" ) > $@.new && rm -f $@ && mv $@.new $@ + + ifneq ($(PERL),no) +$(objpfx)iso8859-5.h: ../localedata/charmaps/ISO-8859-5 Makefile + $(generate-8bit-gap-table) +$(objpfx)iso8859-7.h: ../localedata/charmaps/ISO-8859-7 Makefile + $(generate-8bit-gap-table) +$(objpfx)iso8859-8.h: ../localedata/charmaps/ISO-8859-8 Makefile + $(generate-8bit-gap-table) +$(objpfx)iso8859-10.h: ../localedata/charmaps/ISO-8859-10 Makefile + $(generate-8bit-gap-table) + $(objpfx)koi8-r.h: ../localedata/charmaps/KOI8-R Makefile $(generate-8bit-gap-table) $(objpfx)latin-greek.h: ../localedata/charmaps/LATIN-GREEK Makefile diff --git a/iconvdata/euckr.c b/iconvdata/euckr.c index 51848843d9..1a2ff89568 100644 --- a/iconvdata/euckr.c +++ b/iconvdata/euckr.c @@ -88,27 +88,20 @@ euckr_from_ucs4 (uint32_t ch, unsigned char *cp) { \ /* Two-byte character. First test whether the next character \ is also available. */ \ - int ch2; \ - \ - if (NEED_LENGTH_TEST && inptr + 1 >= inend) \ + ch = ksc5601_to_ucs4 (&inptr, \ + NEED_LENGTH_TEST ? inptr - inbufend : 2, x080); \ + if (NEED_LENGTH_TEST && ch == 0) \ { \ /* The second character is not available. */ \ result = GCONV_INCOMPLETE_INPUT; \ break; \ } \ - \ - ch2 = inptr[1]; \ - \ - if (ch2 < 0xa1 || ch2 >= 0xfe \ - || ((ch = ksc5601_to_ucs4 ((uint16_t) (ch * 256 + ch2) & 0x7f7f)) \ - == UNKNOWN_10646_CHAR)) \ + if (ch == UNKNOWN_10646_CHAR)) \ { \ /* This is an illegal character. */ \ result = GCONV_ILLEGAL_INPUT; \ break; \ } \ - \ - inptr += 2; \ } \ \ *((uint32_t *) outptr)++ = ch; \ diff --git a/iconvdata/gb2312.h b/iconvdata/gb2312.h index 5b405a1138..3de274e06b 100644 --- a/iconvdata/gb2312.h +++ b/iconvdata/gb2312.h @@ -29,7 +29,7 @@ extern const uint16_t __gb2312_to_ucs[]; static inline uint32_t -gb2312_to_ucs4 (const char **s, size_t avail, unsigned char offset) +gb2312_to_ucs4 (const unsigned char **s, size_t avail, unsigned char offset) { unsigned char ch = *(*s); unsigned char ch2; @@ -66,7 +66,7 @@ extern const char __gb2312_from_ucs4_tab8[][2]; extern const char __gb2312_from_ucs4_tab9[][2]; static inline size_t -ucs4_to_gb2312 (uint32_t wch, char *s, size_t avail) +ucs4_to_gb2312 (uint32_t wch, unsigned char **s, size_t avail) { unsigned int ch = (unsigned int) wch; char buf[2]; @@ -212,11 +212,16 @@ ucs4_to_gb2312 (uint32_t wch, char *s, size_t avail) return UNKNOWN_10646_CHAR; } - if (cp[1] != '\0' && avail < 2) + if (cp[0] == '\0') + return UNKNOWN_10646_CHAR; + + assert (cp[1] != '\0'); + + if (avail < 2) return 0; - s[0] = cp[0]; - s[1] = cp[1]; + *(*s)++ = cp[0]; + *(*s)++ = cp[1]; return 2; } diff --git a/iconvdata/iso-2022-jp.c b/iconvdata/iso-2022-jp.c new file mode 100644 index 0000000000..cb4257baf8 --- /dev/null +++ b/iconvdata/iso-2022-jp.c @@ -0,0 +1,693 @@ +/* Conversion module for ISO-2022-JP. + Copyright (C) 1998 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with the GNU C Library; see the file COPYING.LIB. If not, + write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. */ + +#include <gconv.h> +#include <stdint.h> +#include <string.h> +#include "jis0201.h" +#include "jis0212.h" +#include "jis0201.h" +#include "gb2312.h" +#include "ksc5601.h" +#include "iso8859-7jp.h" + +/* This makes obvious what everybody knows: 0x1b is the Esc character. */ +#define ESC 0x1b + +/* We provide our own initialization and destructor function. */ +#define DEFINE_INIT 0 +#define DEFINE_FINI 0 + +/* Definitions used in the body of the `gconv' function. */ +#define FROM_LOOP from_iso2022jp +#define TO_LOOP to_iso2022jp +#define MIN_NEEDED_FROM 1 +#define MAX_NEEDED_FROM 4 +#define MIN_NEEDED_TO 4 +#define FROM_DIRECTION dir == from_iso2022jp +#define PREPARE_LOOP \ + enum direction dir = ((struct iso2022jp_data *) step->data)->dir; \ + enum variant var = ((struct iso2022jp_data *) step->data)->var; \ + int set = data->statep->count; +#define END_LOOP \ + data->statep->count = set; +#define EXTRA_LOOP_ARGS , var, set + + +/* Direction of the transformation. */ +enum direction +{ + illegal_dir, + to_iso2022jp, + from_iso2022jp +}; + +/* We handle ISO-2022-jp and ISO-2022-JP-2 here. */ +enum variant +{ + illegal_var, + iso2022jp, + iso2022jp2 +}; + + +struct iso2022jp_data +{ + enum direction dir; + enum variant var; + mbstate_t save_state; +}; + + +/* The COUNT element of the state keeps track of the currently selected + character set. The possible values are: */ +enum +{ + ASCII_set = 0, + JISX0208_1978_set, + JISX0208_1983_set, + JISX0201_set, + GB2312_set, + KSC5601_set, + JISX0212_set, + ISO88591_set, + ISO88597_set +}; + + +int +gconv_init (struct gconv_step *step) +{ + /* Determine which direction. */ + struct iso2022jp_data *new_data; + enum direction dir = illegal_dir; + enum variant var; + int result; + + if (__strcasecmp (step->from_name, "ISO-2022-JP//") == 0) + { + dir = from_iso2022jp; + var = iso2022jp; + } + else if (__strcasecmp (step->to_name, "ISO-2022-JP//") == 0) + { + dir = to_iso2022jp; + var = iso2022jp; + } + else if (__strcasecmp (step->from_name, "ISO-2022-JP-2//") == 0) + { + dir = from_iso2022jp; + var = iso2022jp2; + } + else if (__strcasecmp (step->to_name, "ISO-2022-JP-2//") == 0) + { + dir = to_iso2022jp; + var = iso2022jp2; + } + + result = GCONV_NOCONV; + if (dir != illegal_dir + && ((new_data + = (struct iso2022jp_data *) malloc (sizeof (struct iso2022jp_data))) + != NULL)) + { + new_data->dir = dir; + new_data->var = var; + step->data = new_data; + + if (dir == from_iso2022jp) + { + step->min_needed_from = MIN_NEEDED_FROM; + step->max_needed_from = MAX_NEEDED_FROM; + step->min_needed_to = MIN_NEEDED_TO; + step->max_needed_to = MIN_NEEDED_TO; + } + else + { + step->min_needed_from = MIN_NEEDED_TO; + step->max_needed_from = MAX_NEEDED_TO; + step->min_needed_to = MIN_NEEDED_FROM; + step->max_needed_to = MIN_NEEDED_FROM + 2; + } + + /* Yes, this is a stateful encoding. */ + step->stateful = 1; + + result = GCONV_OK; + } + + return result; +} + + +void +gconv_end (struct gconv_step *data) +{ + free (data->data); +} + + +/* Since this is a stateful encoding we have to provide code which resets + the output state to the initial state. This has to be done during the + flushing. */ +#define EMIT_SHIFT_TO_INIT \ + if (data->statep->count != 0) \ + { \ + enum direction dir = ((struct iso2022jp_data *) step->data)->dir; \ + \ + if (dir == from_iso2022jp) \ + /* It's easy, we don't have to emit anything, we just reset the \ + state for the input. */ \ + data->statep->count = 0; \ + else \ + { \ + char *outbuf = data->outbuf; \ + \ + /* We are not in the initial state. To switch back we have \ + to emit the sequence `Esc ( B'. */ \ + if (outbuf + 3 > data->outbufend) \ + /* We don't have enough room in the output buffer. */ \ + status = GCONV_FULL_OUTPUT; \ + else \ + { \ + /* Write out the shift sequence. */ \ + *outbuf++ = ESC; \ + *outbuf++ = '('; \ + *outbuf++ = 'B'; \ + data->outbuf = outbuf; \ + data->statep->count = 0; \ + } \ + } \ + } + + +/* Since we might have to reset input pointer we must be able to save + and retore the state. */ +#define SAVE_RESET_STATE(Save) \ + if (Save) \ + ((struct iso2022jp_data *) step->data)->save_state.count \ + = data->statep->count; \ + else \ + data->statep->count \ + = ((struct iso2022jp_data *) step->data)->save_state.count + + +/* First define the conversion function from ISO-2022-JP to UCS4. */ +#define MIN_NEEDED_INPUT MIN_NEEDED_FROM +#define MAX_NEEDED_INPUT MAX_NEEDED_FROM +#define MIN_NEEDED_OUTPUT MIN_NEEDED_TO +#define LOOPFCT FROM_LOOP +#define BODY \ + { \ + uint32_t ch = *inptr; \ + \ + /* This is a 7bit character set, disallow all 8bit characters. */ \ + if (ch > 0x7f) \ + { \ + result = GCONV_ILLEGAL_INPUT; \ + break; \ + } \ + \ + /* Recognize escape sequences. */ \ + if (ch == ESC) \ + { \ + /* We now must be prepared to read two to three more \ + chracters. If we have a match in the first character but \ + then the input buffer ends we terminate with an error since \ + we must not risk missing an escape sequence just because it \ + is not entirely in the current input buffer. */ \ + if (inptr + 2 >= inbufend \ + || (var == iso2022jp2 && inptr[1] == '$' && inptr[2] == '(' \ + && inptr +3 >= inbufend)) \ + { \ + /* Not enough input available. */ \ + result = GCONV_EMPTY_INPUT; \ + break; \ + } \ + \ + if (inptr[1] == '(') \ + { \ + if (inptr[2] = 'B') \ + { \ + /* ASCII selected. */ \ + set = ASCII_set; \ + inptr += 3; \ + continue; \ + } \ + else if (inptr[2] == 'J') \ + { \ + /* JIS X 0201 selected. */ \ + set = JISX0201_set; \ + inptr += 3; \ + continue; \ + } \ + } \ + else if (inptr[1] == '$') \ + { \ + if (inptr[2] == '@') \ + { \ + /* JIS X 0208-1978 selected. */ \ + set = JISX0208_1978_set; \ + inptr += 3; \ + continue; \ + } \ + else if (inptr[2] == 'B') \ + { \ + /* JIS X 0208-1983 selected. */ \ + set = JISX0208_1983_set; \ + inptr += 3; \ + continue; \ + } \ + else if (var == iso2022jp2) \ + { \ + if (inptr[2] == 'A') \ + { \ + /* GB 2312-1980 selected. */ \ + set = GB2312_set; \ + inptr += 3; \ + continue; \ + } \ + else if (inptr[2] == '(') \ + { \ + if (inptr[3] == 'C') \ + { \ + /* KSC 5601-1987 selected. */ \ + set = KSC5601_set; \ + inptr += 4; \ + continue; \ + } \ + else (inptr[3] == 'D') \ + { \ + /* JIS X 0212-1990 selected. */ \ + set = JISX0212_set; \ + inptr += 4; \ + continue; \ + } \ + } \ + } \ + } \ + else if (var == iso2022jp2 && inptr[1] == '.') \ + { \ + if (inptr[2] == 'A') \ + { \ + /* ISO 8859-1-GR selected. */ \ + set = ISO88591_set; \ + inptr += 3; \ + continue; \ + } \ + else if (inptr[2] == 'F') \ + { \ + /* ISO 8859-7-GR selected. */ \ + set = ISO88597_set; \ + inptr += 3; \ + continue; \ + } \ + } \ + } \ + \ + if (set == ASCII_set \ + || (var < ISO88591_set && (ch < 0x21 || ch == 0x7f)) \ + || (var >= ISO88591_set && ch < 0x20)) \ + /* Almost done, just advance the input pointer. */ \ + ++inptr; \ + else if (set == JISX0201_set) \ + { \ + /* Use the JIS X 0201 table. */ \ + ch = jisx0201_to_ucs4 (ch + 0x80); \ + if (ch == UNKNOWN_10646_CHAR) \ + { \ + result = GCONV_ILLEGAL_INPUT; \ + break; \ + } \ + ++inptr; \ + } \ + else if (set == ISO88591_set) \ + { \ + /* This is quite easy. All characters are defined and the \ + ISO 10646 value is computed by adding 0x80. */ \ + ch += 0x80; \ + ++inptr; \ + } \ + else if (set == ISO88597_set) \ + { \ + /* We use the table from the ISO 8859-7 module. */ \ + ch = iso88597_to_ucs4[ch - 0x20]; \ + if (ch == 0) \ + { \ + result = GCONV_ILLEGAL_INPUT; \ + break; \ + } \ + ++inptr; \ + } \ + else \ + { \ + if (set == JISX0208_1978_set || set == JISX0208_1983_set) \ + /* XXX I don't have the tables for these two old variants of \ + JIS X 0208. Therefore I'm using the tables for JIS X \ + 0208-1990. If somebody has problems with this please \ + provide the appropriate tables. */ \ + ch = jisx0208_to_ucs4 (&inptr, \ + NEED_LENGTH_TEST ? inbufend - inptr : 2, 0); \ + else if (set == JISX0212_set) \ + /* Use the JIS X 0212 table. */ \ + ch = jisx0212_to_ucs4 (&inptr, \ + NEED_LENGTH_TEST ? inbufend - inptr : 2, 0); \ + else if (set == GB2312_set) \ + /* Use the GB 2312 table. */ \ + ch = gb2312_to_ucs4 (&inptr, \ + NEED_LENGTH_TEST ? inbufend - inptr : 2, 0); \ + else \ + { \ + assert (set == KSC5601_set); \ + \ + /* Use the KSC 5601 table. */ \ + ch = ksc5601_to_ucs4 (&inptr, \ + NEED_LENGTH_TEST ? inbufend - inptr : 2, \ + 0); \ + } \ + \ + if (NEED_LENGTH_TEST && ch == 0) \ + { \ + result = GCONV_EMPTY_INPUT; \ + break; \ + } \ + else if (ch == UNKNOWN_10646_CHAR) \ + { \ + result = GCONV_ILLEGAL_INPUT; \ + break; \ + } \ + } \ + \ + *((uint32_t *) outptr)++ = ch; \ + } +#define EXTRA_LOOP_DECLS , enum variant var, int set +#include <iconv/loop.c> + + +/* Next, define the other direction. */ +#define MIN_NEEDED_INPUT MIN_NEEDED_TO +#define MIN_NEEDED_OUTPUT MIN_NEEDED_FROM +#def |