diff options
Diffstat (limited to 'iconvdata')
47 files changed, 14845 insertions, 4503 deletions
diff --git a/iconvdata/.cvsignore b/iconvdata/.cvsignore new file mode 100644 index 0000000000..8e5dcb0c65 --- /dev/null +++ b/iconvdata/.cvsignore @@ -0,0 +1,14 @@ +ebcdic-at-de-a.h +ebcdic-at-de.h +ebcdic-ca-fr.h +hp-roman8.h +iso8859-10.h +iso8859-2.h +iso8859-3.h +iso8859-4.h +iso8859-5.h +iso8859-6.h +iso8859-7.h +iso8859-8.h +iso8859-9.h +koi-8.h diff --git a/iconvdata/8bit-gap.c b/iconvdata/8bit-gap.c new file mode 100644 index 0000000000..0bcc91ba28 --- /dev/null +++ b/iconvdata/8bit-gap.c @@ -0,0 +1,250 @@ +/* Generic conversion to and from 8bit charsets, + converting from UCS using gaps. + Copyright (C) 1997 Free Software Foundation, Inc. + This file is part of the GNU C Library. + Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997. + + The GNU C Library is free software; you can redistribute it and/or + modify it under the terms of the GNU Library General Public License as + published by the Free Software Foundation; either version 2 of the + License, or (at your option) any later version. + + The GNU C Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Library General Public License for more details. + + You should have received a copy of the GNU Library General Public + License along with the GNU C Library; see the file COPYING.LIB. If not, + write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. */ + +#include <gconv.h> +#include <inttypes.h> +#include <stdlib.h> +#include <string.h> + + +struct gap +{ + uint16_t start; + uint16_t end; + int16_t idx; +}; + +/* Now we can include the tables. */ +#include TABLES + +/* Direction of the transformation. */ +enum direction +{ + illegal, + to_8bit, + from_8bit +}; + +struct s_8bit_data +{ + enum direction dir; +}; + + +int +gconv_init (struct gconv_step *step, struct gconv_step_data *data) +{ + /* Determine which direction. */ + struct s_8bit_data *new_data; + enum direction dir; + int result; + + if (__strcasestr (step->from_name, NAME) != NULL) + dir = from_8bit; + else if (__strcasestr (step->to_name, NAME) != NULL) + dir = to_8bit; + else + dir = illegal; + + result = GCONV_NOCONV; + if (dir != illegal + && ((new_data + = (struct s_8bit_data *) malloc (sizeof (struct s_8bit_data))) + != NULL)) + { + new_data->dir = dir; + data->data = new_data; + result = GCONV_OK; + } + + return result; +} + + +void +gconv_end (struct gconv_step_data *data) +{ + free (data->data); +} + + +int +gconv (struct gconv_step *step, struct gconv_step_data *data, + const char *inbuf, size_t *inbufsize, size_t *written, int do_flush) +{ + struct gconv_step *next_step = step + 1; + struct gconv_step_data *next_data = data + 1; + gconv_fct fct = next_step->fct; + size_t do_write; + int result; + + /* If the function is called with no input this means we have to reset + to the initial state. The possibly partly converted input is + dropped. */ + if (do_flush) + { + do_write = 0; + + /* Call the steps down the chain if there are any. */ + if (data->is_last) + result = GCONV_OK; + else + { + struct gconv_step *next_step = step + 1; + struct gconv_step_data *next_data = data + 1; + + result = (*fct) (next_step, next_data, NULL, 0, written, 1); + + /* Clear output buffer. */ + data->outbufavail = 0; + } + } + else + { + enum direction dir = ((struct s_8bit_data *) data->data)->dir; + + do_write = 0; + + do + { + result = GCONV_OK; + + if (dir == from_8bit) + { + size_t inchars = *inbufsize; + size_t outwchars = data->outbufavail; + char *outbuf = data->outbuf; + size_t cnt = 0; + + while (cnt < inchars + && (outwchars + sizeof (wchar_t) <= data->outbufsize)) + { + wchar_t ch = to_ucs4[(unsigned int) inbuf[cnt]]; + + if (ch == L'\0' && inbuf[cnt] != '\0') + { + /* This is an illegal character. */ + result = GCONV_ILLEGAL_INPUT; + break; + } + + *((wchar_t *) (outbuf + outwchars)) = ch; + ++do_write; + outwchars += sizeof (wchar_t); + ++cnt; + } + *inbufsize -= cnt; + data->outbufavail = outwchars; + } + else + { + size_t inwchars = *inbufsize; + size_t outchars = data->outbufavail; + char *outbuf = data->outbuf; + size_t cnt = 0; + + while (inwchars >= cnt + sizeof (wchar_t) + && outchars < data->outbufsize) + { + const struct gap *rp = from_idx; + unsigned int ch = *((wchar_t *) (inbuf + cnt)); + char res; + + while (ch > rp->end) + ++rp; + if (ch < rp->start) + /* No valid character. */ + break; + + res = from_ucs4[ch + rp->idx]; + if (res == '\0' && ch != 0) + /* No valid character. */ + break; + + outbuf[outchars] = res; + ++do_write; + ++outchars; + cnt += sizeof (wchar_t); + } + *inbufsize -= cnt; + data->outbufavail = outchars; + + if (outchars < data->outbufsize) + { + /* If there is still room in the output buffer something + is wrong with the input. */ + if (inwchars >= cnt + sizeof (wchar_t)) + { + /* An error occurred. */ + result = GCONV_ILLEGAL_INPUT; + break; + } + if (inwchars != cnt) + { + /* There are some unprocessed bytes at the end of the + input buffer. */ + result = GCONV_INCOMPLETE_INPUT; + break; + } + } + } + + if (result != GCONV_OK) + break; + + if (data->is_last) + { + /* This is the last step. */ + result = (*inbufsize > (dir == from_8bit + ? 0 : sizeof (wchar_t) - 1) + ? GCONV_FULL_OUTPUT : GCONV_EMPTY_INPUT); + break; + } + + /* Status so far. */ + result = GCONV_EMPTY_INPUT; + + if (data->outbufavail > 0) + { + /* Call the functions below in the chain. */ + size_t newavail = data->outbufavail; + + result = (*fct) (next_step, next_data, data->outbuf, &newavail, + written, 0); + + /* Correct the output buffer. */ + if (newavail != data->outbufavail && newavail > 0) + { + memmove (data->outbuf, + &data->outbuf[data->outbufavail - newavail], + newavail); + data->outbufavail = newavail; + } + } + } + while (*inbufsize > 0 && result == GCONV_EMPTY_INPUT); + } + + if (written != NULL && data->is_last) + *written = do_write; + + return result; +} diff --git a/iconvdata/iso8859-generic.c b/iconvdata/8bit-generic.c index e33e1d0a87..fe0a09bd3e 100644 --- a/iconvdata/iso8859-generic.c +++ b/iconvdata/8bit-generic.c @@ -1,4 +1,4 @@ -/* Generic conversion to and from ISO 8859-XXX. +/* Generic conversion to and from 8bit charsets. Copyright (C) 1997 Free Software Foundation, Inc. This file is part of the GNU C Library. Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997. @@ -26,11 +26,11 @@ enum direction { illegal, - to_iso8859, - from_iso8859 + to_8bit, + from_8bit }; -struct iso8859_data +struct s_8bit_data { enum direction dir; }; @@ -40,21 +40,21 @@ int gconv_init (struct gconv_step *step, struct gconv_step_data *data) { /* Determine which direction. */ - struct iso8859_data *new_data; + struct s_8bit_data *new_data; enum direction dir; int result; if (__strcasestr (step->from_name, NAME) != NULL) - dir = from_iso8859; + dir = from_8bit; else if (__strcasestr (step->to_name, NAME) != NULL) - dir = to_iso8859; + dir = to_8bit; else dir = illegal; result = GCONV_NOCONV; if (dir != illegal && ((new_data - = (struct iso8859_data *) malloc (sizeof (struct iso8859_data))) + = (struct s_8bit_data *) malloc (sizeof (struct s_8bit_data))) != NULL)) { new_data->dir = dir; @@ -106,7 +106,7 @@ gconv (struct gconv_step *step, struct gconv_step_data *data, } else { - enum direction dir = ((struct iso8859_data *) data->data)->dir; + enum direction dir = ((struct s_8bit_data *) data->data)->dir; do_write = 0; @@ -114,7 +114,7 @@ gconv (struct gconv_step *step, struct gconv_step_data *data, { result = GCONV_OK; - if (dir == from_iso8859) + if (dir == from_8bit) { size_t inchars = *inbufsize; size_t outwchars = data->outbufavail; @@ -124,7 +124,7 @@ gconv (struct gconv_step *step, struct gconv_step_data *data, while (cnt < inchars && (outwchars + sizeof (wchar_t) <= data->outbufsize)) { - wchar_t ch = to_ucs4[(int) inbuf[cnt]]; + wchar_t ch = to_ucs4[(unsigned int) inbuf[cnt]]; if (ch == L'\0' && inbuf[cnt] != '\0') { @@ -191,7 +191,7 @@ gconv (struct gconv_step *step, struct gconv_step_data *data, if (data->is_last) { /* This is the last step. */ - result = (*inbufsize > (dir == from_iso8859 + result = (*inbufsize > (dir == from_8bit ? 0 : sizeof (wchar_t) - 1) ? GCONV_FULL_OUTPUT : GCONV_EMPTY_INPUT); break; diff --git a/iconvdata/Makefile b/iconvdata/Makefile index a79e12a291..f08a090363 100644 --- a/iconvdata/Makefile +++ b/iconvdata/Makefile @@ -22,12 +22,15 @@ subdir := iconvdata # Names of all the shared objects which implement the transformations. -modules := ISO8859-1 ISO8859-2 ISO8859-3 ISO8859-4 ISO8859-5 \ - ISO8859-6 ISO8859-7 ISO8859-8 ISO8859-9 ISO8859-10 \ - T.61 ISO_6937 +modules := ISO8859-1 ISO8859-2 ISO8859-3 ISO8859-4 ISO8859-5 \ + ISO8859-6 ISO8859-7 ISO8859-8 ISO8859-9 ISO8859-10 \ + T.61 ISO_6937 SJIS KOI-8 KOI8-R LATIN-GREEK LATIN-GREEK-1 \ + HP-ROMAN8 EBCDIC-AT-DE EBCDIC-AT-DE-A EBCDIC-CA-FR modules.so := $(addsuffix .so, $(modules)) +include ../Makeconfig + ISO8859-1-routines := iso8859-1 ISO8859-2-routines := iso8859-2 ISO8859-3-routines := iso8859-3 @@ -40,15 +43,29 @@ ISO8859-9-routines := iso8859-9 ISO8859-10-routines := iso8859-10 T.61-routines := t61 ISO_6937-routines := iso6937 - -distribute := iso8859-generic.c \ +SJIS-routines := sjis +KOI-8-routines := koi-8 +KOI8-R-routines := koi8-r +LATIN-GREEK-routines := latin-greek +LATIN-GREEK-1-routines := latin-greek-1 +HP-ROMAN8-routines := hp-roman8 +EBCDIC-AT-DE-routines := ebcdic-at-de +EBCDIC-AT-DE-A-routines := ebcdic-at-de-a +EBCDIC-CA-FR-routines := ebcdic-ca-fr + +extra-libs = libJIS +extra-libs-others = $(extra-libs) + +libJIS-routines := jis0201 jis0208 jis0212 +libJIS-inhibit-o := $(filter-out .os,$(object-suffixes)) + +distribute := 8bit-generic.c 8bit-gap.c gap.pl gaptab.pl \ iso8859-1.c iso8859-2.c iso8859-3.c iso8859-4.c iso8859-5.c \ iso8859-6.c iso8859-7.c iso8859-8.c iso8859-9.c iso8859-10.c\ - iso8859-2.h iso8859-3.h iso8859-4.h iso8859-5.h iso8859-6.h \ - iso8859-7.h iso8859-8.h iso8859-9.h iso8859-10.h t61.c \ - iso6937.c - -include ../Makeconfig + t61.c iso6937.c sjis.c jis0201.h jis0208.h jis0212.h \ + koi-8.c koi8-r.c koi8-r.h hp-roman8.c latin-greek.c \ + latin-greek.h latin-greek-1.c latin-greek-1.h ebcdic-at-de.c\ + ebcdic-at-de-a.c ebcdic-ca-fr.c # We build the transformation modules only when we build shared libs. ifeq (yes,$(build-shared)) @@ -76,39 +93,83 @@ install-others = $(addprefix $(inst_gconvdir)/, $(modules.so)) \ # If we have the localedata add-on available we can build the conversion # tables for numerous charsets. -ifneq (,$(findstring localedata,$(add-ons))) define generate-8bit-table ( echo "static const wchar_t to_ucs4[256] = {"; \ - sed -e 's/^[^[:space:]]*[[:space:]]*.x\(..\)[[:space:]]*<U\(....\)>.*/ [0x\1] = 0x\2,/p' -e d $^ | sort -u; \ + sed -e '/^[^[:space:]]*[[:space:]]*.x00/d' -e 's/^[^[:space:]]*[[:space:]]*.x\(..\)[[:space:]]*<U\(....\)>.*/ [0x\1] = 0x\2,/p' -e d $^ | sort -u; \ + echo "};"; \ + echo "static const char from_ucs4[] = {"; \ + sed -e '/^[^[:space:]]*[[:space:]]*.x00/d' -e 's/^[^[:space:]]*[[:space:]]*.x\(..\)[[:space:]]*<U\(....\)>.*/ [0x\2] = 0x\1,/p' -e d $^ | sort -u; \ + echo "};" ) > $@.new && rm -f $@ && mv $@.new $@ +endef + +sed-generated-headers := iso8859-2.h iso8859-3.h iso8859-4.h iso8859-5.h \ + iso8859-6.h iso8859-7.h iso8859-8.h iso8859-9.h \ + iso8859-10.h koi-8.h hp-roman8.h ebcdic-at-de.h \ + ebcdic-at-de-a.h ebcdic-ca-fr.h + +define generate-8bit-gap-table +( echo "static const wchar_t to_ucs4[256] = {"; \ + sed -e '/^[^[:space:]]*[[:space:]]*.x00/d' -e 's/^[^[:space:]]*[[:space:]]*.x\(..\)[[:space:]]*<U\(....\)>.*/ [0x\1] = 0x\2,/p' -e d $^ | sort -u; \ + echo "};"; \ + echo "static struct gap from_idx[] = {"; \ |