diff options
27 files changed, 594 insertions, 325 deletions
@@ -1,3 +1,44 @@ +2001-12-09 Ulrich Drepper <drepper@redhat.com> + + Implement transliteration of characters in strings of the locale + definitions. + * locale/programs/linereader.c: Adjust for additional parameter to + lr_token. + (get_string): If character <Uxxxx> is not found try to transliterate + it. + * locale/programs/ld-ctype.c: Adjust for additional parameter to + lr_token. Add const to charmap parameter of all functions. + (find_translit, find_translit2): New functions. + * locale/programs/charmap.c: Adjust for additional parameter to + lr_token. + * locale/programs/repertoire.c: Likewise. + * locale/programs/linereader.h: Likewise. + * locale/programs/ld-address.c: Likewise. Add const to charmap + parameter of all functions. + * locale/programs/ld-collate.c: Likewise. + * locale/programs/ld-identification.c: Likewise. + * locale/programs/ld-measurement.c: Likewise. + * locale/programs/ld-messages.c: Likewise. + * locale/programs/ld-monetary.c: Likewise. + * locale/programs/ld-name.c: Likewise. + * locale/programs/ld-numeric.c: Likewise. + * locale/programs/ld-paper.c: Likewise. + * locale/programs/ld-paper.c: Likewise. + * locale/programs/ld-telephone.c: Likewise. + * locale/programs/ld-time.c: Likewise. + * locale/programs/locfile.c: Likewise. + * locale/programs/localedef.c: Likewise. + * locale/programs/localedef.h: Likewise. + * locale/programs/locfile.h: Likewise. Add declaration for + find_translit. + * locale/programs/simple-hash.c: Add const to first parameter of + find_entry, iterate_table, and lookup. + * locale/programs/simple-hash.h: Likewise. + * locale/localeinfo.h: Don't define __LC_LAST here. Include <locale.h> + instead. + * include/locale.h: Define __LC_LAST. + * iconv/Makefile (CFLAGS-linereader.c): Define to -DNO_TRANSLITERATION. + 2001-12-07 Geoff Keating <geoffk@redhat.com> * sysdeps/powerpc/Dist: Update for recent FP changes. @@ -1,4 +1,4 @@ -GNU C Library NEWS -- history of user-visible changes. 2001-11-28 +GNU C Library NEWS -- history of user-visible changes. 2001-12-09 Copyright (C) 1992-2000, 2001 Free Software Foundation, Inc. See the end for copying conditions. @@ -12,6 +12,9 @@ Version 2.3 * iconv (the program and the interface) now accepts empty names (excluding options like //TRANSLIT) to mean "use charset of current locale". + +* localedef now can transliterate characters in strings which are not in + the provided charmap. The information from the input locale is used. Version 2.2.5 diff --git a/iconv/Makefile b/iconv/Makefile index c5d0cd3833..155089b183 100644 --- a/iconv/Makefile +++ b/iconv/Makefile @@ -46,6 +46,7 @@ CFLAGS-iconv_charmap.c = -I../locale/programs CFLAGS-dummy-repertoire.c = -I../locale/programs CFLAGS-charmap.c = -DCHARMAP_PATH='"$(i18ndir)/charmaps"' \ -DDEFAULT_CHARMAP=null_pointer -DNEED_NULL_POINTER +CFLAGS-linereader.c = -DNO_TRANSLITERATION tests = tst-iconv1 tst-iconv2 tst-iconv3 diff --git a/include/locale.h b/include/locale.h index 4374792d14..0877a48775 100644 --- a/include/locale.h +++ b/include/locale.h @@ -1,6 +1,9 @@ #ifndef _LOCALE_H #include <locale/locale.h> +/* This has to be changed whenever a new locale is defined. */ +#define __LC_LAST 13 + /* Locale object for C locale. */ extern struct __locale_struct _nl_C_locobj; diff --git a/locale/localeinfo.h b/locale/localeinfo.h index e7528e9edb..6cd7f0dca2 100644 --- a/locale/localeinfo.h +++ b/locale/localeinfo.h @@ -23,13 +23,11 @@ #include <stddef.h> #include <langinfo.h> #include <limits.h> +#include <locale.h> #include <time.h> #include <stdint.h> #include <sys/types.h> -/* This has to be changed whenever a new locale is defined. */ -#define __LC_LAST 13 - #include <intl/loadinfo.h> /* For loaded_l10nfile definition. */ /* Magic number at the beginning of a locale data file for CATEGORY. */ diff --git a/locale/programs/charmap.c b/locale/programs/charmap.c index 5d2441d17a..7840242d8d 100644 --- a/locale/programs/charmap.c +++ b/locale/programs/charmap.c @@ -117,18 +117,14 @@ charmap_read (const char *filename, int verbose, int be_quiet, int use_default) cmfile = cmlr_open (path, filename, charmap_hash); if (cmfile == NULL) - { - /* Try without the "/charmaps" part. */ - cmfile = cmlr_open (next, filename, charmap_hash); - } + /* Try without the "/charmaps" part. */ + cmfile = cmlr_open (next, filename, charmap_hash); } } if (cmfile == NULL) - { - /* Try the default directory. */ - cmfile = cmlr_open (CHARMAP_PATH, filename, charmap_hash); - } + /* Try the default directory. */ + cmfile = cmlr_open (CHARMAP_PATH, filename, charmap_hash); } } @@ -298,7 +294,7 @@ parse_charmap (struct linereader *cmfile, int verbose, int be_quiet) while (1) { /* What's on? */ - struct token *now = lr_token (cmfile, NULL, NULL, verbose); + struct token *now = lr_token (cmfile, NULL, NULL, NULL, verbose); enum token_t nowtok = now->tok; struct token *arg; @@ -356,7 +352,7 @@ parse_charmap (struct linereader *cmfile, int verbose, int be_quiet) } /* We know that we need an argument. */ - arg = lr_token (cmfile, NULL, NULL, verbose); + arg = lr_token (cmfile, NULL, NULL, NULL, verbose); switch (nowtok) { diff --git a/locale/programs/ld-address.c b/locale/programs/ld-address.c index 96866a2ef6..adf6bbeb28 100644 --- a/locale/programs/ld-address.c +++ b/locale/programs/ld-address.c @@ -95,7 +95,7 @@ address_startup (struct linereader *lr, struct localedef_t *locale, void -address_finish (struct localedef_t *locale, struct charmap_t *charmap) +address_finish (struct localedef_t *locale, const struct charmap_t *charmap) { struct locale_address_t *address = locale->categories[LC_ADDRESS].address; size_t cnt; @@ -324,7 +324,7 @@ address_finish (struct localedef_t *locale, struct charmap_t *charmap) void -address_output (struct localedef_t *locale, struct charmap_t *charmap, +address_output (struct localedef_t *locale, const struct charmap_t *charmap, const char *output_path) { struct locale_address_t *address = locale->categories[LC_ADDRESS].address; @@ -425,7 +425,7 @@ address_output (struct localedef_t *locale, struct charmap_t *charmap, /* The parser for the LC_ADDRESS section of the locale definition. */ void address_read (struct linereader *ldfile, struct localedef_t *result, - struct charmap_t *charmap, const char *repertoire_name, + const struct charmap_t *charmap, const char *repertoire_name, int ignore_content) { struct locale_address_t *address; @@ -439,7 +439,7 @@ address_read (struct linereader *ldfile, struct localedef_t *result, do { - now = lr_token (ldfile, charmap, NULL, verbose); + now = lr_token (ldfile, charmap, result, NULL, verbose); nowtok = now->tok; } while (nowtok == tok_eol); @@ -465,7 +465,7 @@ address_read (struct linereader *ldfile, struct localedef_t *result, /* Ignore empty lines. */ if (nowtok == tok_eol) { - now = lr_token (ldfile, charmap, NULL, verbose); + now = lr_token (ldfile, charmap, result, NULL, verbose); nowtok = now->tok; continue; } @@ -482,7 +482,7 @@ address_read (struct linereader *ldfile, struct localedef_t *result, break; \ } \ \ - arg = lr_token (ldfile, charmap, NULL, verbose); \ + arg = lr_token (ldfile, charmap, result, NULL, verbose); \ if (arg->tok != tok_string) \ goto err_label; \ if (address->cat != NULL) \ @@ -519,7 +519,7 @@ address_read (struct linereader *ldfile, struct localedef_t *result, break; \ } \ \ - arg = lr_token (ldfile, charmap, NULL, verbose); \ + arg = lr_token (ldfile, charmap, result, NULL, verbose); \ if (arg->tok != tok_string && arg->tok != tok_number) \ goto err_label; \ if (address->cat != NULL) \ @@ -557,7 +557,7 @@ address_read (struct linereader *ldfile, struct localedef_t *result, break; \ } \ \ - arg = lr_token (ldfile, charmap, NULL, verbose); \ + arg = lr_token (ldfile, charmap, result, NULL, verbose); \ if (arg->tok != tok_number) \ goto err_label; \ else if (address->cat != 0) \ @@ -571,7 +571,7 @@ address_read (struct linereader *ldfile, struct localedef_t *result, case tok_end: /* Next we assume `LC_ADDRESS'. */ - arg = lr_token (ldfile, charmap, NULL, verbose); + arg = lr_token (ldfile, charmap, result, NULL, verbose); if (arg->tok == tok_eof) break; if (arg->tok == tok_eol) @@ -589,7 +589,7 @@ address_read (struct linereader *ldfile, struct localedef_t *result, } /* Prepare for the next round. */ - now = lr_token (ldfile, charmap, NULL, verbose); + now = lr_token (ldfile, charmap, result, NULL, verbose); nowtok = now->tok; } diff --git a/locale/programs/ld-collate.c b/locale/programs/ld-collate.c index 6d238a658e..4f587c113c 100644 --- a/locale/programs/ld-collate.c +++ b/locale/programs/ld-collate.c @@ -384,8 +384,9 @@ new_symbol (struct locale_collate_t *collate, const char *name, size_t len) /* Test whether this name is already defined somewhere. */ static int check_duplicate (struct linereader *ldfile, struct locale_collate_t *collate, - struct charmap_t *charmap, struct repertoire_t *repertoire, - const char *symbol, size_t symbol_len) + const struct charmap_t *charmap, + struct repertoire_t *repertoire, const char *symbol, + size_t symbol_len) { void *ignore = NULL; @@ -426,13 +427,14 @@ check_duplicate (struct linereader *ldfile, struct locale_collate_t *collate, /* Read the direction specification. */ static void read_directions (struct linereader *ldfile, struct token *arg, - struct charmap_t *charmap, struct repertoire_t *repertoire, - struct locale_collate_t *collate) + const struct charmap_t *charmap, + struct repertoire_t *repertoire, struct localedef_t *result) { int cnt = 0; int max = nrules ?: 10; enum coll_sort_rule *rules = calloc (max, sizeof (*rules)); int warned = 0; + struct locale_collate_t *collate = result->categories[LC_COLLATE].collate; while (1) { @@ -508,7 +510,7 @@ read_directions (struct linereader *ldfile, struct token *arg, } if (valid) - arg = lr_token (ldfile, charmap, repertoire, verbose); + arg = lr_token (ldfile, charmap, result, repertoire, verbose); if (arg->tok == tok_eof || arg->tok == tok_eol || arg->tok == tok_comma || arg->tok == tok_semicolon) @@ -572,7 +574,7 @@ read_directions (struct linereader *ldfile, struct token *arg, } } - arg = lr_token (ldfile, charmap, repertoire, verbose); + arg = lr_token (ldfile, charmap, result, repertoire, verbose); } if (nrules == 0) @@ -660,11 +662,13 @@ unlink_element (struct locale_collate_t *collate) static void insert_weights (struct linereader *ldfile, struct element_t *elem, - struct charmap_t *charmap, struct repertoire_t *repertoire, - struct locale_collate_t *collate, enum token_t ellipsis) + const struct charmap_t *charmap, + struct repertoire_t *repertoire, struct localedef_t *result, + enum token_t ellipsis) { int weight_cnt; struct token *arg; + struct locale_collate_t *collate = result->categories[LC_COLLATE].collate; /* Initialize all the fields. */ elem->file = ldfile->fname; @@ -697,7 +701,7 @@ insert_weights (struct linereader *ldfile, struct element_t *elem, weight_cnt = 0; - arg = lr_token (ldfile, charmap, repertoire, verbose); + arg = lr_token (ldfile, charmap, result, repertoire, verbose); do { if (arg->tok == tok_eof || arg->tok == tok_eol) @@ -867,11 +871,11 @@ insert_weights (struct linereader *ldfile, struct element_t *elem, break; } - arg = lr_token (ldfile, charmap, repertoire, verbose); + arg = lr_token (ldfile, charmap, result, repertoire, verbose); /* This better should be the end of the line or a semicolon. */ if (arg->tok == tok_semicolon) /* OK, ignore this and read the next token. */ - arg = lr_token (ldfile, charmap, repertoire, verbose); + arg = lr_token (ldfile, charmap, result, repertoire, verbose); else if (arg->tok != tok_eof && arg->tok != tok_eol) { /* It's a syntax error. */ @@ -914,13 +918,14 @@ insert_weights (struct linereader *ldfile, struct element_t *elem, static int insert_value (struct linereader *ldfile, const char *symstr, size_t symlen, - struct charmap_t *charmap, struct repertoire_t *repertoire, - struct locale_collate_t *collate) + const struct charmap_t *charmap, struct repertoire_t *repertoire, + struct localedef_t *result) { /* First find out what kind of symbol this is. */ struct charseq *seq; uint32_t wc; struct element_t *elem = NULL; + struct locale_collate_t *collate = result->categories[LC_COLLATE].collate; /* Try to find the character in the charmap. */ seq = charmap_find_value (charmap, symstr, symlen); @@ -1021,7 +1026,7 @@ insert_value (struct linereader *ldfile, const char *symstr, size_t symlen, return 1; } - insert_weights (ldfile, elem, charmap, repertoire, collate, tok_none); + insert_weights (ldfile, elem, charmap, repertoire, result, tok_none); return 0; } @@ -1029,12 +1034,13 @@ insert_value (struct linereader *ldfile, const char *symstr, size_t symlen, static void handle_ellipsis (struct linereader *ldfile, const char *symstr, size_t symlen, - enum token_t ellipsis, struct charmap_t *charmap, + enum token_t ellipsis, const struct charmap_t *charmap, struct repertoire_t *repertoire, - struct locale_collate_t *collate) + struct localedef_t *result) { struct element_t *startp; struct element_t *endp; + struct locale_collate_t *collate = result->categories[LC_COLLATE].collate; /* Unlink the entry added for the ellipsis. */ unlink_element (collate); @@ -1042,7 +1048,7 @@ handle_ellipsis (struct linereader *ldfile, const char *symstr, size_t symlen, /* Process and add the end-entry. */ if (symstr != NULL - && insert_value (ldfile, symstr, symlen, charmap, repertoire, collate)) + && insert_value (ldfile, symstr, symlen, charmap, repertoire, result)) /* Something went wrong with inserting the to-value. This means we cannot process the ellipsis. */ return; @@ -1469,7 +1475,7 @@ collate_startup (struct linereader *ldfile, struct localedef_t *locale, void -collate_finish (struct localedef_t *locale, struct charmap_t *charmap) +collate_finish (struct localedef_t *locale, const struct charmap_t *charmap) { /* Now is the time when we can assign the individual collation values for all the symbols. We have possibly different values @@ -1897,7 +1903,7 @@ output_weightwc (struct obstack *pool, struct locale_collate_t *collate, void -collate_output (struct localedef_t *locale, struct charmap_t *charmap, +collate_output (struct localedef_t *locale, const struct charmap_t *charmap, const char *output_path) { struct locale_collate_t *collate = locale->categories[LC_COLLATE].collate; @@ -2555,7 +2561,7 @@ collate_output (struct localedef_t *locale, struct charmap_t *charmap, void collate_read (struct linereader *ldfile, struct localedef_t *result, - struct charmap_t *charmap, const char *repertoire_name, + const struct charmap_t *charmap, const char *repertoire_name, int ignore_content) { struct repertoire_t *repertoire = NULL; @@ -2585,7 +2591,7 @@ collate_read (struct linereader *ldfile, struct localedef_t *result, do { - now = lr_token (ldfile, charmap, NULL, verbose); + now = lr_token (ldfile, charmap, result, NULL, verbose); nowtok = now->tok; } while (nowtok == tok_eol); @@ -2593,18 +2599,18 @@ collate_read (struct linereader *ldfile, struct localedef_t *result, if (nowtok == tok_copy) { state = 2; - now = lr_token (ldfile, charmap, NULL, verbose); + now = lr_token (ldfile, charmap, result, NULL, verbose); if (now->tok != tok_string) { SYNTAX_ERROR (_("%s: syntax error"), "LC_COLLATE"); skip_category: do - now = lr_token (ldfile, charmap, NULL, verbose); + now = lr_token (ldfile, charmap, result, NULL, verbose); while (now->tok != tok_eof && now->tok != tok_end); if (now->tok != tok_eof - || (now = lr_token (ldfile, charmap, NULL, verbose), + || (now = lr_token (ldfile, charmap, result, NULL, verbose), now->tok == tok_eof)) lr_error (ldfile, _("%s: premature end of file"), "LC_COLLATE"); else if (now->tok != tok_lc_collate) @@ -2634,7 +2640,7 @@ collate_read (struct linereader *ldfile, struct localedef_t *result, lr_ignore_rest (ldfile, 1); - now = lr_token (ldfile, charmap, NULL, verbose); + now = lr_token (ldfile, charmap, result, NULL, verbose); nowtok = now->tok; } @@ -2655,7 +2661,7 @@ collate_read (struct linereader *ldfile, struct localedef_t *result, /* Ingore empty lines. */ if (nowtok == tok_eol) { - now = lr_token (ldfile, charmap, NULL, verbose); + now = lr_token (ldfile, charmap, result, NULL, verbose); nowtok = now->tok; continue; } @@ -2664,7 +2670,7 @@ collate_read (struct linereader *ldfile, struct localedef_t *result, { case tok_copy: /* Allow copying other locales. */ - now = lr_token (ldfile, charmap, NULL, verbose); + now = lr_token (ldfile, charmap, result, NULL, verbose); if (now->tok != tok_string) goto err_label; @@ -2687,7 +2693,7 @@ collate_read (struct linereader *ldfile, struct localedef_t *result, if (state != 0) goto err_label; - arg = lr_token (ldfile, charmap, NULL, verbose); + arg = lr_token (ldfile, charmap, result, NULL, verbose); if (arg->tok != tok_ |