Date: Sat, 12 Jul 2008 14:46:28 GMT From: Konrad Jankowski <konrad@FreeBSD.org> To: Perforce Change Reviews <perforce@FreeBSD.org> Subject: PERFORCE change 145099 for review Message-ID: <200807121446.m6CEkSeI087091@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=145099 Change 145099 by konrad@vspredator on 2008/07/12 14:46:18 My deltas. Affected files ... .. //depot/projects/soc2008/konrad_collation/colldef.apple/Makefile#2 edit .. //depot/projects/soc2008/konrad_collation/colldef.apple/collate.h#2 edit .. //depot/projects/soc2008/konrad_collation/colldef.apple/colldef.1#2 edit .. //depot/projects/soc2008/konrad_collation/colldef.apple/common.h#2 edit .. //depot/projects/soc2008/konrad_collation/colldef.apple/parse.y#2 edit .. //depot/projects/soc2008/konrad_collation/colldef.apple/scan.l#2 edit Differences ... ==== //depot/projects/soc2008/konrad_collation/colldef.apple/Makefile#2 (text+ko) ==== @@ -4,7 +4,7 @@ SRCS= parse.y scan.l y.tab.h LFLAGS= -8 -i CFLAGS+=-I. -CFLAGS+=-DCOLLATE_DEBUG -DYY_NO_UNPUT +CFLAGS+=-DCOLLATE_DEBUG -DYY_NO_UNPUT -Wall -W -ansi -pedantic -Werror LDADD= -ll DPADD= ${LIBL} ==== //depot/projects/soc2008/konrad_collation/colldef.apple/collate.h#2 (text+ko) ==== @@ -62,11 +62,11 @@ __uint8_t directive[COLL_WEIGHTS_MAX]; __uint8_t flags; #if __DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN - __uint8_t directive_count:4; - __uint8_t chain_max_len:4; + int directive_count:4; + int chain_max_len:4; #else - __uint8_t chain_max_len:4; - __uint8_t directive_count:4; + int chain_max_len:4; + int directive_count:4; #endif __int32_t undef_pri[COLL_WEIGHTS_MAX]; __int32_t subst_count[COLL_WEIGHTS_MAX]; ==== //depot/projects/soc2008/konrad_collation/colldef.apple/colldef.1#2 (text+ko) ==== ==== //depot/projects/soc2008/konrad_collation/colldef.apple/common.h#2 (text+ko) ==== @@ -6,8 +6,8 @@ #include <db.h> #include <fcntl.h> -#define CHARMAP_SYMBOL_LEN 64 -#define BUFSIZE 80 +#define CHARMAP_SYMBOL_LEN 256 +#define BUFSIZE 280 #define NOTEXISTS 0 #define EXISTS 1 ==== //depot/projects/soc2008/konrad_collation/colldef.apple/parse.y#2 (text+ko) ==== @@ -39,6 +39,7 @@ #include <unistd.h> #include <sysexits.h> #include <limits.h> +#include <locale.h> #include "collate.h" #include "common.h" @@ -65,13 +66,11 @@ static struct symbol *hassymbolbychar(wchar_t); static void setsymbolbychar(struct symbol *); struct symbol *getstring(const wchar_t *); -static void makeforwardref(int, const struct symbol *, const struct symbol *); static int charpricompar(const void *, const void *); static int substcompar(const void *, const void *); static int chainpricompar(const void *, const void *); static void putsubst(int32_t, int, const wchar_t *); static int hassubst(int32_t, int); -static const wchar_t *__collate_wcsnchr(const wchar_t *, wchar_t, int); static int __collate_wcsnlen(const wchar_t *, int); char *showwcs(const wchar_t *, int); static char *charname(wchar_t); @@ -90,7 +89,6 @@ static DB *chaindb; static int nchain = 0; static DB *stringdb; -static DB *forward_ref[COLL_WEIGHTS_MAX]; static struct symbol *prev_weight_table[COLL_WEIGHTS_MAX]; static struct symbol *prev2_weight_table[COLL_WEIGHTS_MAX]; static struct symbol *weight_table[COLL_WEIGHTS_MAX]; @@ -98,9 +96,9 @@ static struct symbol *prev_elem; static int weight_index = 0; static int allow_ellipsis = 0; -static struct symbol sym_ellipsis = {SYMBOL_ELLIPSIS, PRI_UNDEFINED}; -static struct symbol sym_ignore = {SYMBOL_IGNORE, PRI_IGNORE}; -static struct symbol sym_undefined = {SYMBOL_CHAR, PRI_UNDEFINED}; +static struct symbol sym_ellipsis = {SYMBOL_ELLIPSIS, PRI_UNDEFINED, L"", {0}}; +static struct symbol sym_ignore = {SYMBOL_IGNORE, PRI_IGNORE, L"", {0}}; +static struct symbol sym_undefined = {SYMBOL_CHAR, PRI_UNDEFINED, L"", {0}}; static int order_pass = 0; #undef __collate_char_pri_table @@ -113,7 +111,7 @@ #ifdef COLLATE_DEBUG int debug; #endif -struct __collate_st_info info = {{DIRECTIVE_FORWARD, DIRECTIVE_FORWARD}, 0, 0, 0, {PRI_UNDEFINED, PRI_UNDEFINED}}; +struct __collate_st_info info = {{DIRECTIVE_FORWARD, DIRECTIVE_FORWARD}, 0, 0, 0, {PRI_UNDEFINED, PRI_UNDEFINED}, {PRI_UNDEFINED}, 0, 0}; /* Some of the code expects COLL_WEIGHTS_MAX == 2 */ int directive_count = COLL_WEIGHTS_MAX; @@ -133,6 +131,7 @@ %token <str> ELEM %token <ch> CHAR %token <ch> ORDER_DIRECTIVE +%token EXPAND %% collate : datafile { FILE *fp; @@ -142,7 +141,6 @@ if (nchain > 0) { DBT key, val; struct __collate_st_chain_pri *t, *v; - wchar_t *wp, *tp; int flags, i, len; if ((__collate_chain_pri_table = (struct __collate_st_chain_pri *)malloc(nchain * sizeof(struct __collate_st_chain_pri))) == NULL) @@ -349,7 +347,7 @@ t++; } #endif /* __DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN */ - if (fwrite(__collate_substitute_table[z], sizeof(struct __collate_st_subst), nsubst[z], fp) != nsubst[z]) + if ((int)fwrite(__collate_substitute_table[z], sizeof(struct __collate_st_subst), nsubst[z], fp) != nsubst[z]) err(EX_IOERR, "IO error writting large substprim table %d to destination file %s", z, out_file); @@ -385,7 +383,7 @@ t++; } #endif /* __DARWIN_BYTE_ORDER == __DARWIN_LITTLE_ENDIAN */ - if (fwrite(__collate_large_char_pri_table, sizeof(struct __collate_st_large_char_pri), nlargemap, fp) != nlargemap) + if ((int)fwrite(__collate_large_char_pri_table, sizeof(struct __collate_st_large_char_pri), nlargemap, fp) != nlargemap) err(EX_IOERR, "IO error writting large pri tables to destination file %s", out_file); @@ -482,7 +480,7 @@ } | order_start_list ';' order_start_list_directives { if (directive_count >= COLL_WEIGHTS_MAX) - yyerror("only COLL_WEIGHTS_MAX weights allowed"); + yyerror("only %d weights allowed", COLL_WEIGHTS_MAX); if ((info.directive[directive_count] & DIRECTIVE_DIRECTION_MASK) == 0) info.directive[directive_count] |= DIRECTIVE_FORWARD; directive_count++; @@ -511,6 +509,7 @@ struct symbol *m; wchar_t i; int v; + switch (s->type) { case SYMBOL_CHAIN: yyerror("Chain <%s> can't be endpoints of ellipsis", showwcs($1, CHARMAP_SYMBOL_LEN)); @@ -530,7 +529,7 @@ s->val = prim_pri; prim_pri = s->val + 1; weight_index = 0; -} weights { +} weights2 { int i; struct symbol *s = getsymbol($1, EXISTS); if (s->type == SYMBOL_SYMBOL) { @@ -599,9 +598,10 @@ | order_lines2 order_line2 '\n' ; order_line2 : - | ELEM { weight_index = 0; } weights { + | ELEM { weight_index = 0; } weights2 { int i; struct symbol *s = getsymbol($1, EXISTS); + if (s->val == PRI_UNDEFINED) yyerror("<%s> undefined", showwcs($1, CHARMAP_SYMBOL_LEN)); if (s->type == SYMBOL_SYMBOL) { @@ -623,7 +623,7 @@ for (w = prev_elem->u.wc + 1; w < s->u.wc; w++) { struct __collate_st_char_pri *p = getpri(w); if (p->pri[i] != PRI_UNDEFINED) - yyerror("Char 0x02x previously defined", w); + yyerror("Char 0x%02x previously defined", w); p->pri[i] = prev_weight_table[i]->val; } break; @@ -645,7 +645,7 @@ for (w = prev_elem->u.wc + 1; w < s->u.wc; w++) { struct __collate_st_char_pri *p = getpri(w); if (p->pri[i] != PRI_UNDEFINED) - yyerror("Char 0x02x previously defined", w); + yyerror("Char 0x%02x previously defined", w); p->pri[i] = x++; } break; @@ -653,7 +653,7 @@ for (w = prev_elem->u.wc + 1; w < s->u.wc; w++) { struct __collate_st_char_pri *p = getpri(w); if (p->pri[i] != PRI_UNDEFINED) - yyerror("Char 0x02x previously defined", w); + yyerror("Char 0x%02x previously defined", w); putsubst(w, i, prev_weight_table[i]->u.str); p->pri[i] = prev_weight_table[i]->val; } @@ -671,12 +671,12 @@ case SYMBOL_IGNORE: case SYMBOL_SYMBOL: if (p->pri[i] != PRI_UNDEFINED) - yyerror("Char 0x02x previously defined", s->u.wc); + yyerror("Char 0x%02x previously defined", s->u.wc); p->pri[i] = weight_table[i]->val; break; case SYMBOL_STRING: if (p->pri[i] != PRI_UNDEFINED) - yyerror("Char 0x02x previously defined", s->u.wc); + yyerror("Char 0x%02x previously defined", s->u.wc); putsubst(s->u.wc, i, weight_table[i]->u.str); p->pri[i] = weight_table[i]->val; break; @@ -777,19 +777,22 @@ prev_line = LINE_UNDEFINED; } ; +weights2 : + | weight2 + | weights2 ';' weight2 +; +expansion : + weight_ex + | expansion weight_ex +; +weight2 : weight + | EXPAND expansion EXPAND { weight_index++; } +; weights : | weight | weights ';' weight ; -weight : ELEM { - struct symbol *s; - if (weight_index >= directive_count) - yyerror("More weights than specified by order_start"); - s = getsymbol($1, EXISTS); - if (order_pass && s->val == PRI_UNDEFINED) - yyerror("<%s> is undefined", showwcs($1, CHARMAP_SYMBOL_LEN)); - weight_table[weight_index++] = s; -} +weight : weight_ex1 | ELLIPSIS { if (weight_index >= directive_count) yyerror("More weights than specified by order_start"); @@ -810,6 +813,20 @@ weight_table[weight_index++] = getstring($1); } ; +weight_ex1 : weight_ex { weight_index++; } +; +weight_ex : ELEM { + struct symbol *s; + + if (weight_index >= directive_count) + yyerror("More weights than specified by order_start (%d >= %d, %s)", weight_index, + directive_count, showwcs($1, CHARMAP_SYMBOL_LEN)); + s = getsymbol($1, EXISTS); + if (order_pass && s->val == PRI_UNDEFINED) + printf("<%s> is undefined\n", showwcs($1, CHARMAP_SYMBOL_LEN)); + weight_table[weight_index] = s; +} +; order_end : ORDER_END '\n' ; charmap : DEFN CHAR { @@ -848,7 +865,7 @@ if (tb[0] == '\0') yyerror("\\0 can't be chained"); if (wcslen(curr_chain) + 1 > STR_LEN) - yyerror("Chain '%s' grows too long", curr_chain); + yyerror("Chain '%s' grows too long", showwcs(curr_chain, STR_LEN)); (void)wcscat(curr_chain, tb); } ; @@ -865,7 +882,7 @@ sec_pri = ++prim_pri; } | CHAR RANGE CHAR { - u_int i; + int i; struct __collate_st_char_pri *p; if ($3 <= $1) @@ -907,7 +924,7 @@ p->pri[1] = sec_pri; } | CHAR RANGE CHAR { - u_int i; + int i; struct __collate_st_char_pri *p; if ($3 <= $1) @@ -935,7 +952,7 @@ p->pri[1] = sec_pri++; } | CHAR RANGE CHAR { - u_int i; + int i; struct __collate_st_char_pri *p; if ($3 <= $1) @@ -1006,6 +1023,7 @@ if ((yyin = fopen(*av, "r")) == NULL) err(EX_UNAVAILABLE, "can't open source file %s", *av); } + setlocale(LC_ALL, "en_US.UTF-8"); yyparse(); return 0; } @@ -1104,20 +1122,20 @@ if ((ret = charmapdb->get(charmapdb, &key, &val, 0)) < 0) err(1, "getsymbol: Error getting \"%s\"", showwcs(wcs, CHARMAP_SYMBOL_LEN)); if (ret != 0) { - struct symbol *pn; if (exists > 0) errx(1, "getsymbol: \"%s\" is not defined", showwcs(wcs, CHARMAP_SYMBOL_LEN)); - if ((pn = (struct symbol *)malloc(sizeof(struct symbol))) == NULL) + if ((p = (struct symbol *)malloc(sizeof(struct symbol))) == NULL) err(1, "getsymbol: malloc"); - pn->val = PRI_UNDEFINED; - wcsncpy(pn->name, wcs, CHARMAP_SYMBOL_LEN); - val.data = &pn; + p->val = PRI_UNDEFINED; + wcsncpy(p->name, wcs, CHARMAP_SYMBOL_LEN); + val.data = &p; val.size = sizeof(struct symbol *); if (charmapdb->put(charmapdb, &key, &val, 0) < 0) err(1, "getsymbol: Error storing \"%s\"", showwcs(wcs, CHARMAP_SYMBOL_LEN)); } else if (exists == 0) errx(1, "getsymbol: \"%s\" already exists", showwcs(wcs, CHARMAP_SYMBOL_LEN)); - memcpy(&p, val.data, sizeof(struct symbol *)); + p = *(struct symbol **)val.data; + return p; } @@ -1159,8 +1177,6 @@ setsymbolbychar(struct symbol *s) { DBT key, val; - struct symbol *p; - int ret; key.data = &s->u.wc; key.size = sizeof(wchar_t); @@ -1198,11 +1214,6 @@ } static void -makeforwardref(int i, const struct symbol *from, const struct symbol * to) -{ -} - -static void putsubst(int32_t c, int i, const wchar_t *str) { DBT key, val; @@ -1258,18 +1269,6 @@ return ((struct __collate_st_subst *)a)->val - ((struct __collate_st_subst *)b)->val; } -static const wchar_t * -__collate_wcsnchr(const wchar_t *s, wchar_t c, int len) -{ - while (*s && len > 0) { - if (*s == c) - return s; - s++; - len--; - } - return NULL; -} - static int __collate_wcsnlen(const wchar_t *s, int len) { @@ -1383,8 +1382,9 @@ } } if (info.chain_count > 0) { + struct __collate_st_chain_pri *p2 = __collate_chain_pri_table; + printf("Chain priority table:\n"); - struct __collate_st_chain_pri *p2 = __collate_chain_pri_table; for (i = info.chain_count; i-- > 0; p2++) { printf("\t\"%s\" :", showwcs(p2->str, STR_LEN)); for(z = 0; z < info.directive_count; z++) ==== //depot/projects/soc2008/konrad_collation/colldef.apple/scan.l#2 (text+ko) ==== @@ -53,11 +53,11 @@ #endif /* FLEX_DEBUG */ int yylex(void); static int localedefmode = 0; -static orderpass = 0; +static int orderpass = 0; %} %% <INITIAL,charmap,nchar,subs,subs2,ldef>[ \t]+ ; -<subs2,ldef>\" { ptr = buf; BEGIN(string); } +<subs2,ldef>\" return EXPAND; <string>\< { if(localedefmode) { ptrsave = ptr; @@ -89,11 +89,13 @@ /* The first pass only defined the left-most symbol. We reread the * order lines, and forward references should now be resolved. */ orderpass++; + line_no = 1; YY_FLUSH_BUFFER; rewind(yyin); for(;;) { if (fgets(line, sizeof(line), yyin) == NULL) errx(EX_UNAVAILABLE, "EOF rescanning for order_start"); + line_no++; if (*line == '#') continue; if (strstr(line, "order_start") != NULL) @@ -152,11 +154,22 @@ yylval.ch = v; return CHAR; } -<INITIAL,nchar,subs>\\x([0-9a-fA-F]{2}) { +<INITIAL,nchar,subs>(\\x([0-9a-fA-F]{2})){1,3} { u_int v; + char synthesis[4], *p; + int i; + wchar_t ret; - sscanf(&yytext[2], "%x", &v); - yylval.ch = v; + for (p = yytext, i = 0; *p; p += 4, i++) { + *p = '0'; + sscanf(p, "%x", &v); + synthesis[i] = v; + } + synthesis[i] = 0; + if ((int)(v = mbtowc(&ret, synthesis, i)) != i && synthesis[0]) + errx(EX_UNAVAILABLE, "charmap symbol value decoding error (%d != %d)", v, i); + yylval.ch = ret; + return CHAR; } <INITIAL,nchar,subs>\\. { yylval.ch = yytext[1]; return CHAR; }
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200807121446.m6CEkSeI087091>