Date: Sun, 3 Aug 2008 21:41:34 GMT From: Konrad Jankowski <konrad@FreeBSD.org> To: Perforce Change Reviews <perforce@FreeBSD.org> Subject: PERFORCE change 146565 for review Message-ID: <200808032141.m73LfYoH000912@repoman.freebsd.org>
next in thread | raw e-mail | index | archive | help
http://perforce.freebsd.org/chv.cgi?CH=146565 Change 146565 by konrad@vspredator on 2008/08/03 21:41:32 Finally, only the relevant deltas. Affected files ... .. //depot/projects/soc2008/konrad_collation/libc/locale/collate.h#6 edit .. //depot/projects/soc2008/konrad_collation/libc/locale/collcmp.c#6 edit .. //depot/projects/soc2008/konrad_collation/libc/locale/setlocale.c#6 edit .. //depot/projects/soc2008/konrad_collation/libc/string/strcoll.c#6 edit .. //depot/projects/soc2008/konrad_collation/libc/string/strxfrm.c#6 edit .. //depot/projects/soc2008/konrad_collation/libc/string/wcscoll.c#6 edit .. //depot/projects/soc2008/konrad_collation/libc/string/wcsxfrm.c#6 edit Differences ... ==== //depot/projects/soc2008/konrad_collation/libc/locale/collate.h#6 (text+ko) ==== @@ -24,7 +24,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $FreeBSD: src/lib/libc/locale/collate.h,v 1.15 2005/02/27 20:31:13 ru Exp $ + * $FreeBSD: src/lib/libc/locale/collate.h,v 1.14 2002/08/30 20:26:02 ache Exp $ */ #ifndef _COLLATE_H_ @@ -33,34 +33,91 @@ #include <sys/cdefs.h> #include <sys/types.h> #include <limits.h> +#include "setlocale.h" /* for ENCODING_LEN */ + +#undef COLL_WEIGHTS_MAX +#define COLL_WEIGHTS_MAX 4 #define STR_LEN 10 #define TABLE_SIZE 100 -#define COLLATE_VERSION "1.0\n" -#define COLLATE_VERSION1_2 "1.2\n" +#define COLLATE_VERSION "1.0\n" +#define COLLATE_VERSION1_1 "1.1\n" +#define COLLATE_VERSION1_1A "1.1A\n" +#define COLLATE_VERSION1_2 "1.2\n" +#define COLLATE_VERSION1_3 "1.3\n" +/* see discussion in string/FreeBSD/strxfrm for this value */ +#define COLLATE_MAX_PRIORITY ((1 << 24) - 1) + +#define DIRECTIVE_UNDEF 0x00 +#define DIRECTIVE_FORWARD 0x01 +#define DIRECTIVE_BACKWARD 0x02 +#define DIRECTIVE_POSITION 0x04 + +#define DIRECTIVE_DIRECTION_MASK (DIRECTIVE_FORWARD | DIRECTIVE_BACKWARD) + +#define COLLATE_SUBST_DUP 0x0001 + +#define IGNORE_EQUIV_CLASS 1 + +struct __collate_st_info { + __uint8_t directive[COLL_WEIGHTS_MAX]; + __uint8_t flags; +#if _BYTE_ORDER == _LITTLE_ENDIAN + unsigned int directive_count:4; + unsigned int chain_max_len:4; +#else + unsigned int chain_max_len:4; + unsigned int directive_count:4; +#endif + __int32_t undef_pri[COLL_WEIGHTS_MAX]; + __int32_t subst_count[COLL_WEIGHTS_MAX]; + __int32_t chain_count; + __int32_t large_pri_count; +}; struct __collate_st_char_pri { - int prim, sec; + __int32_t pri[COLL_WEIGHTS_MAX]; }; struct __collate_st_chain_pri { - u_char str[STR_LEN]; - int prim, sec; + wchar_t str[STR_LEN]; + __int32_t pri[COLL_WEIGHTS_MAX]; +}; +struct __collate_st_large_char_pri { + __int32_t val; + struct __collate_st_char_pri pri; +}; +struct __collate_st_subst { + __int32_t val; + wchar_t str[STR_LEN]; +}; + +struct __locale_st_collate { + char __encoding[ENCODING_LEN + 1]; + struct __collate_st_info __info; + struct __collate_st_subst *__substitute_table[COLL_WEIGHTS_MAX]; + struct __collate_st_chain_pri *__chain_pri_table; + struct __collate_st_large_char_pri *__large_char_pri_table; + struct __collate_st_char_pri __char_pri_table[UCHAR_MAX + 1]; }; extern int __collate_load_error; extern int __collate_substitute_nontrivial; -#define __collate_substitute_table (*__collate_substitute_table_ptr) -extern u_char __collate_substitute_table[UCHAR_MAX + 1][STR_LEN]; -#define __collate_char_pri_table (*__collate_char_pri_table_ptr) -extern struct __collate_st_char_pri __collate_char_pri_table[UCHAR_MAX + 1]; -extern struct __collate_st_chain_pri *__collate_chain_pri_table; +extern struct __locale_st_collate *__collate_data; __BEGIN_DECLS -u_char *__collate_strdup(u_char *); -u_char *__collate_substitute(const u_char *); +wchar_t *__collate_mbstowcs(const char *); +wchar_t *__collate_wcsdup(const wchar_t *); +wchar_t *__collate_substitute(const wchar_t *, int); int __collate_load_tables(const char *); -void __collate_lookup(const u_char *, int *, int *, int *); -int __collate_range_cmp(int, int); +void __collate_lookup(const wchar_t *, int *, int *, int *); +void __collate_lookup_which(const wchar_t *, int *, int *, int); +void __collate_xfrm(const wchar_t *, wchar_t **); +int __collate_range_cmp(wchar_t, wchar_t); +size_t __collate_collating_symbol(wchar_t *, size_t, const char *, size_t, + mbstate_t *); +int __collate_equiv_class(const char *, size_t, mbstate_t *); +size_t __collate_equiv_match(int, wchar_t *, size_t, wchar_t, const char *, + size_t, mbstate_t *, size_t *); #ifdef COLLATE_DEBUG void __collate_print_tables(void); #endif ==== //depot/projects/soc2008/konrad_collation/libc/locale/collcmp.c#6 (text+ko) ==== @@ -28,17 +28,18 @@ __FBSDID("$FreeBSD: src/lib/libc/locale/collcmp.c,v 1.18 2005/02/27 14:54:23 phantom Exp $"); #include <string.h> +#include <wchar.h> #include "collate.h" /* * Compare two characters using collate */ -int __collate_range_cmp(int c1, int c2) +int __collate_range_cmp(wchar_t c1, wchar_t c2) { - static char s1[2], s2[2]; + static wchar_t s1[2], s2[2]; s1[0] = c1; s2[0] = c2; - return (strcoll(s1, s2)); + return (wcscoll(s1, s2)); } ==== //depot/projects/soc2008/konrad_collation/libc/locale/setlocale.c#6 (text+ko) ==== @@ -46,6 +46,7 @@ #include <stdlib.h> #include <string.h> #include <unistd.h> +#include <wchar.h> #include "collate.h" #include "lmonetary.h" /* for __monetary_load_locale() */ #include "lnumeric.h" /* for __numeric_load_locale() */ @@ -105,6 +106,10 @@ int i, j, len, saverr; const char *env, *r; +#ifdef LOCALE_DEBUG + fprintf(stderr, "setlocale: %s: %s\n", categories[category], + locale ? locale : "NULL"); +#endif if (category < LC_ALL || category >= _LC_LAST) { errno = EINVAL; return (NULL); @@ -191,11 +196,16 @@ for (i = 1; i < _LC_LAST; ++i) { (void)strcpy(saved_categories[i], current_categories[i]); - if (loadlocale(i) == NULL) { + if ((r = loadlocale(i)) == NULL) { saverr = errno; for (j = 1; j < i; j++) { (void)strcpy(new_categories[j], saved_categories[j]); +#ifdef LOCALE_DEBUG + fprintf(stderr, "setlocale: resetting " + "locale to %s\n", + saved_categories[j]); +#endif if (loadlocale(j) == NULL) { (void)strcpy(new_categories[j], "C"); (void)loadlocale(j); @@ -204,6 +214,9 @@ errno = saverr; return (NULL); } +#ifdef LOCALE_DEBUG + fprintf(stderr, "setlocale: loadlocale returned %s\n", r); +#endif } return (currentlocale()); } @@ -236,6 +249,10 @@ int (*func)(const char *); int saved_errno; +#ifdef LOCALE_DEBUG + fprintf(stderr, "loadlocale: %s->%s (was %s)\n", + categories[category], new, old); +#endif if ((new[0] == '.' && (new[1] == '\0' || (new[1] == '.' && new[2] == '\0'))) || strchr(new, '/') != NULL) { @@ -280,6 +297,9 @@ (void)strcpy(old, new); return (old); } +#ifdef LOCALE_DEBUG + fprintf(stderr, "loadlocale: returning NULL\n"); +#endif return (NULL); } @@ -327,6 +347,9 @@ } else _PathLocale = _PATH_LOCALE; } +#ifdef LOCALE_DEBUG + fprintf(stderr, "__detect_path_locale: %s\n", _PathLocale); +#endif return (0); } ==== //depot/projects/soc2008/konrad_collation/libc/string/strcoll.c#6 (text+ko) ==== @@ -30,57 +30,31 @@ #include <stdlib.h> #include <string.h> +#include <wchar.h> +#include <errno.h> #include "collate.h" int -strcoll(s, s2) - const char *s, *s2; +strcoll(const char *s, const char *s2) { - int len, len2, prim, prim2, sec, sec2, ret, ret2; - const char *t, *t2; - char *tt, *tt2; + int ret; + const wchar_t *t = NULL, *t2 = NULL; + int sverrno; - if (__collate_load_error) + if (__collate_load_error || (t = __collate_mbstowcs(s)) == NULL || + (t2 = __collate_mbstowcs(s2)) == NULL) { + sverrno = errno; + free((void *)t); + free((void *)t2); + errno = sverrno; return strcmp(s, s2); + } - len = len2 = 1; - ret = ret2 = 0; - if (__collate_substitute_nontrivial) { - t = tt = __collate_substitute(s); - t2 = tt2 = __collate_substitute(s2); - } else { - tt = tt2 = NULL; - t = s; - t2 = s2; - } - while(*t && *t2) { - prim = prim2 = 0; - while(*t && !prim) { - __collate_lookup(t, &len, &prim, &sec); - t += len; - } - while(*t2 && !prim2) { - __collate_lookup(t2, &len2, &prim2, &sec2); - t2 += len2; - } - if(!prim || !prim2) - break; - if(prim != prim2) { - ret = prim - prim2; - goto end; - } - if(!ret2) - ret2 = sec - sec2; - } - if(!*t && *t2) - ret = -(int)((u_char)*t2); - else if(*t && !*t2) - ret = (u_char)*t; - else if(!*t && !*t2) - ret = ret2; - end: - free(tt); - free(tt2); + ret = wcscoll(t, t2); + sverrno = errno; + free((void *)t); + free((void *)t2); + errno = sverrno; return ret; } ==== //depot/projects/soc2008/konrad_collation/libc/string/strxfrm.c#6 (text+ko) ==== @@ -30,22 +30,53 @@ #include <stdlib.h> #include <string.h> +#include <wchar.h> +#include <errno.h> #include "collate.h" +/* + * In the non-POSIX case, we transform each character into a string of + * characters representing the character's priority. Since char is usually + * signed, we are limited by 7 bits per byte. To avoid zero, we need to add + * XFRM_OFFSET, so we can't use a full 7 bits. For simplicity, we choose 6 + * bits per byte. We choose 4 bytes per character as a good compromise + * between maximum coverage and minimum size. This gives 24 bits, or 16M + * priorities. So we choose COLLATE_MAX_PRIORITY to be (2^24 - 1). This + * this can be increased if more is needed. + */ + +#define XFRM_BYTES 4 +#define XFRM_OFFSET ('0') /* make all printable characters */ +#define XFRM_SHIFT 6 +#define XFRM_MASK ((1 << XFRM_SHIFT) - 1) + +static void +xfrm(unsigned char *p, int pri) +{ + + p[3] = (pri & XFRM_MASK) + XFRM_OFFSET; + pri >>= XFRM_SHIFT; + p[2] = (pri & XFRM_MASK) + XFRM_OFFSET; + pri >>= XFRM_SHIFT; + p[1] = (pri & XFRM_MASK) + XFRM_OFFSET; + pri >>= XFRM_SHIFT; + p[0] = (pri & XFRM_MASK) + XFRM_OFFSET; +} + size_t strxfrm(char * __restrict dest, const char * __restrict src, size_t len) { - int prim, sec, l; size_t slen; - char *s, *ss; + wchar_t *wcs, *xf[2]; + int sverrno; - if (!*src) { + if (!*src && dest) { if (len > 0) *dest = '\0'; return 0; } - if (__collate_load_error) { + if (__collate_load_error || (wcs = __collate_mbstowcs(src)) == NULL) { slen = strlen(src); if (len > 0) { if (slen < len) @@ -58,26 +89,57 @@ return slen; } - slen = 0; - prim = sec = 0; - ss = s = __collate_substitute(src); - while (*s) { - while (*s && !prim) { - __collate_lookup(s, &l, &prim, &sec); - s += l; + __collate_xfrm(wcs, xf); + + slen = wcslen(xf[0]) * XFRM_BYTES; + if (xf[1]) + slen += (wcslen(xf[1]) + 1) * XFRM_BYTES; + if (len > 0) { + wchar_t *w = xf[0]; + int b = 0; + unsigned char buf[XFRM_BYTES]; + unsigned char *bp; + while (len > 1) { + if (!b) { + if (!*w) + break; + xfrm(bp = buf, *w++); + b = XFRM_BYTES; + } + *dest++ = *(char *)bp++; + b--; + len--; } - if (prim) { - if (len > 1) { - *dest++ = (char)prim; + if ((w = xf[1]) != NULL) { + xfrm(bp = buf, 0); + b = XFRM_BYTES; + while (len > 1) { + if (!b) + break; + *dest++ = *(char *)bp++; + b--; + len--; + } + b = 0; + while (len > 1) { + if (!b) { + if (!*w) + break; + xfrm(bp = buf, *w++); + b = XFRM_BYTES; + } + *dest++ = *(char *)bp++; + b--; len--; } - slen++; - prim = 0; } - } - free(ss); - if (len > 0) - *dest = '\0'; + *dest = 0; + } + sverrno = errno; + free(wcs); + free(xf[0]); + free(xf[1]); + errno = sverrno; return slen; } ==== //depot/projects/soc2008/konrad_collation/libc/string/wcscoll.c#6 (text+ko) ==== @@ -27,72 +27,220 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD: src/lib/libc/string/wcscoll.c,v 1.3 2004/04/07 09:47:56 tjr Exp $"); + #include <errno.h> #include <stdlib.h> #include <string.h> #include <wchar.h> #include "collate.h" -static char *__mbsdup(const wchar_t *); +#define NOTFORWARD (DIRECTIVE_BACKWARD | DIRECTIVE_POSITION) -/* - * Placeholder implementation of wcscoll(). Attempts to use the single-byte - * collation ordering where possible, and falls back on wcscmp() in locales - * with extended character sets. - */ int wcscoll(const wchar_t *ws1, const wchar_t *ws2) { - char *mbs1, *mbs2; - int diff, sverrno; + int sverrno; + int len, len2, prim, prim2, sec, sec2, ret, ret2; + const wchar_t *t, *t2; + wchar_t *tt = NULL, *tt2 = NULL; + wchar_t *tr = NULL, *tr2 = NULL; + struct __collate_st_info *info; - if (__collate_load_error || MB_CUR_MAX > 1) + if (__collate_load_error) /* - * Locale has no special collating order, could not be - * loaded, or has an extended character set; do a fast binary - * comparison. + * Locale has no special collating order or could not be + * loaded, do a fast binary comparison. */ return (wcscmp(ws1, ws2)); - if ((mbs1 = __mbsdup(ws1)) == NULL || (mbs2 = __mbsdup(ws2)) == NULL) { - /* - * Out of memory or illegal wide chars; fall back to wcscmp() - * but leave errno indicating the error. Callers that don't - * check for error will get a reasonable but often slightly - * incorrect result. - */ - sverrno = errno; - free(mbs1); - errno = sverrno; - return (wcscmp(ws1, ws2)); + info = &__collate_data->__info; + len = len2 = 1; + ret = ret2 = 0; + + if ((info->directive[0] & NOTFORWARD) || + (info->directive[1] & NOTFORWARD) || + (!(info->flags && COLLATE_SUBST_DUP) && + (info->subst_count[0] > 0 || info->subst_count[1] > 0))) { + int direc, pass; + for(pass = 0; pass < info->directive_count; pass++) { + direc = info->directive[pass]; + if (pass == 0 || !(info->flags & COLLATE_SUBST_DUP)) { + free(tt); + tt = __collate_substitute(ws1, pass); + free(tt2); + tt2 = tt ? __collate_substitute(ws2, pass) : + NULL; + } + if (direc & DIRECTIVE_BACKWARD) { + wchar_t *bp, *fp, c; + tr = __collate_wcsdup(tt ? tt : ws1); + bp = tr; + fp = tr + wcslen(tr) - 1; + while(bp < fp) { + c = *bp; + *bp++ = *fp; + *fp-- = c; + } + tr2 = __collate_wcsdup(tt2 ? tt2 : ws2); + bp = tr2; + fp = tr2 + wcslen(tr2) - 1; + while(bp < fp) { + c = *bp; + *bp++ = *fp; + *fp-- = c; + } + t = (const wchar_t *)tr; + t2 = (const wchar_t *)tr2; + } else if (tt) { + t = (const wchar_t *)tt; + t2 = (const wchar_t *)tt2; + } else { + t = (const wchar_t *)ws1; + t2 = (const wchar_t *)ws2; + } + if(direc & DIRECTIVE_POSITION) { + while(*t && *t2) { + prim = prim2 = 0; + __collate_lookup_which(t, &len, + &prim, pass); + if (prim <= 0) { + if (prim < 0) { + errno = EINVAL; + ret = -1; + goto end; + } + prim = COLLATE_MAX_PRIORITY; + } + __collate_lookup_which(t2, &len2, + &prim2, pass); + if (prim2 <= 0) { + if (prim2 < 0) { + errno = EINVAL; + ret = -1; + goto end; + } + prim2 = COLLATE_MAX_PRIORITY; + } + if(prim != prim2) { + ret = prim - prim2; + goto end; + } + t += len; + t2 += len2; + } + } else { + while(*t && *t2) { + prim = prim2 = 0; + while(*t) { + __collate_lookup_which(t, &len, + &prim, pass); + if(prim > 0) + break; + if (prim < 0) { + errno = EINVAL; + ret = -1; + goto end; + } + t += len; + } + while(*t2) { + __collate_lookup_which(t2, + &len2, &prim2, pass); + if(prim2 > 0) + break; + if (prim2 < 0) { + errno = EINVAL; + ret = -1; + goto end; + } + t2 += len2; + } + if(!prim || !prim2) + break; + if(prim != prim2) { + ret = prim - prim2; + goto end; + } + t += len; + t2 += len2; + } + } + if(!*t) { + if(*t2) { + ret = -(int)*t2; + goto end; + } + } else { + ret = *t; + goto end; + } + } + ret = 0; + goto end; } - diff = strcoll(mbs1, mbs2); + /* + * Optimized common case: order_start forward;forward and duplicate + * (or no) substitute tables. + */ + tt = __collate_substitute(ws1, 0); + if (tt == NULL) { + tt2 = NULL; + t = (const wchar_t *)ws1; + t2 = (const wchar_t *)ws2; + } else { + tt2 = __collate_substitute(ws2, 0); + t = (const wchar_t *)tt; + t2 = (const wchar_t *)tt2; + } + while(*t && *t2) { + prim = prim2 = 0; + while(*t) { + __collate_lookup(t, &len, &prim, &sec); + if (prim > 0) + break; + if (prim < 0) { + errno = EINVAL; + ret = -1; + goto end; + } + t += len; + } + while(*t2) { + __collate_lookup(t2, &len2, &prim2, &sec2); + if (prim2 > 0) + break; + if (prim2 < 0) { + errno = EINVAL; + ret = -1; + goto end; + } + t2 += len2; + } + if(!prim || !prim2) + break; + if(prim != prim2) { + ret = prim - prim2; + goto end; + } + if(!ret2) + ret2 = sec - sec2; + t += len; + t2 += len2; + } + if(!*t && *t2) + ret = -(int)*t2; + else if(*t && !*t2) + ret = *t; + else if(!*t && !*t2) + ret = ret2; + end: sverrno = errno; - free(mbs1); - free(mbs2); + free(tt); + free(tt2); + free(tr); + free(tr2); errno = sverrno; - return (diff); -} - -static char * -__mbsdup(const wchar_t *ws) -{ - static const mbstate_t initial; - mbstate_t st; - const wchar_t *wcp; - size_t len; - char *mbs; - - wcp = ws; - st = initial; - if ((len = wcsrtombs(NULL, &wcp, 0, &st)) == (size_t)-1) - return (NULL); - if ((mbs = malloc(len + 1)) == NULL) - return (NULL); - st = initial; - wcsrtombs(mbs, &ws, len + 1, &st); - - return (mbs); + return ret; } ==== //depot/projects/soc2008/konrad_collation/libc/string/wcsxfrm.c#6 (text+ko) ==== @@ -34,20 +34,17 @@ #include <stdlib.h> #include <string.h> #include <wchar.h> +#include <errno.h> #include "collate.h" -static char *__mbsdup(const wchar_t *); +#define WCS_XFRM_OFFSET 1 -/* - * Placeholder wcsxfrm() implementation. See wcscoll.c for a description of - * the logic used. - */ size_t wcsxfrm(wchar_t * __restrict dest, const wchar_t * __restrict src, size_t len) { - int prim, sec, l; size_t slen; - char *mbsrc, *s, *ss; + wchar_t *xf[2]; + int sverrno; if (*src == L'\0') { if (len != 0) @@ -55,7 +52,7 @@ return (0); } - if (__collate_load_error || MB_CUR_MAX > 1) { + if (__collate_load_error) { slen = wcslen(src); if (len > 0) { if (slen < len) @@ -68,49 +65,35 @@ return (slen); } - mbsrc = __mbsdup(src); - slen = 0; - prim = sec = 0; - ss = s = __collate_substitute(mbsrc); - while (*s != '\0') { - while (*s != '\0' && prim == 0) { - __collate_lookup(s, &l, &prim, &sec); - s += l; + __collate_xfrm(src, xf); + + slen = wcslen(xf[0]); + if (xf[1]) + slen += wcslen(xf[1]) + 1; + if (len > 0) { + wchar_t *w = xf[0]; + while (len > 1) { + if (!*w) + break; + *dest++ = *w++ + WCS_XFRM_OFFSET; + len--; } - if (prim != 0) { - if (len > 1) { - *dest++ = (wchar_t)prim; + if ((w = xf[1]) != NULL) { + if (len > 1) + *dest++ = WCS_XFRM_OFFSET; + while (len > 1) { + if (!*w) + break; + *dest++ = *w++ + WCS_XFRM_OFFSET; len--; } - slen++; - prim = 0; } - } - free(ss); - free(mbsrc); - if (len != 0) - *dest = L'\0'; - + *dest = 0; + } + sverrno = errno; + free(xf[0]); + free(xf[1]); + errno = sverrno; + return (slen); } - -static char * -__mbsdup(const wchar_t *ws) -{ - static const mbstate_t initial; - mbstate_t st; - const wchar_t *wcp; - size_t len; - char *mbs; - - wcp = ws; - st = initial; - if ((len = wcsrtombs(NULL, &wcp, 0, &st)) == (size_t)-1) - return (NULL); - if ((mbs = malloc(len + 1)) == NULL) - return (NULL); - st = initial; - wcsrtombs(mbs, &ws, len + 1, &st); - - return (mbs); -}
Want to link to this message? Use this URL: <https://mail-archive.FreeBSD.org/cgi/mid.cgi?200808032141.m73LfYoH000912>