X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=blobdiff_plain;f=utf8.c;h=a926735c9e228fe985d0af46f1ddb5c3a5bcfd4d;hb=669ffa903665df001970da4ac0e76e0b64d36fab;hp=743dea4b5c6f8fa4e1cd6f36b81047f419b4e6ec;hpb=551405c409d33bc8cd0a20177c4ee21a204d18b5;p=p5sagit%2Fp5-mst-13.2.git diff --git a/utf8.c b/utf8.c index 743dea4..a926735 100644 --- a/utf8.c +++ b/utf8.c @@ -1,7 +1,7 @@ /* utf8.c * - * Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005 by Larry Wall and - * others + * Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, + * by Larry Wall and others * * You may distribute under the terms of either the GNU General Public * License or the Artistic License, as specified in the README file. @@ -166,12 +166,6 @@ Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) #endif #endif /* Loop style */ } - -U8 * -Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv) -{ - return Perl_uvuni_to_utf8_flags(aTHX_ d, uv, 0); -} /* @@ -192,7 +186,7 @@ five bytes or more. =cut */ STATIC STRLEN -S_is_utf8_char_slow(pTHX_ const U8 *s, const STRLEN len) +S_is_utf8_char_slow(const U8 *s, const STRLEN len) { U8 u = *s; STRLEN slen; @@ -307,9 +301,19 @@ Perl_is_utf8_string(pTHX_ const U8 *s, STRLEN len) } /* +Implemented as a macro in utf8.h + +=for apidoc A|bool|is_utf8_string_loc|const U8 *s|STRLEN len|const U8 **ep + +Like is_utf8_string() but stores the location of the failure (in the +case of "utf8ness failure") or the location s+len (in the case of +"utf8ness success") in the C. + +See also is_utf8_string_loclen() and is_utf8_string(). + =for apidoc A|bool|is_utf8_string_loclen|const U8 *s|STRLEN len|const U8 **ep|const STRLEN *el -Like is_ut8_string() but stores the location of the failure (in the +Like is_utf8_string() but stores the location of the failure (in the case of "utf8ness failure") or the location s+len (in the case of "utf8ness success") in the C, and the number of UTF-8 encoded characters in the C. @@ -368,24 +372,7 @@ Perl_is_utf8_string_loclen(pTHX_ const U8 *s, STRLEN len, const U8 **ep, STRLEN } /* -=for apidoc A|bool|is_utf8_string_loc|const U8 *s|STRLEN len|const U8 **ep|const STRLEN *el - -Like is_ut8_string() but stores the location of the failure (in the -case of "utf8ness failure") or the location s+len (in the case of -"utf8ness success") in the C. - -See also is_utf8_string_loclen() and is_utf8_string(). - -=cut -*/ - -bool -Perl_is_utf8_string_loc(pTHX_ const U8 *s, STRLEN len, const U8 **ep) -{ - return is_utf8_string_loclen(s, len, ep, 0); -} -/* =for apidoc A|UV|utf8n_to_uvuni|const U8 *s|STRLEN curlen|STRLEN *retlen|U32 flags Bottom level UTF-8 decode routine. @@ -412,6 +399,7 @@ Most code should use utf8_to_uvchr() rather than call this directly. UV Perl_utf8n_to_uvuni(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) { + dVAR; const U8 *s0 = s; UV uv = *s, ouv = 0; STRLEN len = 1; @@ -546,12 +534,12 @@ malformed: } if (dowarn) { - SV* const sv = sv_2mortal(newSVpv("Malformed UTF-8 character ", 0)); + SV* const sv = sv_2mortal(newSVpvs("Malformed UTF-8 character ")); switch (warning) { case 0: /* Intentionally empty. */ break; case UTF8_WARN_EMPTY: - Perl_sv_catpv(aTHX_ sv, "(empty string)"); + sv_catpvs(sv, "(empty string)"); break; case UTF8_WARN_CONTINUATION: Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf", with no preceding start byte)", uv); @@ -590,7 +578,7 @@ malformed: Perl_sv_catpvf(aTHX_ sv, "(character 0x%04"UVxf")", uv); break; default: - Perl_sv_catpv(aTHX_ sv, "(unknown reason)"); + sv_catpvs(sv, "(unknown reason)"); break; } @@ -627,8 +615,8 @@ returned and retlen is set, if possible, to -1. UV Perl_utf8_to_uvchr(pTHX_ const U8 *s, STRLEN *retlen) { - return Perl_utf8n_to_uvchr(aTHX_ s, UTF8_MAXBYTES, retlen, - ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); + return utf8n_to_uvchr(s, UTF8_MAXBYTES, retlen, + ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); } /* @@ -668,6 +656,7 @@ up past C, croaks. STRLEN Perl_utf8_length(pTHX_ const U8 *s, const U8 *e) { + dVAR; STRLEN len = 0; /* Note: cannot use UTF8_IS_...() too eagerly here since e.g. @@ -711,6 +700,7 @@ same UTF-8 buffer. IV Perl_utf8_distance(pTHX_ const U8 *a, const U8 *b) { + dVAR; IV off = 0; /* Note: cannot use UTF8_IS_...() too eagerly here since e.g. @@ -862,7 +852,7 @@ Perl_bytes_from_utf8(pTHX_ const U8 *s, STRLEN *len, bool *is_utf8) *is_utf8 = 0; - Newxz(d, (*len) - count + 1, U8); + Newx(d, (*len) - count + 1, U8); s = start; start = d; while (s < send) { U8 c = *s++; @@ -898,7 +888,7 @@ Perl_bytes_to_utf8(pTHX_ const U8 *s, STRLEN *len) U8 *d; U8 *dst; - Newxz(d, (*len) * 2 + 1, U8); + Newx(d, (*len) * 2 + 1, U8); dst = d; while (s < send) { @@ -1249,185 +1239,136 @@ Perl_to_uni_lower_lc(pTHX_ U32 c) return (U32)to_uni_lower(c, tmpbuf, &len); } -bool -Perl_is_utf8_alnum(pTHX_ const U8 *p) +static bool +S_is_utf8_common(pTHX_ const U8 *const p, SV **swash, + const char *const swashname) { + dVAR; if (!is_utf8_char(p)) return FALSE; - if (!PL_utf8_alnum) - /* NOTE: "IsWord", not "IsAlnum", since Alnum is a true - * descendant of isalnum(3), in other words, it doesn't - * contain the '_'. --jhi */ - PL_utf8_alnum = swash_init("utf8", "IsWord", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_alnum, p, TRUE) != 0; -/* return *p == '_' || is_utf8_alpha(p) || is_utf8_digit(p); */ -#ifdef SURPRISINGLY_SLOWER /* probably because alpha is usually true */ - if (!PL_utf8_alnum) - PL_utf8_alnum = swash_init("utf8", "", - sv_2mortal(newSVpv("+utf8::IsAlpha\n+utf8::IsDigit\n005F\n",0)), 0, 0); - return swash_fetch(PL_utf8_alnum, p, TRUE) != 0; -#endif + if (!*swash) + *swash = swash_init("utf8", swashname, &PL_sv_undef, 1, 0); + return swash_fetch(*swash, p, TRUE) != 0; +} + +bool +Perl_is_utf8_alnum(pTHX_ const U8 *p) +{ + dVAR; + /* NOTE: "IsWord", not "IsAlnum", since Alnum is a true + * descendant of isalnum(3), in other words, it doesn't + * contain the '_'. --jhi */ + return S_is_utf8_common(aTHX_ p, &PL_utf8_alnum, "IsWord"); } bool Perl_is_utf8_alnumc(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_alnum) - PL_utf8_alnum = swash_init("utf8", "IsAlnumC", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_alnum, p, TRUE) != 0; -/* return is_utf8_alpha(p) || is_utf8_digit(p); */ -#ifdef SURPRISINGLY_SLOWER /* probably because alpha is usually true */ - if (!PL_utf8_alnum) - PL_utf8_alnum = swash_init("utf8", "", - sv_2mortal(newSVpv("+utf8::IsAlpha\n+utf8::IsDigit\n005F\n",0)), 0, 0); - return swash_fetch(PL_utf8_alnum, p, TRUE) != 0; -#endif + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_alnumc, "IsAlnumC"); } bool Perl_is_utf8_idfirst(pTHX_ const U8 *p) /* The naming is historical. */ { + dVAR; if (*p == '_') return TRUE; - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_idstart) /* is_utf8_idstart would be more logical. */ - PL_utf8_idstart = swash_init("utf8", "IdStart", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_idstart, p, TRUE) != 0; + /* is_utf8_idstart would be more logical. */ + return S_is_utf8_common(aTHX_ p, &PL_utf8_idstart, "IdStart"); } bool Perl_is_utf8_idcont(pTHX_ const U8 *p) { + dVAR; if (*p == '_') return TRUE; - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_idcont) - PL_utf8_idcont = swash_init("utf8", "IdContinue", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_idcont, p, TRUE) != 0; + return S_is_utf8_common(aTHX_ p, &PL_utf8_idcont, "IdContinue"); } bool Perl_is_utf8_alpha(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_alpha) - PL_utf8_alpha = swash_init("utf8", "IsAlpha", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_alpha, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_alpha, "IsAlpha"); } bool Perl_is_utf8_ascii(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_ascii) - PL_utf8_ascii = swash_init("utf8", "IsAscii", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_ascii, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_ascii, "IsAscii"); } bool Perl_is_utf8_space(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_space) - PL_utf8_space = swash_init("utf8", "IsSpacePerl", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_space, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_space, "IsSpacePerl"); } bool Perl_is_utf8_digit(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_digit) - PL_utf8_digit = swash_init("utf8", "IsDigit", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_digit, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_digit, "IsDigit"); } bool Perl_is_utf8_upper(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_upper) - PL_utf8_upper = swash_init("utf8", "IsUppercase", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_upper, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_upper, "IsUppercase"); } bool Perl_is_utf8_lower(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_lower) - PL_utf8_lower = swash_init("utf8", "IsLowercase", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_lower, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_lower, "IsLowercase"); } bool Perl_is_utf8_cntrl(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_cntrl) - PL_utf8_cntrl = swash_init("utf8", "IsCntrl", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_cntrl, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_cntrl, "IsCntrl"); } bool Perl_is_utf8_graph(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_graph) - PL_utf8_graph = swash_init("utf8", "IsGraph", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_graph, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_graph, "IsGraph"); } bool Perl_is_utf8_print(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_print) - PL_utf8_print = swash_init("utf8", "IsPrint", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_print, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_print, "IsPrint"); } bool Perl_is_utf8_punct(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_punct) - PL_utf8_punct = swash_init("utf8", "IsPunct", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_punct, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_punct, "IsPunct"); } bool Perl_is_utf8_xdigit(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_xdigit) - PL_utf8_xdigit = swash_init("utf8", "IsXDigit", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_xdigit, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_xdigit, "Isxdigit"); } bool Perl_is_utf8_mark(pTHX_ const U8 *p) { - if (!is_utf8_char(p)) - return FALSE; - if (!PL_utf8_mark) - PL_utf8_mark = swash_init("utf8", "IsM", &PL_sv_undef, 0, 0); - return swash_fetch(PL_utf8_mark, p, TRUE) != 0; + dVAR; + return S_is_utf8_common(aTHX_ p, &PL_utf8_mark, "IsM"); } /* @@ -1443,7 +1384,7 @@ of the result. The "swashp" is a pointer to the swash to use. Both the special and normal mappings are stored lib/unicore/To/Foo.pl, -and loaded by SWASHGET, using lib/utf8_heavy.pl. The special (usually, +and loaded by SWASHNEW, using lib/utf8_heavy.pl. The special (usually, but not always, a multicharacter mapping), is tried first. The "special" is a string like "utf8::ToSpecLower", which means the @@ -1459,6 +1400,7 @@ UV Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, SV **swashp, const char *normal, const char *special) { + dVAR; U8 tmpbuf[UTF8_MAXBYTES_CASE+1]; STRLEN len = 0; @@ -1559,6 +1501,7 @@ The first character of the uppercased version is returned UV Perl_to_utf8_upper(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) { + dVAR; return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, &PL_utf8_toupper, "ToUpper", "utf8::ToSpecUpper"); } @@ -1579,6 +1522,7 @@ The first character of the titlecased version is returned UV Perl_to_utf8_title(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) { + dVAR; return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, &PL_utf8_totitle, "ToTitle", "utf8::ToSpecTitle"); } @@ -1599,6 +1543,7 @@ The first character of the lowercased version is returned UV Perl_to_utf8_lower(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) { + dVAR; return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, &PL_utf8_tolower, "ToLower", "utf8::ToSpecLower"); } @@ -1620,12 +1565,17 @@ The first character of the foldcased version is returned UV Perl_to_utf8_fold(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) { + dVAR; return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, &PL_utf8_tofold, "ToFold", "utf8::ToSpecFold"); } -/* a "swash" is a swatch hash */ - +/* Note: + * A "swash" is a swatch hash. + * A "swatch" is a bit vector generated by utf8.c:S_swash_get(). + * C is a pointer to a package name for SWASHNEW, should be "utf8". + * For other parameters, see utf8::SWASHNEW in lib/utf8_heavy.pl. + */ SV* Perl_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 minbits, I32 none) { @@ -1647,7 +1597,7 @@ Perl_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 minbits ENTER; errsv_save = newSVsv(ERRSV); Perl_load_module(aTHX_ PERL_LOADMOD_NOIMPORT, newSVpvn(pkg,pkg_len), - Nullsv); + NULL); if (!SvTRUE(ERRSV)) sv_setsv(ERRSV, errsv_save); SvREFCNT_dec(errsv_save); @@ -1700,25 +1650,33 @@ Perl_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 minbits * (see lib/unicore/SpecCase.txt) The SWASHGET in lib/utf8_heavy.pl is * the lower-level routine, and it is similarly broken for returning * multiple values. --jhi */ +/* Now SWASHGET is recasted into S_swash_get in this file. */ + +/* Note: + * Returns the value of property/mapping C for the first character + * of the string C. If C is true, the string C is + * assumed to be in utf8. If C is false, the string C is + * assumed to be in native 8-bit encoding. Caches the swatch in C. + */ UV -Perl_swash_fetch(pTHX_ SV *sv, const U8 *ptr, bool do_utf8) +Perl_swash_fetch(pTHX_ SV *swash, const U8 *ptr, bool do_utf8) { dVAR; - HV* const hv = (HV*)SvRV(sv); + HV* const hv = (HV*)SvRV(swash); U32 klen; U32 off; STRLEN slen; STRLEN needents; const U8 *tmps = NULL; U32 bit; - SV *retval; + SV *swatch; U8 tmputf8[2]; UV c = NATIVE_TO_ASCII(*ptr); if (!do_utf8 && !UNI_IS_INVARIANT(c)) { - tmputf8[0] = (U8)UTF8_EIGHT_BIT_HI(c); - tmputf8[1] = (U8)UTF8_EIGHT_BIT_LO(c); - ptr = tmputf8; + tmputf8[0] = (U8)UTF8_EIGHT_BIT_HI(c); + tmputf8[1] = (U8)UTF8_EIGHT_BIT_LO(c); + ptr = tmputf8; } /* Given a UTF-X encoded char 0xAA..0xYY,0xZZ * then the "swatch" is a vec() for al the chars which start @@ -1728,20 +1686,18 @@ Perl_swash_fetch(pTHX_ SV *sv, const U8 *ptr, bool do_utf8) klen = UTF8SKIP(ptr) - 1; off = ptr[klen]; - if (klen == 0) - { + if (klen == 0) { /* If char in invariant then swatch is for all the invariant chars * In both UTF-8 and UTF-8-MOD that happens to be UTF_CONTINUATION_MARK */ - needents = UTF_CONTINUATION_MARK; - off = NATIVE_TO_UTF(ptr[klen]); - } - else - { + needents = UTF_CONTINUATION_MARK; + off = NATIVE_TO_UTF(ptr[klen]); + } + else { /* If char is encoded then swatch is for the prefix */ - needents = (1 << UTF_ACCUMULATION_SHIFT); - off = NATIVE_TO_UTF(ptr[klen]) & UTF_CONTINUATION_MASK; - } + needents = (1 << UTF_ACCUMULATION_SHIFT); + off = NATIVE_TO_UTF(ptr[klen]) & UTF_CONTINUATION_MASK; + } /* * This single-entry cache saves about 1/3 of the utf8 overhead in test @@ -1763,46 +1719,28 @@ Perl_swash_fetch(pTHX_ SV *sv, const U8 *ptr, bool do_utf8) /* Try our second-level swatch cache, kept in a hash. */ SV** svp = hv_fetch(hv, (const char*)ptr, klen, FALSE); - /* If not cached, generate it via utf8::SWASHGET */ - if (!svp || !SvPOK(*svp) || !(tmps = (const U8*)SvPV_const(*svp, slen))) { - dSP; + /* If not cached, generate it via swash_get */ + if (!svp || !SvPOK(*svp) + || !(tmps = (const U8*)SvPV_const(*svp, slen))) { /* We use utf8n_to_uvuni() as we want an index into Unicode tables, not a native character number. */ const UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXBYTES, 0, ckWARN(WARN_UTF8) ? 0 : UTF8_ALLOW_ANY); - SV *errsv_save; - ENTER; - SAVETMPS; - save_re_context(); - PUSHSTACKi(PERLSI_MAGIC); - PUSHMARK(SP); - EXTEND(SP,3); - PUSHs((SV*)sv); - /* On EBCDIC & ~(0xA0-1) isn't a useful thing to do */ - PUSHs(sv_2mortal(newSViv((klen) ? - (code_point & ~(needents - 1)) : 0))); - PUSHs(sv_2mortal(newSViv(needents))); - PUTBACK; - errsv_save = newSVsv(ERRSV); - if (call_method("SWASHGET", G_SCALAR)) - retval = newSVsv(*PL_stack_sp--); - else - retval = &PL_sv_undef; - if (!SvTRUE(ERRSV)) - sv_setsv(ERRSV, errsv_save); - SvREFCNT_dec(errsv_save); - POPSTACK; - FREETMPS; - LEAVE; + swatch = swash_get(swash, + /* On EBCDIC & ~(0xA0-1) isn't a useful thing to do */ + (klen) ? (code_point & ~(needents - 1)) : 0, + needents); + if (IN_PERL_COMPILETIME) PL_curcop->op_private = (U8)(PL_hints & HINT_PRIVATE_MASK); - svp = hv_store(hv, (const char *)ptr, klen, retval, 0); + svp = hv_store(hv, (const char *)ptr, klen, swatch, 0); - if (!svp || !(tmps = (U8*)SvPV(*svp, slen)) || (slen << 3) < needents) - Perl_croak(aTHX_ "SWASHGET didn't return result of proper length"); + if (!svp || !(tmps = (U8*)SvPV(*svp, slen)) + || (slen << 3) < needents) + Perl_croak(aTHX_ "panic: swash_fetch got improper swatch"); } PL_last_swash_hv = hv; @@ -1828,10 +1766,321 @@ Perl_swash_fetch(pTHX_ SV *sv, const U8 *ptr, bool do_utf8) off <<= 2; return (tmps[off] << 24) + (tmps[off+1] << 16) + (tmps[off+2] << 8) + tmps[off + 3] ; } - Perl_croak(aTHX_ "panic: swash_fetch"); + Perl_croak(aTHX_ "panic: swash_fetch got swatch of unexpected bit width"); return 0; } +/* Note: + * Returns a swatch (a bit vector string) for a code point sequence + * that starts from the value C and comprises the number C. + * A C must be an object created by SWASHNEW (see lib/utf8_heavy.pl). + * Should be used via swash_fetch, which will cache the swatch in C. + */ +STATIC SV* +S_swash_get(pTHX_ SV* swash, UV start, UV span) +{ + SV *swatch; + U8 *l, *lend, *x, *xend, *s; + STRLEN lcur, xcur, scur; + + HV* const hv = (HV*)SvRV(swash); + SV** const listsvp = hv_fetchs(hv, "LIST", FALSE); + SV** const typesvp = hv_fetchs(hv, "TYPE", FALSE); + SV** const bitssvp = hv_fetchs(hv, "BITS", FALSE); + SV** const nonesvp = hv_fetchs(hv, "NONE", FALSE); + SV** const extssvp = hv_fetchs(hv, "EXTRAS", FALSE); + const U8* const typestr = (U8*)SvPV_nolen(*typesvp); + const int typeto = typestr[0] == 'T' && typestr[1] == 'o'; + const STRLEN bits = SvUV(*bitssvp); + const STRLEN octets = bits >> 3; /* if bits == 1, then octets == 0 */ + const UV none = SvUV(*nonesvp); + const UV end = start + span; + + if (bits != 1 && bits != 8 && bits != 16 && bits != 32) { + Perl_croak(aTHX_ "panic: swash_get doesn't expect bits %"UVuf, + (UV)bits); + } + + /* create and initialize $swatch */ + swatch = newSVpvs(""); + scur = octets ? (span * octets) : (span + 7) / 8; + SvGROW(swatch, scur + 1); + s = (U8*)SvPVX(swatch); + if (octets && none) { + const U8* const e = s + scur; + while (s < e) { + if (bits == 8) + *s++ = (U8)(none & 0xff); + else if (bits == 16) { + *s++ = (U8)((none >> 8) & 0xff); + *s++ = (U8)( none & 0xff); + } + else if (bits == 32) { + *s++ = (U8)((none >> 24) & 0xff); + *s++ = (U8)((none >> 16) & 0xff); + *s++ = (U8)((none >> 8) & 0xff); + *s++ = (U8)( none & 0xff); + } + } + *s = '\0'; + } + else { + (void)memzero((U8*)s, scur + 1); + } + SvCUR_set(swatch, scur); + s = (U8*)SvPVX(swatch); + + /* read $swash->{LIST} */ + l = (U8*)SvPV(*listsvp, lcur); + lend = l + lcur; + while (l < lend) { + UV min, max, val, key; + STRLEN numlen; + I32 flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX; + + U8* const nl = (U8*)memchr(l, '\n', lend - l); + + numlen = lend - l; + min = grok_hex((char *)l, &numlen, &flags, NULL); + if (numlen) + l += numlen; + else if (nl) { + l = nl + 1; /* 1 is length of "\n" */ + continue; + } + else { + l = lend; /* to LIST's end at which \n is not found */ + break; + } + + if (isBLANK(*l)) { + ++l; + flags = PERL_SCAN_SILENT_ILLDIGIT | PERL_SCAN_DISALLOW_PREFIX; + numlen = lend - l; + max = grok_hex((char *)l, &numlen, &flags, NULL); + if (numlen) + l += numlen; + else + max = min; + + if (octets) { + if (isBLANK(*l)) { + ++l; + flags = PERL_SCAN_SILENT_ILLDIGIT | + PERL_SCAN_DISALLOW_PREFIX; + numlen = lend - l; + val = grok_hex((char *)l, &numlen, &flags, NULL); + if (numlen) + l += numlen; + else + val = 0; + } + else { + val = 0; + if (typeto) { + Perl_croak(aTHX_ "%s: illegal mapping '%s'", + typestr, l); + } + } + } + else + val = 0; /* bits == 1, then val should be ignored */ + } + else { + max = min; + if (octets) { + val = 0; + if (typeto) { + Perl_croak(aTHX_ "%s: illegal mapping '%s'", typestr, l); + } + } + else + val = 0; /* bits == 1, then val should be ignored */ + } + + if (nl) + l = nl + 1; + else + l = lend; + + if (max < start) + continue; + + if (octets) { + if (min < start) { + if (!none || val < none) { + val += start - min; + } + min = start; + } + for (key = min; key <= max; key++) { + STRLEN offset; + if (key >= end) + goto go_out_list; + /* offset must be non-negative (start <= min <= key < end) */ + offset = octets * (key - start); + if (bits == 8) + s[offset] = (U8)(val & 0xff); + else if (bits == 16) { + s[offset ] = (U8)((val >> 8) & 0xff); + s[offset + 1] = (U8)( val & 0xff); + } + else if (bits == 32) { + s[offset ] = (U8)((val >> 24) & 0xff); + s[offset + 1] = (U8)((val >> 16) & 0xff); + s[offset + 2] = (U8)((val >> 8) & 0xff); + s[offset + 3] = (U8)( val & 0xff); + } + + if (!none || val < none) + ++val; + } + } + else { /* bits == 1, then val should be ignored */ + if (min < start) + min = start; + for (key = min; key <= max; key++) { + const STRLEN offset = (STRLEN)(key - start); + if (key >= end) + goto go_out_list; + s[offset >> 3] |= 1 << (offset & 7); + } + } + } /* while */ + go_out_list: + + /* read $swash->{EXTRAS} */ + x = (U8*)SvPV(*extssvp, xcur); + xend = x + xcur; + while (x < xend) { + STRLEN namelen; + U8 *namestr; + SV** othersvp; + HV* otherhv; + STRLEN otherbits; + SV **otherbitssvp, *other; + U8 *s, *o, *nl; + STRLEN slen, olen; + + U8 opc = *x++; + if (opc == '\n') + continue; + + nl = (U8*)memchr(x, '\n', xend - x); + + if (opc != '-' && opc != '+' && opc != '!' && opc != '&') { + if (nl) { + x = nl + 1; /* 1 is length of "\n" */ + continue; + } + else { + x = xend; /* to EXTRAS' end at which \n is not found */ + break; + } + } + + namestr = x; + if (nl) { + namelen = nl - namestr; + x = nl + 1; + } + else { + namelen = xend - namestr; + x = xend; + } + + othersvp = hv_fetch(hv, (char *)namestr, namelen, FALSE); + otherhv = (HV*)SvRV(*othersvp); + otherbitssvp = hv_fetchs(otherhv, "BITS", FALSE); + otherbits = (STRLEN)SvUV(*otherbitssvp); + if (bits < otherbits) + Perl_croak(aTHX_ "panic: swash_get found swatch size mismatch"); + + /* The "other" swatch must be destroyed after. */ + other = swash_get(*othersvp, start, span); + o = (U8*)SvPV(other, olen); + + if (!olen) + Perl_croak(aTHX_ "panic: swash_get got improper swatch"); + + s = (U8*)SvPV(swatch, slen); + if (bits == 1 && otherbits == 1) { + if (slen != olen) + Perl_croak(aTHX_ "panic: swash_get found swatch length mismatch"); + + switch (opc) { + case '+': + while (slen--) + *s++ |= *o++; + break; + case '!': + while (slen--) + *s++ |= ~*o++; + break; + case '-': + while (slen--) + *s++ &= ~*o++; + break; + case '&': + while (slen--) + *s++ &= *o++; + break; + default: + break; + } + } + else { + STRLEN otheroctets = otherbits >> 3; + STRLEN offset = 0; + U8* send = s + slen; + + while (s < send) { + UV otherval = 0; + + if (otherbits == 1) { + otherval = (o[offset >> 3] >> (offset & 7)) & 1; + ++offset; + } + else { + STRLEN vlen = otheroctets; + otherval = *o++; + while (--vlen) { + otherval <<= 8; + otherval |= *o++; + } + } + + if (opc == '+' && otherval) + ; /* replace with otherval */ + else if (opc == '!' && !otherval) + otherval = 1; + else if (opc == '-' && otherval) + otherval = 0; + else if (opc == '&' && !otherval) + otherval = 0; + else { + s += octets; /* no replacement */ + continue; + } + + if (bits == 8) + *s++ = (U8)( otherval & 0xff); + else if (bits == 16) { + *s++ = (U8)((otherval >> 8) & 0xff); + *s++ = (U8)( otherval & 0xff); + } + else if (bits == 32) { + *s++ = (U8)((otherval >> 24) & 0xff); + *s++ = (U8)((otherval >> 16) & 0xff); + *s++ = (U8)((otherval >> 8) & 0xff); + *s++ = (U8)( otherval & 0xff); + } + } + } + sv_free(other); /* through with it! */ + } /* while */ + return swatch; +} /* =for apidoc A|U8 *|uvchr_to_utf8|U8 *d|UV uv @@ -1853,7 +2102,6 @@ is the recommended wide native character-aware way of saying /* On ASCII machines this is normally a macro but we want a real function in case XS code wants it */ -#undef Perl_uvchr_to_utf8 U8 * Perl_uvchr_to_utf8(pTHX_ U8 *d, UV uv) { @@ -1867,9 +2115,11 @@ Perl_uvchr_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) } /* -=for apidoc A|UV|utf8n_to_uvchr|U8 *s|STRLEN curlen|STRLEN *retlen|U32 flags +=for apidoc A|UV|utf8n_to_uvchr|U8 *s|STRLEN curlen|STRLEN *retlen|U32 +flags -Returns the native character value of the first character in the string C +Returns the native character value of the first character in the string +C which is assumed to be in UTF-8 encoding; C will be set to the length, in bytes, of that character. @@ -1880,9 +2130,9 @@ Allows length and flags to be passed to low level routine. /* On ASCII machines this is normally a macro but we want a real function in case XS code wants it */ -#undef Perl_utf8n_to_uvchr UV -Perl_utf8n_to_uvchr(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) +Perl_utf8n_to_uvchr(pTHX_ const U8 *s, STRLEN curlen, STRLEN *retlen, +U32 flags) { const UV uv = Perl_utf8n_to_uvuni(aTHX_ s, curlen, retlen, flags); return UNI_TO_NATIVE(uv); @@ -1926,7 +2176,7 @@ Perl_pv_uni_display(pTHX_ SV *dsv, const U8 *spv, STRLEN len, STRLEN pvlim, UV f u = utf8_to_uvchr((U8*)s, 0); if (u < 256) { const unsigned char c = (unsigned char)u & 0xFF; - if (!ok && (flags & UNI_DISPLAY_BACKSLASH)) { + if (flags & UNI_DISPLAY_BACKSLASH) { switch (c) { case '\n': ok = 'n'; break; @@ -1956,7 +2206,7 @@ Perl_pv_uni_display(pTHX_ SV *dsv, const U8 *spv, STRLEN len, STRLEN pvlim, UV f Perl_sv_catpvf(aTHX_ dsv, "\\x{%"UVxf"}", u); } if (truncated) - sv_catpvn(dsv, "...", 3); + sv_catpvs(dsv, "..."); return SvPVX(dsv); } @@ -2008,11 +2258,15 @@ http://www.unicode.org/unicode/reports/tr21/ (Case Mappings). I32 Perl_ibcmp_utf8(pTHX_ const char *s1, char **pe1, register UV l1, bool u1, const char *s2, char **pe2, register UV l2, bool u2) { + dVAR; register const U8 *p1 = (const U8*)s1; register const U8 *p2 = (const U8*)s2; - register const U8 *f1 = 0, *f2 = 0; - register U8 *e1 = 0, *q1 = 0; - register U8 *e2 = 0, *q2 = 0; + register const U8 *f1 = NULL; + register const U8 *f2 = NULL; + register U8 *e1 = NULL; + register U8 *q1 = NULL; + register U8 *e2 = NULL; + register U8 *q2 = NULL; STRLEN n1 = 0, n2 = 0; U8 foldbuf1[UTF8_MAXBYTES_CASE+1]; U8 foldbuf2[UTF8_MAXBYTES_CASE+1];