X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=blobdiff_plain;f=utf8.c;h=debfb9ceac8b95c4a1d40d0151c5db5ef3b614dc;hb=b6512f489e761186d508cf0b8b7705805cfefc52;hp=b682cf65caabf5c7f3766bf7e50ef6d93db5d510;hpb=3568d8383f3d0b22eb07927391114af2a91b06ed;p=p5sagit%2Fp5-mst-13.2.git diff --git a/utf8.c b/utf8.c index b682cf6..debfb9c 100644 --- a/utf8.c +++ b/utf8.c @@ -27,15 +27,23 @@ /* Unicode support */ /* -=for apidoc A|U8 *|uvuni_to_utf8|U8 *d|UV uv +=for apidoc A|U8 *|uvuni_to_utf8_flags|U8 *d|UV uv|UV flags Adds the UTF8 representation of the Unicode codepoint C to the end of the string C; C should be have at least C free bytes available. The return value is the pointer to the byte after the end of the new character. In other words, + d = uvuni_to_utf8_flags(d, uv, flags); + +or, in most cases, + d = uvuni_to_utf8(d, uv); +(which is equivalent to) + + d = uvuni_to_utf8_flags(d, uv, 0); + is the recommended Unicode-aware way of saying *(d++) = uv; @@ -44,13 +52,34 @@ is the recommended Unicode-aware way of saying */ U8 * -Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv) -{ +Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) +{ + if (ckWARN_d(WARN_UTF8)) { + if (UNICODE_IS_SURROGATE(uv) && + !(flags & UNICODE_ALLOW_SURROGATE)) + Perl_warner(aTHX_ WARN_UTF8, "UTF-16 surrogate 0x%04"UVxf, uv); + else if ( + ((uv >= 0xFDD0 && uv <= 0xFDEF && + !(flags & UNICODE_ALLOW_FDD0)) + || + ((uv & 0xFFFF) == 0xFFFE && + !(flags & UNICODE_ALLOW_FFFE)) + || + ((uv & 0xFFFF) == 0xFFFF && + !(flags & UNICODE_ALLOW_FFFF))) && + /* UNICODE_ALLOW_SUPER includes + * FFFEs and FFFFs beyond 0x10FFFF. */ + ((uv <= PERL_UNICODE_MAX) || + !(flags & UNICODE_ALLOW_SUPER)) + ) + Perl_warner(aTHX_ WARN_UTF8, + "Unicode character 0x%04"UVxf" is illegal", uv); + } if (UNI_IS_INVARIANT(uv)) { *d++ = UTF_TO_NATIVE(uv); return d; } -#if defined(EBCDIC) || 1 /* always for testing */ +#if defined(EBCDIC) else { STRLEN len = UNISKIP(uv); U8 *p = d+len-1; @@ -130,7 +159,12 @@ Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv) #endif #endif /* Loop style */ } - + +U8 * +Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv) +{ + return Perl_uvuni_to_utf8_flags(aTHX_ d, uv, 0); +} /* @@ -243,9 +277,11 @@ Most code should use utf8_to_uvchr() rather than call this directly. UV Perl_utf8n_to_uvuni(pTHX_ U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) { - UV uv = *s, ouv; + U8 *s0 = s; + UV uv = *s, ouv = 0; STRLEN len = 1; bool dowarn = ckWARN_d(WARN_UTF8); + UV startbyte = *s; STRLEN expectlen = 0; U32 warning = 0; @@ -388,22 +424,28 @@ malformed: Perl_sv_catpvf(aTHX_ sv, "(empty string)"); break; case UTF8_WARN_CONTINUATION: - Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf")", uv); + Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf", with no preceding start byte)", uv); break; case UTF8_WARN_NON_CONTINUATION: - Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf" after start byte 0x%02"UVxf")", - (UV)s[1], uv); + if (s == s0) + Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", immediately after start byte 0x%02"UVxf")", + (UV)s[1], startbyte); + else + Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", %d byte%s after start byte 0x%02"UVxf", expected %d bytes)", + (UV)s[1], s - s0, s - s0 > 1 ? "s" : "", startbyte, expectlen); + break; case UTF8_WARN_FE_FF: Perl_sv_catpvf(aTHX_ sv, "(byte 0x%02"UVxf")", uv); break; case UTF8_WARN_SHORT: - Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d)", - curlen, curlen == 1 ? "" : "s", expectlen); + Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", + curlen, curlen == 1 ? "" : "s", expectlen, startbyte); + expectlen = curlen; /* distance for caller to skip */ break; case UTF8_WARN_OVERFLOW: - Perl_sv_catpvf(aTHX_ sv, "(overflow at 0x%"UVxf", byte 0x%02x)", - ouv, *s); + Perl_sv_catpvf(aTHX_ sv, "(overflow at 0x%"UVxf", byte 0x%02x, after start byte 0x%02"UVxf")", + ouv, *s, startbyte); break; case UTF8_WARN_SURROGATE: Perl_sv_catpvf(aTHX_ sv, "(UTF-16 surrogate 0x%04"UVxf")", uv); @@ -412,8 +454,8 @@ malformed: Perl_sv_catpvf(aTHX_ sv, "(byte order mark 0x%04"UVxf")", uv); break; case UTF8_WARN_LONG: - Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d)", - expectlen, expectlen == 1 ? "": "s", UNISKIP(uv)); + Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", + expectlen, expectlen == 1 ? "": "s", UNISKIP(uv), startbyte); break; case UTF8_WARN_FFFF: Perl_sv_catpvf(aTHX_ sv, "(character 0x%04"UVxf")", uv); @@ -428,7 +470,7 @@ malformed: if (PL_op) Perl_warner(aTHX_ WARN_UTF8, - "%s in %s", s, PL_op_desc[PL_op->op_type]); + "%s in %s", s, OP_DESC(PL_op)); else Perl_warner(aTHX_ WARN_UTF8, "%s", s); } @@ -507,7 +549,7 @@ Perl_utf8_length(pTHX_ U8 *s, U8 *e) U8 t = UTF8SKIP(s); if (e - s < t) - Perl_croak(aTHX_ "panic: utf8_length: s=%p (%02X) e=%p l=%d - unaligned end",s,*s,e,t); + Perl_croak(aTHX_ "panic: utf8_length: unaligned end"); s += t; len++; } @@ -796,7 +838,7 @@ Perl_utf16_to_utf8_reversed(pTHX_ U8* p, U8* d, I32 bytelen, I32 *newlen) /* for now these are all defined (inefficiently) in terms of the utf8 versions */ bool -Perl_is_uni_alnum(pTHX_ U32 c) +Perl_is_uni_alnum(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -804,7 +846,7 @@ Perl_is_uni_alnum(pTHX_ U32 c) } bool -Perl_is_uni_alnumc(pTHX_ U32 c) +Perl_is_uni_alnumc(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -812,7 +854,7 @@ Perl_is_uni_alnumc(pTHX_ U32 c) } bool -Perl_is_uni_idfirst(pTHX_ U32 c) +Perl_is_uni_idfirst(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -820,7 +862,7 @@ Perl_is_uni_idfirst(pTHX_ U32 c) } bool -Perl_is_uni_alpha(pTHX_ U32 c) +Perl_is_uni_alpha(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -828,7 +870,7 @@ Perl_is_uni_alpha(pTHX_ U32 c) } bool -Perl_is_uni_ascii(pTHX_ U32 c) +Perl_is_uni_ascii(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -836,7 +878,7 @@ Perl_is_uni_ascii(pTHX_ U32 c) } bool -Perl_is_uni_space(pTHX_ U32 c) +Perl_is_uni_space(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -844,7 +886,7 @@ Perl_is_uni_space(pTHX_ U32 c) } bool -Perl_is_uni_digit(pTHX_ U32 c) +Perl_is_uni_digit(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -852,7 +894,7 @@ Perl_is_uni_digit(pTHX_ U32 c) } bool -Perl_is_uni_upper(pTHX_ U32 c) +Perl_is_uni_upper(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -860,7 +902,7 @@ Perl_is_uni_upper(pTHX_ U32 c) } bool -Perl_is_uni_lower(pTHX_ U32 c) +Perl_is_uni_lower(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -868,7 +910,7 @@ Perl_is_uni_lower(pTHX_ U32 c) } bool -Perl_is_uni_cntrl(pTHX_ U32 c) +Perl_is_uni_cntrl(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -876,7 +918,7 @@ Perl_is_uni_cntrl(pTHX_ U32 c) } bool -Perl_is_uni_graph(pTHX_ U32 c) +Perl_is_uni_graph(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -884,7 +926,7 @@ Perl_is_uni_graph(pTHX_ U32 c) } bool -Perl_is_uni_print(pTHX_ U32 c) +Perl_is_uni_print(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -892,7 +934,7 @@ Perl_is_uni_print(pTHX_ U32 c) } bool -Perl_is_uni_punct(pTHX_ U32 c) +Perl_is_uni_punct(pTHX_ UV c) { U8 tmpbuf[UTF8_MAXLEN+1]; uvchr_to_utf8(tmpbuf, (UV)c); @@ -900,119 +942,127 @@ Perl_is_uni_punct(pTHX_ U32 c) } bool -Perl_is_uni_xdigit(pTHX_ U32 c) +Perl_is_uni_xdigit(pTHX_ UV c) { - U8 tmpbuf[UTF8_MAXLEN+1]; + U8 tmpbuf[UTF8_MAXLEN_UCLC+1]; uvchr_to_utf8(tmpbuf, (UV)c); return is_utf8_xdigit(tmpbuf); } -U32 -Perl_to_uni_upper(pTHX_ U32 c) +UV +Perl_to_uni_upper(pTHX_ UV c, U8* p, STRLEN *lenp) { - U8 tmpbuf[UTF8_MAXLEN+1]; + U8 tmpbuf[UTF8_MAXLEN_UCLC+1]; uvchr_to_utf8(tmpbuf, (UV)c); - return to_utf8_upper(tmpbuf); + return to_utf8_upper(tmpbuf, p, lenp); } -U32 -Perl_to_uni_title(pTHX_ U32 c) +UV +Perl_to_uni_title(pTHX_ UV c, U8* p, STRLEN *lenp) { - U8 tmpbuf[UTF8_MAXLEN+1]; + U8 tmpbuf[UTF8_MAXLEN_UCLC+1]; uvchr_to_utf8(tmpbuf, (UV)c); - return to_utf8_title(tmpbuf); + return to_utf8_title(tmpbuf, p, lenp); } -U32 -Perl_to_uni_lower(pTHX_ U32 c) +UV +Perl_to_uni_lower(pTHX_ UV c, U8* p, STRLEN *lenp) { - U8 tmpbuf[UTF8_MAXLEN+1]; + U8 tmpbuf[UTF8_MAXLEN_UCLC+1]; + uvchr_to_utf8(tmpbuf, (UV)c); + return to_utf8_lower(tmpbuf, p, lenp); +} + +UV +Perl_to_uni_fold(pTHX_ UV c, U8* p, STRLEN *lenp) +{ + U8 tmpbuf[UTF8_MAXLEN_FOLD+1]; uvchr_to_utf8(tmpbuf, (UV)c); - return to_utf8_lower(tmpbuf); + return to_utf8_fold(tmpbuf, p, lenp); } /* for now these all assume no locale info available for Unicode > 255 */ bool -Perl_is_uni_alnum_lc(pTHX_ U32 c) +Perl_is_uni_alnum_lc(pTHX_ UV c) { return is_uni_alnum(c); /* XXX no locale support yet */ } bool -Perl_is_uni_alnumc_lc(pTHX_ U32 c) +Perl_is_uni_alnumc_lc(pTHX_ UV c) { return is_uni_alnumc(c); /* XXX no locale support yet */ } bool -Perl_is_uni_idfirst_lc(pTHX_ U32 c) +Perl_is_uni_idfirst_lc(pTHX_ UV c) { return is_uni_idfirst(c); /* XXX no locale support yet */ } bool -Perl_is_uni_alpha_lc(pTHX_ U32 c) +Perl_is_uni_alpha_lc(pTHX_ UV c) { return is_uni_alpha(c); /* XXX no locale support yet */ } bool -Perl_is_uni_ascii_lc(pTHX_ U32 c) +Perl_is_uni_ascii_lc(pTHX_ UV c) { return is_uni_ascii(c); /* XXX no locale support yet */ } bool -Perl_is_uni_space_lc(pTHX_ U32 c) +Perl_is_uni_space_lc(pTHX_ UV c) { return is_uni_space(c); /* XXX no locale support yet */ } bool -Perl_is_uni_digit_lc(pTHX_ U32 c) +Perl_is_uni_digit_lc(pTHX_ UV c) { return is_uni_digit(c); /* XXX no locale support yet */ } bool -Perl_is_uni_upper_lc(pTHX_ U32 c) +Perl_is_uni_upper_lc(pTHX_ UV c) { return is_uni_upper(c); /* XXX no locale support yet */ } bool -Perl_is_uni_lower_lc(pTHX_ U32 c) +Perl_is_uni_lower_lc(pTHX_ UV c) { return is_uni_lower(c); /* XXX no locale support yet */ } bool -Perl_is_uni_cntrl_lc(pTHX_ U32 c) +Perl_is_uni_cntrl_lc(pTHX_ UV c) { return is_uni_cntrl(c); /* XXX no locale support yet */ } bool -Perl_is_uni_graph_lc(pTHX_ U32 c) +Perl_is_uni_graph_lc(pTHX_ UV c) { return is_uni_graph(c); /* XXX no locale support yet */ } bool -Perl_is_uni_print_lc(pTHX_ U32 c) +Perl_is_uni_print_lc(pTHX_ UV c) { return is_uni_print(c); /* XXX no locale support yet */ } bool -Perl_is_uni_punct_lc(pTHX_ U32 c) +Perl_is_uni_punct_lc(pTHX_ UV c) { return is_uni_punct(c); /* XXX no locale support yet */ } bool -Perl_is_uni_xdigit_lc(pTHX_ U32 c) +Perl_is_uni_xdigit_lc(pTHX_ UV c) { return is_uni_xdigit(c); /* XXX no locale support yet */ } @@ -1020,19 +1070,31 @@ Perl_is_uni_xdigit_lc(pTHX_ U32 c) U32 Perl_to_uni_upper_lc(pTHX_ U32 c) { - return to_uni_upper(c); /* XXX no locale support yet */ + /* XXX returns only the first character -- do not use XXX */ + /* XXX no locale support yet */ + STRLEN len; + U8 tmpbuf[UTF8_MAXLEN_UCLC+1]; + return (U32)to_uni_upper(c, tmpbuf, &len); } U32 Perl_to_uni_title_lc(pTHX_ U32 c) { - return to_uni_title(c); /* XXX no locale support yet */ + /* XXX returns only the first character XXX -- do not use XXX */ + /* XXX no locale support yet */ + STRLEN len; + U8 tmpbuf[UTF8_MAXLEN_UCLC+1]; + return (U32)to_uni_title(c, tmpbuf, &len); } U32 Perl_to_uni_lower_lc(pTHX_ U32 c) { - return to_uni_lower(c); /* XXX no locale support yet */ + /* XXX returns only the first character -- do not use XXX */ + /* XXX no locale support yet */ + STRLEN len; + U8 tmpbuf[UTF8_MAXLEN_UCLC+1]; + return (U32)to_uni_lower(c, tmpbuf, &len); } bool @@ -1198,37 +1260,96 @@ Perl_is_utf8_mark(pTHX_ U8 *p) return swash_fetch(PL_utf8_mark, p, TRUE); } +/* +=for apidoc A|UV|to_utf8_case|U8 *p|U8* ustrp|STRLEN *lenp|SV **swash|char *normal|char *special + +The "p" contains the pointer to the UTF-8 string encoding +the character that is being converted. + +The "ustrp" is a pointer to the character buffer to put the +conversion result to. The "lenp" is a pointer to the length +of the result. + +The "swash" is a pointer to the swash to use. + +The "normal" is a string like "ToLower" which means the swash +$utf8::ToLower, which is stored in lib/unicore/To/Lower.pl, +and loaded by SWASHGET, using lib/utf8_heavy.pl. + +The "special" is a string like "utf8::ToSpecLower", which means +the hash %utf8::ToSpecLower, which is stored in the same file, +lib/unicore/To/Lower.pl, and also loaded by SWASHGET. The access +to the hash is by Perl_to_utf8_case(). + +=cut + */ + UV -Perl_to_utf8_upper(pTHX_ U8 *p) +Perl_to_utf8_case(pTHX_ U8 *p, U8* ustrp, STRLEN *lenp, SV **swashp,char *normal, char *special) { UV uv; - if (!PL_utf8_toupper) - PL_utf8_toupper = swash_init("utf8", "ToUpper", &PL_sv_undef, 4, 0); - uv = swash_fetch(PL_utf8_toupper, p, TRUE); - return uv ? UNI_TO_NATIVE(uv) : utf8_to_uvchr(p,0); + if (!*swashp) + *swashp = swash_init("utf8", normal, &PL_sv_undef, 4, 0); + uv = swash_fetch(*swashp, p, TRUE); + if (uv) + uv = UNI_TO_NATIVE(uv); + else { + HV *hv; + SV *keysv; + HE *he; + + uv = utf8_to_uvchr(p, 0); + + if ((hv = get_hv(special, FALSE)) && + (keysv = sv_2mortal(Perl_newSVpvf(aTHX_ "%04"UVXf, uv))) && + (he = hv_fetch_ent(hv, keysv, FALSE, 0))) { + SV *val = HeVAL(he); + char *s = SvPV(val, *lenp); + U8 c = *(U8*)s; + if (*lenp > 1 || UNI_IS_INVARIANT(c)) + Copy(s, ustrp, *lenp, U8); + else { + /* something in the 0x80..0xFF range */ + ustrp[0] = UTF8_EIGHT_BIT_HI(c); + ustrp[1] = UTF8_EIGHT_BIT_LO(c); + *lenp = 2; + } + return 0; + } + } + if (lenp) + *lenp = UNISKIP(uv); + uvuni_to_utf8(ustrp, uv); + return uv; } UV -Perl_to_utf8_title(pTHX_ U8 *p) +Perl_to_utf8_upper(pTHX_ U8 *p, U8* ustrp, STRLEN *lenp) { - UV uv; + return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, + &PL_utf8_toupper, "ToUpper", "utf8::ToSpecUpper"); +} - if (!PL_utf8_totitle) - PL_utf8_totitle = swash_init("utf8", "ToTitle", &PL_sv_undef, 4, 0); - uv = swash_fetch(PL_utf8_totitle, p, TRUE); - return uv ? UNI_TO_NATIVE(uv) : utf8_to_uvchr(p,0); +UV +Perl_to_utf8_title(pTHX_ U8 *p, U8* ustrp, STRLEN *lenp) +{ + return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, + &PL_utf8_totitle, "ToTitle", "utf8::ToSpecTitle"); } UV -Perl_to_utf8_lower(pTHX_ U8 *p) +Perl_to_utf8_lower(pTHX_ U8 *p, U8* ustrp, STRLEN *lenp) { - UV uv; + return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, + &PL_utf8_tolower, "ToLower", "utf8::ToSpecLower"); +} - if (!PL_utf8_tolower) - PL_utf8_tolower = swash_init("utf8", "ToLower", &PL_sv_undef, 4, 0); - uv = swash_fetch(PL_utf8_tolower, p, TRUE); - return uv ? UNI_TO_NATIVE(uv) : utf8_to_uvchr(p,0); +UV +Perl_to_utf8_fold(pTHX_ U8 *p, U8* ustrp, STRLEN *lenp) +{ + return Perl_to_utf8_case(aTHX_ p, ustrp, lenp, + &PL_utf8_tofold, "ToFold", "utf8::ToSpecFold"); } /* a "swash" is a swatch hash */ @@ -1240,10 +1361,15 @@ Perl_swash_init(pTHX_ char* pkg, char* name, SV *listsv, I32 minbits, I32 none) SV* tokenbufsv = sv_2mortal(NEWSV(0,0)); dSP; HV *stash = gv_stashpvn(pkg, strlen(pkg), FALSE); + SV* errsv_save; if (!gv_fetchmeth(stash, "SWASHNEW", 8, -1)) { /* demand load utf8 */ ENTER; + errsv_save = newSVsv(ERRSV); Perl_load_module(aTHX_ PERL_LOADMOD_NOIMPORT, newSVpv(pkg,0), Nullsv); + if (!SvTRUE(ERRSV)) + sv_setsv(ERRSV, errsv_save); + SvREFCNT_dec(errsv_save); LEAVE; } SPAGAIN; @@ -1263,10 +1389,14 @@ Perl_swash_init(pTHX_ char* pkg, char* name, SV *listsv, I32 minbits, I32 none) if (PL_curcop == &PL_compiling) /* XXX ought to be handled by lex_start */ sv_setpv(tokenbufsv, PL_tokenbuf); + errsv_save = newSVsv(ERRSV); if (call_method("SWASHNEW", G_SCALAR)) retval = newSVsv(*PL_stack_sp--); else retval = &PL_sv_undef; + if (!SvTRUE(ERRSV)) + sv_setsv(ERRSV, errsv_save); + SvREFCNT_dec(errsv_save); LEAVE; POPSTACK; if (PL_curcop == &PL_compiling) { @@ -1281,6 +1411,12 @@ Perl_swash_init(pTHX_ char* pkg, char* name, SV *listsv, I32 minbits, I32 none) return retval; } + +/* This API is wrong for special case conversions since we may need to + * return several Unicode characters for a single Unicode character + * (see lib/unicore/SpecCase.txt) The SWASHGET in lib/utf8_heavy.pl is + * the lower-level routine, and it is similarly broken for returning + * multiple values. --jhi */ UV Perl_swash_fetch(pTHX_ SV *sv, U8 *ptr, bool do_utf8) { @@ -1289,7 +1425,7 @@ Perl_swash_fetch(pTHX_ SV *sv, U8 *ptr, bool do_utf8) U32 off; STRLEN slen; STRLEN needents; - U8 *tmps; + U8 *tmps = NULL; U32 bit; SV *retval; U8 tmputf8[2]; @@ -1350,6 +1486,7 @@ Perl_swash_fetch(pTHX_ SV *sv, U8 *ptr, bool do_utf8) Unicode tables, not a native character number. */ UV code_point = utf8n_to_uvuni(ptr, UTF8_MAXLEN, NULL, 0); + SV *errsv_save; ENTER; SAVETMPS; save_re_context(); @@ -1362,10 +1499,14 @@ Perl_swash_fetch(pTHX_ SV *sv, U8 *ptr, bool do_utf8) (code_point & ~(needents - 1)) : 0))); PUSHs(sv_2mortal(newSViv(needents))); PUTBACK; + errsv_save = newSVsv(ERRSV); if (call_method("SWASHGET", G_SCALAR)) retval = newSVsv(*PL_stack_sp--); else retval = &PL_sv_undef; + if (!SvTRUE(ERRSV)) + sv_setsv(ERRSV, errsv_save); + SvREFCNT_dec(errsv_save); POPSTACK; FREETMPS; LEAVE; @@ -1429,9 +1570,14 @@ is the recommended wide native character-aware way of saying U8 * Perl_uvchr_to_utf8(pTHX_ U8 *d, UV uv) { - return Perl_uvuni_to_utf8(aTHX_ d, NATIVE_TO_UNI(uv)); + return Perl_uvuni_to_utf8_flags(aTHX_ d, NATIVE_TO_UNI(uv), 0); } +U8 * +Perl_uvchr_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) +{ + return Perl_uvuni_to_utf8_flags(aTHX_ d, NATIVE_TO_UNI(uv), flags); +} /* =for apidoc A|UV|utf8n_to_uvchr|U8 *s|STRLEN curlen|STRLEN *retlen|U32 flags @@ -1455,4 +1601,115 @@ Perl_utf8n_to_uvchr(pTHX_ U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) return UNI_TO_NATIVE(uv); } +/* +=for apidoc A|char *|pv_uni_display|SV *dsv|U8 *spv|STRLEN len|STRLEN pvlim|UV flags + +Build to the scalar dsv a displayable version of the string spv, +length len, the displayable version being at most pvlim bytes long +(if longer, the rest is truncated and "..." will be appended). +The flags argument is currently unused but available for future extensions. +The pointer to the PV of the dsv is returned. + +=cut */ +char * +Perl_pv_uni_display(pTHX_ SV *dsv, U8 *spv, STRLEN len, STRLEN pvlim, UV flags) +{ + int truncated = 0; + char *s, *e; + + sv_setpvn(dsv, "", 0); + for (s = (char *)spv, e = s + len; s < e; s += UTF8SKIP(s)) { + UV u; + if (pvlim && SvCUR(dsv) >= pvlim) { + truncated++; + break; + } + u = utf8_to_uvchr((U8*)s, 0); + Perl_sv_catpvf(aTHX_ dsv, "\\x{%"UVxf"}", u); + } + if (truncated) + sv_catpvn(dsv, "...", 3); + + return SvPVX(dsv); +} + +/* +=for apidoc A|char *|sv_uni_display|SV *dsv|SV *ssv|STRLEN pvlim|UV flags + +Build to the scalar dsv a displayable version of the scalar sv, +he displayable version being at most pvlim bytes long +(if longer, the rest is truncated and "..." will be appended). +The flags argument is currently unused but available for future extensions. +The pointer to the PV of the dsv is returned. + +=cut */ +char * +Perl_sv_uni_display(pTHX_ SV *dsv, SV *ssv, STRLEN pvlim, UV flags) +{ + return Perl_pv_uni_display(aTHX_ dsv, (U8*)SvPVX(ssv), SvCUR(ssv), + pvlim, flags); +} + +/* +=for apidoc A|I32|ibcmp_utf8|const char *s1|bool u1|const char *s2|bool u2|register I32 len + +Return true if the strings s1 and s2 differ case-insensitively, false +if not (if they are equal case-insensitively). If u1 is true, the +string s1 is assumed to be in UTF-8-encoded Unicode. If u2 is true, +the string s2 is assumed to be in UTF-8-encoded Unicode. + +For case-insensitiveness, the "casefolding" of Unicode is used +instead of upper/lowercasing both the characters, see +http://www.unicode.org/unicode/reports/tr21/ (Case Mappings). + +=cut */ +I32 +Perl_ibcmp_utf8(pTHX_ const char *s1, bool u1, register I32 len1, const char *s2, bool u2, register I32 len2) +{ + register U8 *a = (U8*)s1; + register U8 *b = (U8*)s2; + register U8 *ae = b + len1; + register U8 *be = b + len2; + STRLEN la, lb; + UV ca, cb; + STRLEN ulen1, ulen2; + U8 tmpbuf1[UTF8_MAXLEN_FOLD+1]; + U8 tmpbuf2[UTF8_MAXLEN_FOLD+1]; + + while (a < ae && b < be) { + if (u1) { + if (a + UTF8SKIP(a) > ae) + break; + ca = utf8_to_uvchr((U8*)a, &la); + } else { + ca = *a; + la = 1; + } + if (u2) { + if (b + UTF8SKIP(b) > be) + break; + cb = utf8_to_uvchr((U8*)b, &lb); + } else { + cb = *b; + lb = 1; + } + if (ca != cb) { + if (u1) + to_uni_fold(NATIVE_TO_UNI(ca), tmpbuf1, &ulen1); + else + ulen1 = 1; + if (u2) + to_uni_fold(NATIVE_TO_UNI(cb), tmpbuf2, &ulen2); + else + ulen2 = 1; + if (ulen1 != ulen2 + || (ulen1 == 1 && PL_fold[ca] != PL_fold[cb]) + || memNE((char *)tmpbuf1, (char *)tmpbuf2, ulen1)) + return 1; /* mismatch */ + } + a += la; + b += lb; + } + return a == ae && b == be ? 0 : 1; /* 0 match, 1 mismatch */ +}