X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=blobdiff_plain;f=utf8.c;h=debfb9ceac8b95c4a1d40d0151c5db5ef3b614dc;hb=b6512f489e761186d508cf0b8b7705805cfefc52;hp=517b2e37ad2b5a6f70c493bb0250cb7c57a4feec;hpb=6bd23935f1e5b7c6f7035440f4b48e4c9659d61f;p=p5sagit%2Fp5-mst-13.2.git diff --git a/utf8.c b/utf8.c index 517b2e3..debfb9c 100644 --- a/utf8.c +++ b/utf8.c @@ -27,15 +27,23 @@ /* Unicode support */ /* -=for apidoc A|U8 *|uvuni_to_utf8|U8 *d|UV uv +=for apidoc A|U8 *|uvuni_to_utf8_flags|U8 *d|UV uv|UV flags Adds the UTF8 representation of the Unicode codepoint C to the end of the string C; C should be have at least C free bytes available. The return value is the pointer to the byte after the end of the new character. In other words, + d = uvuni_to_utf8_flags(d, uv, flags); + +or, in most cases, + d = uvuni_to_utf8(d, uv); +(which is equivalent to) + + d = uvuni_to_utf8_flags(d, uv, 0); + is the recommended Unicode-aware way of saying *(d++) = uv; @@ -44,14 +52,33 @@ is the recommended Unicode-aware way of saying */ U8 * -Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv) -{ +Perl_uvuni_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) +{ + if (ckWARN_d(WARN_UTF8)) { + if (UNICODE_IS_SURROGATE(uv) && + !(flags & UNICODE_ALLOW_SURROGATE)) + Perl_warner(aTHX_ WARN_UTF8, "UTF-16 surrogate 0x%04"UVxf, uv); + else if ( + ((uv >= 0xFDD0 && uv <= 0xFDEF && + !(flags & UNICODE_ALLOW_FDD0)) + || + ((uv & 0xFFFF) == 0xFFFE && + !(flags & UNICODE_ALLOW_FFFE)) + || + ((uv & 0xFFFF) == 0xFFFF && + !(flags & UNICODE_ALLOW_FFFF))) && + /* UNICODE_ALLOW_SUPER includes + * FFFEs and FFFFs beyond 0x10FFFF. */ + ((uv <= PERL_UNICODE_MAX) || + !(flags & UNICODE_ALLOW_SUPER)) + ) + Perl_warner(aTHX_ WARN_UTF8, + "Unicode character 0x%04"UVxf" is illegal", uv); + } if (UNI_IS_INVARIANT(uv)) { *d++ = UTF_TO_NATIVE(uv); return d; } - if (UNICODE_IS_SURROGATE(uv)) - Perl_croak(aTHX_ "UTF-16 surrogate 0x%04"UVxf, uv); #if defined(EBCDIC) else { STRLEN len = UNISKIP(uv); @@ -132,7 +159,12 @@ Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv) #endif #endif /* Loop style */ } - + +U8 * +Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv) +{ + return Perl_uvuni_to_utf8_flags(aTHX_ d, uv, 0); +} /* @@ -245,9 +277,11 @@ Most code should use utf8_to_uvchr() rather than call this directly. UV Perl_utf8n_to_uvuni(pTHX_ U8 *s, STRLEN curlen, STRLEN *retlen, U32 flags) { + U8 *s0 = s; UV uv = *s, ouv = 0; STRLEN len = 1; bool dowarn = ckWARN_d(WARN_UTF8); + UV startbyte = *s; STRLEN expectlen = 0; U32 warning = 0; @@ -390,23 +424,28 @@ malformed: Perl_sv_catpvf(aTHX_ sv, "(empty string)"); break; case UTF8_WARN_CONTINUATION: - Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf")", uv); + Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf", with no preceding start byte)", uv); break; case UTF8_WARN_NON_CONTINUATION: - Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf" after start byte 0x%02"UVxf")", - (UV)s[1], uv); + if (s == s0) + Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", immediately after start byte 0x%02"UVxf")", + (UV)s[1], startbyte); + else + Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", %d byte%s after start byte 0x%02"UVxf", expected %d bytes)", + (UV)s[1], s - s0, s - s0 > 1 ? "s" : "", startbyte, expectlen); + break; case UTF8_WARN_FE_FF: Perl_sv_catpvf(aTHX_ sv, "(byte 0x%02"UVxf")", uv); break; case UTF8_WARN_SHORT: - Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d)", - curlen, curlen == 1 ? "" : "s", expectlen); + Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", + curlen, curlen == 1 ? "" : "s", expectlen, startbyte); expectlen = curlen; /* distance for caller to skip */ break; case UTF8_WARN_OVERFLOW: - Perl_sv_catpvf(aTHX_ sv, "(overflow at 0x%"UVxf", byte 0x%02x)", - ouv, *s); + Perl_sv_catpvf(aTHX_ sv, "(overflow at 0x%"UVxf", byte 0x%02x, after start byte 0x%02"UVxf")", + ouv, *s, startbyte); break; case UTF8_WARN_SURROGATE: Perl_sv_catpvf(aTHX_ sv, "(UTF-16 surrogate 0x%04"UVxf")", uv); @@ -415,8 +454,8 @@ malformed: Perl_sv_catpvf(aTHX_ sv, "(byte order mark 0x%04"UVxf")", uv); break; case UTF8_WARN_LONG: - Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d)", - expectlen, expectlen == 1 ? "": "s", UNISKIP(uv)); + Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")", + expectlen, expectlen == 1 ? "": "s", UNISKIP(uv), startbyte); break; case UTF8_WARN_FFFF: Perl_sv_catpvf(aTHX_ sv, "(character 0x%04"UVxf")", uv); @@ -1028,6 +1067,36 @@ Perl_is_uni_xdigit_lc(pTHX_ UV c) return is_uni_xdigit(c); /* XXX no locale support yet */ } +U32 +Perl_to_uni_upper_lc(pTHX_ U32 c) +{ + /* XXX returns only the first character -- do not use XXX */ + /* XXX no locale support yet */ + STRLEN len; + U8 tmpbuf[UTF8_MAXLEN_UCLC+1]; + return (U32)to_uni_upper(c, tmpbuf, &len); +} + +U32 +Perl_to_uni_title_lc(pTHX_ U32 c) +{ + /* XXX returns only the first character XXX -- do not use XXX */ + /* XXX no locale support yet */ + STRLEN len; + U8 tmpbuf[UTF8_MAXLEN_UCLC+1]; + return (U32)to_uni_title(c, tmpbuf, &len); +} + +U32 +Perl_to_uni_lower_lc(pTHX_ U32 c) +{ + /* XXX returns only the first character -- do not use XXX */ + /* XXX no locale support yet */ + STRLEN len; + U8 tmpbuf[UTF8_MAXLEN_UCLC+1]; + return (U32)to_uni_lower(c, tmpbuf, &len); +} + bool Perl_is_utf8_alnum(pTHX_ U8 *p) { @@ -1501,9 +1570,14 @@ is the recommended wide native character-aware way of saying U8 * Perl_uvchr_to_utf8(pTHX_ U8 *d, UV uv) { - return Perl_uvuni_to_utf8(aTHX_ d, NATIVE_TO_UNI(uv)); + return Perl_uvuni_to_utf8_flags(aTHX_ d, NATIVE_TO_UNI(uv), 0); } +U8 * +Perl_uvchr_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) +{ + return Perl_uvuni_to_utf8_flags(aTHX_ d, NATIVE_TO_UNI(uv), flags); +} /* =for apidoc A|UV|utf8n_to_uvchr|U8 *s|STRLEN curlen|STRLEN *retlen|U32 flags