/* Unicode support */
+/*
+=for apidoc A|U8*|uv_to_utf8|U8 *d|UV uv
+
+Adds the UTF8 representation of the Unicode codepoint C<uv> to the end
+of the string C<d>; C<d> should be have at least C<UTF8_MAXLEN+1> free
+bytes available. The return value is the pointer to the byte after the
+end of the new character. In other words,
+
+ d = uv_to_utf8(d, uv);
+
+is the recommended Unicode-aware way of saying
+
+ *(d++) = uv;
+
+=cut
+*/
+
U8 *
-Perl_uv_to_utf8(pTHX_ U8 *d, UV uv) /* the d must be UTF8_MAXLEN+1 deep */
+Perl_uv_to_utf8(pTHX_ U8 *d, UV uv)
{
if (uv < 0x80) {
*d++ = uv;
- *d = 0;
return d;
}
if (uv < 0x800) {
*d++ = (( uv >> 6) | 0xc0);
*d++ = (( uv & 0x3f) | 0x80);
- *d = 0;
return d;
}
if (uv < 0x10000) {
*d++ = (( uv >> 12) | 0xe0);
*d++ = (((uv >> 6) & 0x3f) | 0x80);
*d++ = (( uv & 0x3f) | 0x80);
- *d = 0;
return d;
}
if (uv < 0x200000) {
*d++ = (((uv >> 12) & 0x3f) | 0x80);
*d++ = (((uv >> 6) & 0x3f) | 0x80);
*d++ = (( uv & 0x3f) | 0x80);
- *d = 0;
return d;
}
if (uv < 0x4000000) {
*d++ = (((uv >> 12) & 0x3f) | 0x80);
*d++ = (((uv >> 6) & 0x3f) | 0x80);
*d++ = (( uv & 0x3f) | 0x80);
- *d = 0;
return d;
}
if (uv < 0x80000000) {
*d++ = (((uv >> 12) & 0x3f) | 0x80);
*d++ = (((uv >> 6) & 0x3f) | 0x80);
*d++ = (( uv & 0x3f) | 0x80);
- *d = 0;
return d;
}
#ifdef HAS_QUAD
*d++ = (((uv >> 12) & 0x3f) | 0x80);
*d++ = (((uv >> 6) & 0x3f) | 0x80);
*d++ = (( uv & 0x3f) | 0x80);
- *d = 0;
return d;
}
#ifdef HAS_QUAD
*d++ = (((uv >> 12) & 0x3f) | 0x80);
*d++ = (((uv >> 6) & 0x3f) | 0x80);
*d++ = (( uv & 0x3f) | 0x80);
- *d = 0;
return d;
}
#endif
}
-/* Tests if some arbitrary number of bytes begins in a valid UTF-8 character.
- * The actual number of bytes in the UTF-8 character will be returned if it
- * is valid, otherwise 0. */
+/*
+=for apidoc A|STRLEN|is_utf8_char|U8 *s
+
+Tests if some arbitrary number of bytes begins in a valid UTF-8
+character. Note that an ASCII character is a valid UTF-8 character.
+The actual number of bytes in the UTF-8 character will be returned if
+it is valid, otherwise 0.
+
+=cut */
STRLEN
Perl_is_utf8_char(pTHX_ U8 *s)
{
}
/*
-=for apidoc Am|is_utf8_string|U8 *s|STRLEN len
+=for apidoc A|bool|is_utf8_string|U8 *s|STRLEN len
-Returns true if first C<len> bytes of the given string form valid a UTF8
-string, false otherwise.
+Returns true if first C<len> bytes of the given string form a valid UTF8
+string, false otherwise. Note that 'a valid UTF8 string' does not mean
+'a string that contains UTF8' because a valid ASCII string is a valid
+UTF8 string.
=cut
*/
}
/*
-=for apidoc Am|U8* s|utf8_to_uv|STRLEN curlen|STRLEN *retlen|U32 flags
+=for apidoc A|U8* s|utf8_to_uv|STRLEN curlen|STRLEN *retlen|U32 flags
Returns the character value of the first character in the string C<s>
which is assumed to be in UTF8 encoding and no longer than C<curlen>;
bool dowarn = ckWARN_d(WARN_UTF8);
#endif
STRLEN expectlen = 0;
-
- if (curlen == 0) {
- if (dowarn)
- Perl_warner(aTHX_ WARN_UTF8,
- "Malformed UTF-8 character (an empty string)");
+ U32 warning = 0;
+
+/* This list is a superset of the UTF8_ALLOW_XXX. */
+
+#define UTF8_WARN_EMPTY 1
+#define UTF8_WARN_CONTINUATION 2
+#define UTF8_WARN_NON_CONTINUATION 3
+#define UTF8_WARN_FE_FF 4
+#define UTF8_WARN_SHORT 5
+#define UTF8_WARN_OVERFLOW 6
+#define UTF8_WARN_SURROGATE 7
+#define UTF8_WARN_BOM 8
+#define UTF8_WARN_LONG 9
+#define UTF8_WARN_FFFF 10
+
+ if (curlen == 0 &&
+ !(flags & UTF8_ALLOW_EMPTY)) {
+ warning = UTF8_WARN_EMPTY;
goto malformed;
}
if (UTF8_IS_CONTINUATION(uv) &&
!(flags & UTF8_ALLOW_CONTINUATION)) {
- if (dowarn)
- Perl_warner(aTHX_ WARN_UTF8,
- "Malformed UTF-8 character (unexpected continuation byte 0x%02"UVxf")",
- uv);
+ warning = UTF8_WARN_CONTINUATION;
goto malformed;
}
if (UTF8_IS_START(uv) && curlen > 1 && !UTF8_IS_CONTINUATION(s[1]) &&
!(flags & UTF8_ALLOW_NON_CONTINUATION)) {
- if (dowarn)
- Perl_warner(aTHX_ WARN_UTF8,
- "Malformed UTF-8 character (unexpected non-continuation byte 0x%02"UVxf" after start byte 0x%02"UVxf")",
- (UV)s[1], uv);
+ warning = UTF8_WARN_NON_CONTINUATION;
goto malformed;
}
if ((uv == 0xfe || uv == 0xff) &&
!(flags & UTF8_ALLOW_FE_FF)) {
- if (dowarn)
- Perl_warner(aTHX_ WARN_UTF8,
- "Malformed UTF-8 character (byte 0x%02"UVxf")",
- uv);
+ warning = UTF8_WARN_FE_FF;
goto malformed;
}
if ((curlen < expectlen) &&
!(flags & UTF8_ALLOW_SHORT)) {
- if (dowarn)
- Perl_warner(aTHX_ WARN_UTF8,
- "Malformed UTF-8 character (%d byte%s, need %d)",
- curlen, curlen == 1 ? "" : "s", expectlen);
+ warning = UTF8_WARN_SHORT;
goto malformed;
}
while (len--) {
if (!UTF8_IS_CONTINUATION(*s) &&
!(flags & UTF8_ALLOW_NON_CONTINUATION)) {
- if (dowarn)
- Perl_warner(aTHX_ WARN_UTF8,
- "Malformed UTF-8 character (unexpected non-continuation byte 0x%02x)",
- *s);
+ s--;
+ warning = UTF8_WARN_NON_CONTINUATION;
goto malformed;
}
else
uv = UTF8_ACCUMULATE(uv, *s);
- if (uv < ouv) {
- /* This cannot be allowed. */
- if (dowarn)
- Perl_warner(aTHX_ WARN_UTF8,
- "Malformed UTF-8 character (overflow at 0x%"UVxf", byte 0x%02x)",
- ouv, *s);
- goto malformed;
+ if (!(uv > ouv)) {
+ /* These cannot be allowed. */
+ if (uv == ouv) {
+ if (!(flags & UTF8_ALLOW_LONG)) {
+ warning = UTF8_WARN_LONG;
+ goto malformed;
+ }
+ }
+ else { /* uv < ouv */
+ /* This cannot be allowed. */
+ warning = UTF8_WARN_OVERFLOW;
+ goto malformed;
+ }
}
s++;
ouv = uv;
if (UNICODE_IS_SURROGATE(uv) &&
!(flags & UTF8_ALLOW_SURROGATE)) {
- if (dowarn)
- Perl_warner(aTHX_ WARN_UTF8,
- "Malformed UTF-8 character (UTF-16 surrogate 0x%04"UVxf")",
- uv);
+ warning = UTF8_WARN_SURROGATE;
goto malformed;
} else if (UNICODE_IS_BYTE_ORDER_MARK(uv) &&
!(flags & UTF8_ALLOW_BOM)) {
- if (dowarn)
- Perl_warner(aTHX_ WARN_UTF8,
- "Malformed UTF-8 character (byte order mark 0x%04"UVxf")",
- uv);
+ warning = UTF8_WARN_BOM;
goto malformed;
} else if ((expectlen > UNISKIP(uv)) &&
!(flags & UTF8_ALLOW_LONG)) {
- if (dowarn)
- Perl_warner(aTHX_ WARN_UTF8,
- "Malformed UTF-8 character (%d byte%s, need %d)",
- expectlen, expectlen == 1 ? "": "s", UNISKIP(uv));
+ warning = UTF8_WARN_LONG;
goto malformed;
} else if (UNICODE_IS_ILLEGAL(uv) &&
!(flags & UTF8_ALLOW_FFFF)) {
- if (dowarn)
- Perl_warner(aTHX_ WARN_UTF8,
- "Malformed UTF-8 character (character 0x%04"UVxf")",
- uv);
+ warning = UTF8_WARN_FFFF;
goto malformed;
}
return 0;
}
+ if (dowarn) {
+ SV* sv = sv_2mortal(newSVpv("Malformed UTF-8 character ", 0));
+
+ switch (warning) {
+ case 0: /* Intentionally empty. */ break;
+ case UTF8_WARN_EMPTY:
+ Perl_sv_catpvf(aTHX_ sv, "(empty string)");
+ break;
+ case UTF8_WARN_CONTINUATION:
+ Perl_sv_catpvf(aTHX_ sv, "(unexpected continuation byte 0x%02"UVxf")", uv);
+ break;
+ case UTF8_WARN_NON_CONTINUATION:
+ Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf" after start byte 0x%02"UVxf")",
+ (UV)s[1], uv);
+ break;
+ case UTF8_WARN_FE_FF:
+ Perl_sv_catpvf(aTHX_ sv, "(byte 0x%02"UVxf")", uv);
+ break;
+ case UTF8_WARN_SHORT:
+ Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d)",
+ curlen, curlen == 1 ? "" : "s", expectlen);
+ break;
+ case UTF8_WARN_OVERFLOW:
+ Perl_sv_catpvf(aTHX_ sv, "(overflow at 0x%"UVxf", byte 0x%02x)",
+ ouv, *s);
+ break;
+ case UTF8_WARN_SURROGATE:
+ Perl_sv_catpvf(aTHX_ sv, "(UTF-16 surrogate 0x%04"UVxf")", uv);
+ break;
+ case UTF8_WARN_BOM:
+ Perl_sv_catpvf(aTHX_ sv, "(byte order mark 0x%04"UVxf")", uv);
+ break;
+ case UTF8_WARN_LONG:
+ Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d)",
+ expectlen, expectlen == 1 ? "": "s", UNISKIP(uv));
+ break;
+ case UTF8_WARN_FFFF:
+ Perl_sv_catpvf(aTHX_ sv, "(character 0x%04"UVxf")", uv);
+ break;
+ default:
+ Perl_sv_catpvf(aTHX_ sv, "(unknown reason)");
+ break;
+ }
+
+ if (warning) {
+ char *s = SvPVX(sv);
+
+ if (PL_op)
+ Perl_warner(aTHX_ WARN_UTF8,
+ "%s in %s", s, PL_op_desc[PL_op->op_type]);
+ else
+ Perl_warner(aTHX_ WARN_UTF8, "%s", s);
+ }
+ }
+
if (retlen)
*retlen = expectlen ? expectlen : len;
}
/*
-=for apidoc Am|U8* s|utf8_to_uv_simple|STRLEN *retlen
+=for apidoc A|U8* s|utf8_to_uv_simple|STRLEN *retlen
Returns the character value of the first character in the string C<s>
which is assumed to be in UTF8 encoding; C<retlen> will be set to the
}
/*
-=for apidoc Am|STRLEN|utf8_length|U8* s|U8 *e
+=for apidoc A|STRLEN|utf8_length|U8* s|U8 *e
Return the length of the UTF-8 char encoded string C<s> in characters.
Stops at C<e> (inclusive). If C<e E<lt> s> or if the scan would end
}
/*
-=for apidoc Am|IV|utf8_distance|U8 *a|U8 *b
+=for apidoc A|IV|utf8_distance|U8 *a|U8 *b
Returns the number of UTF8 characters between the UTF-8 pointers C<a>
and C<b>.
}
/*
-=for apidoc Am|U8*|utf8_hop|U8 *s|I32 off
+=for apidoc A|U8*|utf8_hop|U8 *s|I32 off
Return the UTF-8 pointer C<s> displaced by C<off> characters, either
forward or backward.
}
/*
-=for apidoc Am|U8 *|utf8_to_bytes|U8 *s|STRLEN *len
+=for apidoc A|U8 *|utf8_to_bytes|U8 *s|STRLEN *len
Converts a string C<s> of length C<len> from UTF8 into byte encoding.
Unlike C<bytes_to_utf8>, this over-writes the original string, and
}
/*
-=for apidoc Am|U8 *|bytes_to_utf8|U8 *s|STRLEN *len
+=for apidoc A|U8 *|bytes_from_utf8|U8 *s|STRLEN *len|bool *is_utf8
+
+Converts a string C<s> of length C<len> from UTF8 into byte encoding.
+Unlike <utf8_to_bytes> but like C<bytes_to_utf8>, returns a pointer to
+the newly-created string, and updates C<len> to contain the new
+length. Returns the original string if no conversion occurs, C<len>
+is unchanged. Do nothing if C<is_utf8> points to 0. Sets C<is_utf8> to
+0 if C<s> is converted or contains all 7bit characters.
+
+=cut */
+
+U8 *
+Perl_bytes_from_utf8(pTHX_ U8* s, STRLEN *len, bool *is_utf8)
+{
+ U8 *send;
+ U8 *d;
+ U8 *start = s;
+ I32 count = 0;
+
+ if (!*is_utf8)
+ return start;
+
+ /* ensure valid UTF8 and chars < 256 before converting string */
+ for (send = s + *len; s < send;) {
+ U8 c = *s++;
+ if (!UTF8_IS_ASCII(c)) {
+ if (UTF8_IS_CONTINUATION(c) || s >= send ||
+ !UTF8_IS_CONTINUATION(*s) || UTF8_IS_DOWNGRADEABLE_START(c))
+ return start;
+ s++, count++;
+ }
+ }
+
+ *is_utf8 = 0;
+
+ if (!count)
+ return start;
+
+ Newz(801, d, (*len) - count + 1, U8);
+ s = start; start = d;
+ while (s < send) {
+ U8 c = *s++;
+
+ if (UTF8_IS_ASCII(c))
+ *d++ = c;
+ else
+ *d++ = UTF8_ACCUMULATE(c, *s++);
+ }
+ *d = '\0';
+ *len = d - start;
+ return start;
+}
+
+/*
+=for apidoc A|U8 *|bytes_to_utf8|U8 *s|STRLEN *len
Converts a string C<s> of length C<len> from ASCII into UTF8 encoding.
Returns a pointer to the newly-created string, and sets C<len> to
dst = d;
while (s < send) {
- if (*s < 0x80)
+ if (UTF8_IS_ASCII(*s))
*d++ = *s++;
else {
UV uv = *s++;
- *d++ = (( uv >> 6) | 0xc0);
- *d++ = (( uv & 0x3f) | 0x80);
+
+ *d++ = UTF8_EIGHT_BIT_HI(uv);
+ *d++ = UTF8_EIGHT_BIT_LO(uv);
}
}
*d = '\0';
Perl_swash_init(pTHX_ char* pkg, char* name, SV *listsv, I32 minbits, I32 none)
{
SV* retval;
- char tmpbuf[256];
+ SV* tokenbufsv = sv_2mortal(NEWSV(0,0));
dSP;
if (!gv_stashpv(pkg, 0)) { /* demand load utf8 */
SAVEI32(PL_hints);
PL_hints = 0;
save_re_context();
- if (PL_curcop == &PL_compiling) /* XXX ought to be handled by lex_start */
- strncpy(tmpbuf, PL_tokenbuf, sizeof tmpbuf);
+ if (PL_curcop == &PL_compiling)
+ /* XXX ought to be handled by lex_start */
+ sv_setpv(tokenbufsv, PL_tokenbuf);
if (call_method("SWASHNEW", G_SCALAR))
retval = newSVsv(*PL_stack_sp--);
else
LEAVE;
POPSTACK;
if (PL_curcop == &PL_compiling) {
- strncpy(PL_tokenbuf, tmpbuf, sizeof tmpbuf);
+ STRLEN len;
+ char* pv = SvPV(tokenbufsv, len);
+
+ Copy(pv, PL_tokenbuf, len+1, char);
PL_curcop->op_private = PL_hints;
}
if (!SvROK(retval) || SvTYPE(SvRV(retval)) != SVt_PVHV)
projects / p5sagit/p5-mst-13.2.git / blobdiff