#define PERL_IN_UTF8_C
#include "perl.h"
-static char unees[] = "Malformed UTF-8 character (unexpected end of string)";
+static const char unees[] =
+ "Malformed UTF-8 character (unexpected end of string)";
/*
=head1 Unicode Support
return Perl_uvuni_to_utf8_flags(aTHX_ d, uv, 0);
}
-
/*
-=for apidoc A|STRLEN|is_utf8_char|const U8 *s
Tests if some arbitrary number of bytes begins in a valid UTF-8
character. Note that an INVARIANT (i.e. ASCII) character is a valid
UTF-8 character. The actual number of bytes in the UTF-8 character
will be returned if it is valid, otherwise 0.
+This is the "slow" version as opposed to the "fast" version which is
+the "unrolled" IS_UTF8_CHAR(). E.g. for t/uni/class.t the speed
+difference is a factor of 2 to 3. For lengths (UTF8SKIP(s)) of four
+or less you should use the IS_UTF8_CHAR(), for lengths of five or more
+you should use the _slow(). In practice this means that the _slow()
+will be used very rarely, since the maximum Unicode code point (as of
+Unicode 4.1) is U+10FFFF, which encodes in UTF-8 to four bytes. Only
+the "Perl extended UTF-8" (the infamous 'v-strings') will encode into
+five bytes or more.
+
=cut */
-STRLEN
-Perl_is_utf8_char(pTHX_ const U8 *s)
+STATIC STRLEN
+S_is_utf8_char_slow(pTHX_ const U8 *s, const STRLEN len)
{
U8 u = *s;
- STRLEN slen, len;
+ STRLEN slen;
UV uv, ouv;
if (UTF8_IS_INVARIANT(u))
if (!UTF8_IS_START(u))
return 0;
- len = UTF8SKIP(s);
-
if (len < 2 || !UTF8_IS_CONTINUATION(s[1]))
return 0;
}
/*
+=for apidoc A|STRLEN|is_utf8_char|const U8 *s
+
+Tests if some arbitrary number of bytes begins in a valid UTF-8
+character. Note that an INVARIANT (i.e. ASCII) character is a valid
+UTF-8 character. The actual number of bytes in the UTF-8 character
+will be returned if it is valid, otherwise 0.
+
+=cut */
+STRLEN
+Perl_is_utf8_char(pTHX_ const U8 *s)
+{
+ STRLEN len = UTF8SKIP(s);
+#ifdef IS_UTF8_CHAR
+ if (IS_UTF8_CHAR_FAST(len))
+ return IS_UTF8_CHAR(s, len) ? len : 0;
+#endif /* #ifdef IS_UTF8_CHAR */
+ return is_utf8_char_slow(s, len);
+}
+
+/*
=for apidoc A|bool|is_utf8_string|const U8 *s|STRLEN len
Returns true if first C<len> bytes of the given string form a valid
not mean 'a string that contains code points above 0x7F encoded in UTF-8'
because a valid ASCII string is a valid UTF-8 string.
+See also is_utf8_string_loclen() and is_utf8_string_loc().
+
=cut
*/
{
const U8* x = s;
const U8* send;
- STRLEN c;
if (!len && s)
len = strlen((const char *)s);
send = s + len;
while (x < send) {
+ STRLEN c;
/* Inline the easy bits of is_utf8_char() here for speed... */
if (UTF8_IS_INVARIANT(*x))
c = 1;
else if (!UTF8_IS_START(*x))
- return FALSE;
+ goto out;
else {
/* ... and call is_utf8_char() only if really needed. */
- c = is_utf8_char(x);
+#ifdef IS_UTF8_CHAR
+ c = UTF8SKIP(x);
+ if (IS_UTF8_CHAR_FAST(c)) {
+ if (!IS_UTF8_CHAR(x, c))
+ goto out;
+ } else if (!is_utf8_char_slow(x, c))
+ goto out;
+#else
+ c = is_utf8_char(x);
+#endif /* #ifdef IS_UTF8_CHAR */
if (!c)
- return FALSE;
+ goto out;
}
x += c;
}
+
+ out:
if (x != send)
return FALSE;
}
/*
-=for apidoc A|bool|is_utf8_string_loc|const U8 *s|STRLEN len|const U8 **p
+=for apidoc A|bool|is_utf8_string_loclen|const U8 *s|STRLEN len|const U8 **ep|const STRLEN *el
-Like is_ut8_string but store the location of the failure in
-the last argument.
+Like is_ut8_string() but stores the location of the failure (in the
+case of "utf8ness failure") or the location s+len (in the case of
+"utf8ness success") in the C<ep>, and the number of UTF-8
+encoded characters in the C<el>.
+
+See also is_utf8_string_loc() and is_utf8_string().
=cut
*/
bool
-Perl_is_utf8_string_loc(pTHX_ const U8 *s, STRLEN len, const U8 **p)
+Perl_is_utf8_string_loclen(pTHX_ const U8 *s, STRLEN len, const U8 **ep, STRLEN *el)
{
const U8* x = s;
const U8* send;
if (!len && s)
len = strlen((const char *)s);
send = s + len;
+ if (el)
+ *el = 0;
while (x < send) {
/* Inline the easy bits of is_utf8_char() here for speed... */
if (UTF8_IS_INVARIANT(*x))
- c = 1;
- else if (!UTF8_IS_START(*x)) {
- if (p)
- *p = x;
- return FALSE;
- }
+ c = 1;
+ else if (!UTF8_IS_START(*x))
+ goto out;
else {
- /* ... and call is_utf8_char() only if really needed. */
- c = is_utf8_char(x);
- if (!c) {
- if (p)
- *p = x;
- return FALSE;
- }
+ /* ... and call is_utf8_char() only if really needed. */
+#ifdef IS_UTF8_CHAR
+ c = UTF8SKIP(x);
+ if (IS_UTF8_CHAR_FAST(c)) {
+ if (!IS_UTF8_CHAR(x, c))
+ c = 0;
+ } else
+ c = is_utf8_char_slow(x, c);
+#else
+ c = is_utf8_char(x);
+#endif /* #ifdef IS_UTF8_CHAR */
+ if (!c)
+ goto out;
}
- x += c;
+ x += c;
+ if (el)
+ (*el)++;
}
- if (x != send) {
- if (p)
- *p = x;
+
+ out:
+ if (ep)
+ *ep = x;
+ if (x != send)
return FALSE;
- }
return TRUE;
}
/*
+=for apidoc A|bool|is_utf8_string_loc|const U8 *s|STRLEN len|const U8 **ep|const STRLEN *el
+
+Like is_ut8_string() but stores the location of the failure (in the
+case of "utf8ness failure") or the location s+len (in the case of
+"utf8ness success") in the C<ep>.
+
+See also is_utf8_string_loclen() and is_utf8_string().
+
+=cut
+*/
+
+bool
+Perl_is_utf8_string_loc(pTHX_ const U8 *s, STRLEN len, const U8 **ep)
+{
+ return is_utf8_string_loclen(s, len, ep, 0);
+}
+
+/*
=for apidoc A|UV|utf8n_to_uvuni|const U8 *s|STRLEN curlen|STRLEN *retlen|U32 flags
Bottom level UTF-8 decode routine.
(UV)s[1], startbyte);
else
Perl_sv_catpvf(aTHX_ sv, "(unexpected non-continuation byte 0x%02"UVxf", %d byte%s after start byte 0x%02"UVxf", expected %d bytes)",
- (UV)s[1], s - s0, s - s0 > 1 ? "s" : "", startbyte, expectlen);
+ (UV)s[1], s - s0, s - s0 > 1 ? "s" : "", startbyte, (int)expectlen);
break;
case UTF8_WARN_FE_FF:
break;
case UTF8_WARN_SHORT:
Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")",
- curlen, curlen == 1 ? "" : "s", expectlen, startbyte);
+ (int)curlen, curlen == 1 ? "" : "s", (int)expectlen, startbyte);
expectlen = curlen; /* distance for caller to skip */
break;
case UTF8_WARN_OVERFLOW:
break;
case UTF8_WARN_LONG:
Perl_sv_catpvf(aTHX_ sv, "(%d byte%s, need %d, after start byte 0x%02"UVxf")",
- expectlen, expectlen == 1 ? "": "s", UNISKIP(uv), startbyte);
+ (int)expectlen, expectlen == 1 ? "": "s", UNISKIP(uv), startbyte);
break;
case UTF8_WARN_FFFF:
Perl_sv_catpvf(aTHX_ sv, "(character 0x%04"UVxf")", uv);
}
if (warning) {
- char *s = SvPVX(sv);
+ const char *s = SvPVX_const(sv);
if (PL_op)
Perl_warner(aTHX_ packWARN(WARN_UTF8),
* the bitops (especially ~) can create illegal UTF-8.
* In other words: in Perl UTF-8 is not just for Unicode. */
- if (e < s) {
- if (ckWARN_d(WARN_UTF8)) {
- if (PL_op)
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
- "%s in %s", unees, OP_DESC(PL_op));
- else
- Perl_warner(aTHX_ packWARN(WARN_UTF8), unees);
- }
- return 0;
- }
+ if (e < s)
+ goto warn_and_return;
while (s < e) {
- U8 t = UTF8SKIP(s);
-
+ const U8 t = UTF8SKIP(s);
if (e - s < t) {
+ warn_and_return:
if (ckWARN_d(WARN_UTF8)) {
if (PL_op)
Perl_warner(aTHX_ packWARN(WARN_UTF8),
- unees, OP_DESC(PL_op));
+ "%s in %s", unees, OP_DESC(PL_op));
else
Perl_warner(aTHX_ packWARN(WARN_UTF8), unees);
}
if (a < b) {
while (a < b) {
const U8 c = UTF8SKIP(a);
-
- if (b - a < c) {
- if (ckWARN_d(WARN_UTF8)) {
- if (PL_op)
- Perl_warner(aTHX_ packWARN(WARN_UTF8),
- "%s in %s", unees, OP_DESC(PL_op));
- else
- Perl_warner(aTHX_ packWARN(WARN_UTF8), unees);
- }
- return off;
- }
+ if (b - a < c)
+ goto warn_and_return;
a += c;
off--;
}
}
else {
while (b < a) {
- U8 c = UTF8SKIP(b);
+ const U8 c = UTF8SKIP(b);
if (a - b < c) {
+ warn_and_return:
if (ckWARN_d(WARN_UTF8)) {
if (PL_op)
Perl_warner(aTHX_ packWARN(WARN_UTF8),
*/
U8 *
-Perl_utf8_hop(pTHX_ U8 *s, I32 off)
+Perl_utf8_hop(pTHX_ const U8 *s, I32 off)
{
/* Note: cannot use UTF8_IS_...() too eagerly here since e.g
* the bitops (especially ~) can create illegal UTF-8.
s--;
}
}
- return s;
+ return (U8 *)s;
}
/*
UV
Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, SV **swashp, const char *normal, const char *special)
{
- UV uv1;
U8 tmpbuf[UTF8_MAXBYTES_CASE+1];
STRLEN len = 0;
/* The NATIVE_TO_UNI() and UNI_TO_NATIVE() mappings
* are necessary in EBCDIC, they are redundant no-ops
* in ASCII-ish platforms, and hopefully optimized away. */
- uv1 = NATIVE_TO_UNI(uv0);
+ const UV uv1 = NATIVE_TO_UNI(uv0);
uvuni_to_utf8(tmpbuf, uv1);
if (!*swashp) /* load on-demand */
if ((hv = get_hv(special, FALSE)) &&
(svp = hv_fetch(hv, (const char*)tmpbuf, UNISKIP(uv1), FALSE)) &&
(*svp)) {
- char *s;
+ const char *s;
- s = SvPV(*svp, len);
+ s = SvPV_const(*svp, len);
if (len == 1)
len = uvuni_to_utf8(ustrp, NATIVE_TO_UNI(*(U8*)s)) - ustrp;
else {
SV*
Perl_swash_init(pTHX_ const char* pkg, const char* name, SV *listsv, I32 minbits, I32 none)
{
+ dVAR;
SV* retval;
SV* tokenbufsv = sv_newmortal();
dSP;
HV *stash = gv_stashpvn(pkg, pkg_len, FALSE);
SV* errsv_save;
+ PUSHSTACKi(PERLSI_MAGIC);
+ ENTER;
+ SAVEI32(PL_hints);
+ PL_hints = 0;
+ save_re_context();
if (!gv_fetchmeth(stash, "SWASHNEW", 8, -1)) { /* demand load utf8 */
ENTER;
errsv_save = newSVsv(ERRSV);
LEAVE;
}
SPAGAIN;
- PUSHSTACKi(PERLSI_MAGIC);
PUSHMARK(SP);
EXTEND(SP,5);
PUSHs(sv_2mortal(newSVpvn(pkg, pkg_len)));
PUSHs(sv_2mortal(newSViv(minbits)));
PUSHs(sv_2mortal(newSViv(none)));
PUTBACK;
- ENTER;
- SAVEI32(PL_hints);
- PL_hints = 0;
- save_re_context();
if (IN_PERL_COMPILETIME) {
/* XXX ought to be handled by lex_start */
SAVEI32(PL_in_my);
POPSTACK;
if (IN_PERL_COMPILETIME) {
STRLEN len;
- const char* pv = SvPV(tokenbufsv, len);
+ const char* pv = SvPV_const(tokenbufsv, len);
Copy(pv, PL_tokenbuf, len+1, char);
PL_curcop->op_private = (U8)(PL_hints & HINT_PRIVATE_MASK);
UV
Perl_swash_fetch(pTHX_ SV *sv, const U8 *ptr, bool do_utf8)
{
+ dVAR;
HV* hv = (HV*)SvRV(sv);
U32 klen;
U32 off;
STRLEN slen;
STRLEN needents;
- U8 *tmps = NULL;
+ const U8 *tmps = NULL;
U32 bit;
SV *retval;
U8 tmputf8[2];
if (hv == PL_last_swash_hv &&
klen == PL_last_swash_klen &&
- (!klen || memEQ(ptr, PL_last_swash_key, klen)) )
+ (!klen || memEQ((char *)ptr, (char *)PL_last_swash_key, klen)) )
{
tmps = PL_last_swash_tmps;
slen = PL_last_swash_slen;
SV** svp = hv_fetch(hv, (const char*)ptr, klen, FALSE);
/* If not cached, generate it via utf8::SWASHGET */
- if (!svp || !SvPOK(*svp) || !(tmps = (U8*)SvPV(*svp, slen))) {
+ if (!svp || !SvPOK(*svp) || !(tmps = (const U8*)SvPV_const(*svp, slen))) {
dSP;
/* We use utf8n_to_uvuni() as we want an index into
Unicode tables, not a native character number.
PL_last_swash_hv = hv;
PL_last_swash_klen = klen;
- PL_last_swash_tmps = tmps;
+ /* FIXME change interpvar.h? */
+ PL_last_swash_tmps = (U8 *) tmps;
PL_last_swash_slen = slen;
if (klen)
Copy(ptr, PL_last_swash_key, klen, U8);
}
u = utf8_to_uvchr((U8*)s, 0);
if (u < 256) {
- unsigned char c = u & 0xFF;
+ const unsigned char c = (unsigned char)u & 0xFF;
if (!ok && (flags & UNI_DISPLAY_BACKSLASH)) {
switch (c) {
case '\n':
char *
Perl_sv_uni_display(pTHX_ SV *dsv, SV *ssv, STRLEN pvlim, UV flags)
{
- return Perl_pv_uni_display(aTHX_ dsv, (U8*)SvPVX(ssv), SvCUR(ssv),
- pvlim, flags);
+ return Perl_pv_uni_display(aTHX_ dsv, (const U8*)SvPVX_const(ssv),
+ SvCUR(ssv), pvlim, flags);
}
/*
* indent-tabs-mode: t
* End:
*
- * vim: shiftwidth=4:
-*/
+ * ex: set ts=8 sts=4 sw=4 noet:
+ */
projects / p5sagit/p5-mst-13.2.git / blobdiff