X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=blobdiff_plain;f=utf8.c;h=4f4c3eaffe5ee36cbf52c3271ccee7633d01cbd3;hb=3127de7dcb470aaa3997c4164384a5bfb3162ffe;hp=45f6a1d3e7deccfb7a4990f4a80c3bae5bfa4186;hpb=ef8f7699778dff55cfd4d5ca899723febaa96cd0;p=p5sagit%2Fp5-mst-13.2.git diff --git a/utf8.c b/utf8.c index 45f6a1d..4f4c3ea 100644 --- a/utf8.c +++ b/utf8.c @@ -9,16 +9,23 @@ */ /* - * 'What a fix!' said Sam. 'That's the one place in all the lands we've ever - * heard of that we don't want to see any closer; and that's the one place - * we're trying to get to! And that's just where we can't get, nohow.' + * 'What a fix!' said Sam. 'That's the one place in all the lands we've ever + * heard of that we don't want to see any closer; and that's the one place + * we're trying to get to! And that's just where we can't get, nohow.' + * + * [p.603 of _The Lord of the Rings_, IV/I: "The Taming of Sméagol"] * * 'Well do I understand your speech,' he answered in the same language; * 'yet few strangers do so. Why then do you not speak in the Common Tongue, - * as is the custom in the West, if you wish to be answered?' + * as is the custom in the West, if you wish to be answered?' + * --Gandalf, addressing Théoden's door wardens + * + * [p.508 of _The Lord of the Rings_, III/vi: "The King of the Golden Hall"] * * ...the travellers perceived that the floor was paved with stones of many * hues; branching runes and strange devices intertwined beneath their feet. + * + * [p.512 of _The Lord of the Rings_, III/vi: "The King of the Golden Hall"] */ #include "EXTERN.h" @@ -44,7 +51,7 @@ Unicode characters as a variable number of bytes, in such a way that characters in the ASCII range are unmodified, and a zero byte never appears within non-zero characters. -=for apidoc A|U8 *|uvuni_to_utf8_flags|U8 *d|UV uv|UV flags +=for apidoc uvuni_to_utf8_flags Adds the UTF-8 representation of the Unicode codepoint C to the end of the string C; C should be have at least C free @@ -237,12 +244,12 @@ S_is_utf8_char_slow(const U8 *s, const STRLEN len) } /* -=for apidoc A|STRLEN|is_utf8_char|const U8 *s +=for apidoc is_utf8_char Tests if some arbitrary number of bytes begins in a valid UTF-8 -character. Note that an INVARIANT (i.e. ASCII) character is a valid -UTF-8 character. The actual number of bytes in the UTF-8 character -will be returned if it is valid, otherwise 0. +character. Note that an INVARIANT (i.e. ASCII on non-EBCDIC machines) +character is a valid UTF-8 character. The actual number of bytes in the UTF-8 +character will be returned if it is valid, otherwise 0. =cut */ STRLEN @@ -260,7 +267,7 @@ Perl_is_utf8_char(pTHX_ const U8 *s) } /* -=for apidoc A|bool|is_utf8_string|const U8 *s|STRLEN len +=for apidoc is_utf8_string Returns true if first C bytes of the given string form a valid UTF-8 string, false otherwise. Note that 'a valid UTF-8 string' does @@ -317,7 +324,7 @@ Perl_is_utf8_string(pTHX_ const U8 *s, STRLEN len) /* Implemented as a macro in utf8.h -=for apidoc A|bool|is_utf8_string_loc|const U8 *s|STRLEN len|const U8 **ep +=for apidoc is_utf8_string_loc Like is_utf8_string() but stores the location of the failure (in the case of "utf8ness failure") or the location s+len (in the case of @@ -325,7 +332,7 @@ case of "utf8ness failure") or the location s+len (in the case of See also is_utf8_string_loclen() and is_utf8_string(). -=for apidoc A|bool|is_utf8_string_loclen|const U8 *s|STRLEN len|const U8 **ep|const STRLEN *el +=for apidoc is_utf8_string_loclen Like is_utf8_string() but stores the location of the failure (in the case of "utf8ness failure") or the location s+len (in the case of @@ -384,7 +391,7 @@ Perl_is_utf8_string_loclen(pTHX_ const U8 *s, STRLEN len, const U8 **ep, STRLEN /* -=for apidoc A|UV|utf8n_to_uvuni|const U8 *s|STRLEN curlen|STRLEN *retlen|U32 flags +=for apidoc utf8n_to_uvuni Bottom level UTF-8 decode routine. Returns the Unicode code point value of the first character in the string C @@ -613,7 +620,7 @@ malformed: } /* -=for apidoc A|UV|utf8_to_uvchr|const U8 *s|STRLEN *retlen +=for apidoc utf8_to_uvchr Returns the native character value of the first character in the string C which is assumed to be in UTF-8 encoding; C will be set to the @@ -635,13 +642,13 @@ Perl_utf8_to_uvchr(pTHX_ const U8 *s, STRLEN *retlen) } /* -=for apidoc A|UV|utf8_to_uvuni|const U8 *s|STRLEN *retlen +=for apidoc utf8_to_uvuni Returns the Unicode code point of the first character in the string C which is assumed to be in UTF-8 encoding; C will be set to the length, in bytes, of that character. -This function should only be used when returned UV is considered +This function should only be used when the returned UV is considered an index into the Unicode semantic tables (e.g. swashes). If C does not point to a well-formed UTF-8 character, zero is @@ -661,7 +668,7 @@ Perl_utf8_to_uvuni(pTHX_ const U8 *s, STRLEN *retlen) } /* -=for apidoc A|STRLEN|utf8_length|const U8 *s|const U8 *e +=for apidoc utf8_length Return the length of the UTF-8 char encoded string C in characters. Stops at C (inclusive). If C s> or if the scan would end @@ -706,7 +713,7 @@ Perl_utf8_length(pTHX_ const U8 *s, const U8 *e) } /* -=for apidoc A|IV|utf8_distance|const U8 *a|const U8 *b +=for apidoc utf8_distance Returns the number of UTF-8 characters between the UTF-8 pointers C and C. @@ -726,7 +733,7 @@ Perl_utf8_distance(pTHX_ const U8 *a, const U8 *b) } /* -=for apidoc A|U8 *|utf8_hop|U8 *s|I32 off +=for apidoc utf8_hop Return the UTF-8 pointer C displaced by C characters, either forward or backward. @@ -763,9 +770,9 @@ Perl_utf8_hop(pTHX_ const U8 *s, I32 off) } /* -=for apidoc A|U8 *|utf8_to_bytes|U8 *s|STRLEN *len +=for apidoc utf8_to_bytes -Converts a string C of length C from UTF-8 into byte encoding. +Converts a string C of length C from UTF-8 into native byte encoding. Unlike C, this over-writes the original string, and updates len to contain the new length. Returns zero on failure, setting C to -1. @@ -808,14 +815,15 @@ Perl_utf8_to_bytes(pTHX_ U8 *s, STRLEN *len) } /* -=for apidoc A|U8 *|bytes_from_utf8|const U8 *s|STRLEN *len|bool *is_utf8 +=for apidoc bytes_from_utf8 -Converts a string C of length C from UTF-8 into byte encoding. +Converts a string C of length C from UTF-8 into native byte encoding. Unlike C but like C, returns a pointer to the newly-created string, and updates C to contain the new length. Returns the original string if no conversion occurs, C is unchanged. Do nothing if C points to 0. Sets C to -0 if C is converted or contains all 7bit characters. +0 if C is converted or consisted entirely of characters that are invariant +in utf8 (i.e., US-ASCII on non-EBCDIC machines). =cut */ @@ -865,13 +873,16 @@ Perl_bytes_from_utf8(pTHX_ const U8 *s, STRLEN *len, bool *is_utf8) } /* -=for apidoc A|U8 *|bytes_to_utf8|const U8 *s|STRLEN *len +=for apidoc bytes_to_utf8 -Converts a string C of length C from ASCII into UTF-8 encoding. +Converts a string C of length C from the native encoding into UTF-8. Returns a pointer to the newly-created string, and sets C to reflect the new length. -If you want to convert to UTF-8 from other encodings than ASCII, +A NUL character will be written after the end of the string. + +If you want to convert to UTF-8 from encodings other than +the native (Latin1 or EBCDIC), see sv_recode_to_utf8(). =cut @@ -1439,7 +1450,7 @@ Perl_is_utf8_mark(pTHX_ const U8 *p) } /* -=for apidoc A|UV|to_utf8_case|U8 *p|U8* ustrp|STRLEN *lenp|SV **swash|char *normal|char *special +=for apidoc to_utf8_case The "p" contains the pointer to the UTF-8 string encoding the character that is being converted. @@ -1487,7 +1498,7 @@ Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, if (special && (uv1 == 0xDF || uv1 > 0xFF)) { /* It might be "special" (sometimes, but not always, * a multicharacter mapping) */ - HV * const hv = get_hv(special, FALSE); + HV * const hv = get_hv(special, 0); SV **svp; if (hv && @@ -1554,7 +1565,7 @@ Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp, } /* -=for apidoc A|UV|to_utf8_upper|const U8 *p|U8 *ustrp|STRLEN *lenp +=for apidoc to_utf8_upper Convert the UTF-8 encoded character at p to its uppercase version and store that in UTF-8 in ustrp and its length in bytes in lenp. Note @@ -1578,7 +1589,7 @@ Perl_to_utf8_upper(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) } /* -=for apidoc A|UV|to_utf8_title|const U8 *p|U8 *ustrp|STRLEN *lenp +=for apidoc to_utf8_title Convert the UTF-8 encoded character at p to its titlecase version and store that in UTF-8 in ustrp and its length in bytes in lenp. Note @@ -1602,7 +1613,7 @@ Perl_to_utf8_title(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) } /* -=for apidoc A|UV|to_utf8_lower|const U8 *p|U8 *ustrp|STRLEN *lenp +=for apidoc to_utf8_lower Convert the UTF-8 encoded character at p to its lowercase version and store that in UTF-8 in ustrp and its length in bytes in lenp. Note @@ -1626,7 +1637,7 @@ Perl_to_utf8_lower(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp) } /* -=for apidoc A|UV|to_utf8_fold|const U8 *p|U8 *ustrp|STRLEN *lenp +=for apidoc to_utf8_fold Convert the UTF-8 encoded character at p to its foldcase version and store that in UTF-8 in ustrp and its length in bytes in lenp. Note @@ -2167,7 +2178,7 @@ S_swash_get(pTHX_ SV* swash, UV start, UV span) } /* -=for apidoc A|U8 *|uvchr_to_utf8|U8 *d|UV uv +=for apidoc uvchr_to_utf8 Adds the UTF-8 representation of the Native codepoint C to the end of the string C; C should be have at least C free @@ -2203,7 +2214,7 @@ Perl_uvchr_to_utf8_flags(pTHX_ U8 *d, UV uv, UV flags) } /* -=for apidoc A|UV|utf8n_to_uvchr|U8 *s|STRLEN curlen|STRLEN *retlen|U32 +=for apidoc utf8n_to_uvchr flags Returns the native character value of the first character in the string @@ -2230,7 +2241,7 @@ U32 flags) } /* -=for apidoc A|char *|pv_uni_display|SV *dsv|U8 *spv|STRLEN len|STRLEN pvlim|UV flags +=for apidoc pv_uni_display Build to the scalar dsv a displayable version of the string spv, length len, the displayable version being at most pvlim bytes long @@ -2254,7 +2265,7 @@ Perl_pv_uni_display(pTHX_ SV *dsv, const U8 *spv, STRLEN len, STRLEN pvlim, UV f PERL_ARGS_ASSERT_PV_UNI_DISPLAY; - sv_setpvn(dsv, "", 0); + sv_setpvs(dsv, ""); SvUTF8_off(dsv); for (s = (const char *)spv, e = s + len; s < e; s += UTF8SKIP(s)) { UV u; @@ -2288,7 +2299,7 @@ Perl_pv_uni_display(pTHX_ SV *dsv, const U8 *spv, STRLEN len, STRLEN pvlim, UV f } if (ok) { const char string = ok; - sv_catpvn(dsv, "\\", 1); + sv_catpvs(dsv, "\\"); sv_catpvn(dsv, &string, 1); } } @@ -2309,7 +2320,7 @@ Perl_pv_uni_display(pTHX_ SV *dsv, const U8 *spv, STRLEN len, STRLEN pvlim, UV f } /* -=for apidoc A|char *|sv_uni_display|SV *dsv|SV *ssv|STRLEN pvlim|UV flags +=for apidoc sv_uni_display Build to the scalar dsv a displayable version of the scalar sv, the displayable version being at most pvlim bytes long @@ -2331,7 +2342,7 @@ Perl_sv_uni_display(pTHX_ SV *dsv, SV *ssv, STRLEN pvlim, UV flags) } /* -=for apidoc A|I32|ibcmp_utf8|const char *s1|char **pe1|register UV l1|bool u1|const char *s2|char **pe2|register UV l2|bool u2 +=for apidoc ibcmp_utf8 Return true if the strings s1 and s2 differ case-insensitively, false if not (if they are equal case-insensitively). If u1 is true, the