Update to IO-1.25 from CPAN

[p5sagit/p5-mst-13.2.git] / utf8.c

diff --git a/utf8.c b/utf8.c
index 517a67f..b5a3809 100644 (file)
--- a/utf8.c
+++ b/utf8.c
@@ -9,16 +9,23 @@
  */
 
 /*
- * 'What a fix!' said Sam. 'That's the one place in all the lands we've ever
- * heard of that we don't want to see any closer; and that's the one place
- * we're trying to get to!  And that's just where we can't get, nohow.'
+ * 'What a fix!' said Sam.  'That's the one place in all the lands we've ever
+ *  heard of that we don't want to see any closer; and that's the one place
+ *  we're trying to get to!  And that's just where we can't get, nohow.'
+ *
+ *     [p.603 of _The Lord of the Rings_, IV/I: "The Taming of Sméagol"]
  *
  * 'Well do I understand your speech,' he answered in the same language;
  * 'yet few strangers do so.  Why then do you not speak in the Common Tongue,
- * as is the custom in the West, if you wish to be answered?'
+ *  as is the custom in the West, if you wish to be answered?'
+ *                           --Gandalf, addressing Théoden's door wardens
+ *
+ *     [p.508 of _The Lord of the Rings_, III/vi: "The King of the Golden Hall"]
  *
  * ...the travellers perceived that the floor was paved with stones of many
  * hues; branching runes and strange devices intertwined beneath their feet.
+ *
+ *     [p.512 of _The Lord of the Rings_, III/vi: "The King of the Golden Hall"]
  */
 
 #include "EXTERN.h"
@@ -240,9 +247,9 @@ S_is_utf8_char_slow(const U8 *s, const STRLEN len)
 =for apidoc is_utf8_char
 
 Tests if some arbitrary number of bytes begins in a valid UTF-8
-character.  Note that an INVARIANT (i.e. ASCII) character is a valid
-UTF-8 character.  The actual number of bytes in the UTF-8 character
-will be returned if it is valid, otherwise 0.
+character.  Note that an INVARIANT (i.e. ASCII on non-EBCDIC machines)
+character is a valid UTF-8 character.  The actual number of bytes in the UTF-8
+character will be returned if it is valid, otherwise 0.
 
 =cut */
 STRLEN
@@ -641,7 +648,7 @@ Returns the Unicode code point of the first character in the string C<s>
 which is assumed to be in UTF-8 encoding; C<retlen> will be set to the
 length, in bytes, of that character.
 
-This function should only be used when returned UV is considered
+This function should only be used when the returned UV is considered
 an index into the Unicode semantic tables (e.g. swashes).
 
 If C<s> does not point to a well-formed UTF-8 character, zero is
@@ -675,7 +682,6 @@ Perl_utf8_length(pTHX_ const U8 *s, const U8 *e)
 {
     dVAR;
     STRLEN len = 0;
-    U8 t = 0;
 
     PERL_ARGS_ASSERT_UTF8_LENGTH;
 
@@ -686,20 +692,23 @@ Perl_utf8_length(pTHX_ const U8 *s, const U8 *e)
     if (e < s)
        goto warn_and_return;
     while (s < e) {
-       t = UTF8SKIP(s);
-       if (e - s < t) {
-           warn_and_return:
-           if (ckWARN_d(WARN_UTF8)) {
-               if (PL_op)
-                   Perl_warner(aTHX_ packWARN(WARN_UTF8),
+       if (!UTF8_IS_INVARIANT(*s))
+           s += UTF8SKIP(s);
+       else
+           s++;
+       len++;
+    }
+
+    if (e != s) {
+       len--;
+        warn_and_return:
+       if (ckWARN_d(WARN_UTF8)) {
+           if (PL_op)
+               Perl_warner(aTHX_ packWARN(WARN_UTF8),
                            "%s in %s", unees, OP_DESC(PL_op));
-               else
-                   Perl_warner(aTHX_ packWARN(WARN_UTF8), unees);
-           }
-           return len;
+           else
+               Perl_warner(aTHX_ packWARN(WARN_UTF8), unees);
        }
-       s += t;
-       len++;
     }
 
     return len;
@@ -765,7 +774,7 @@ Perl_utf8_hop(pTHX_ const U8 *s, I32 off)
 /*
 =for apidoc utf8_to_bytes
 
-Converts a string C<s> of length C<len> from UTF-8 into byte encoding.
+Converts a string C<s> of length C<len> from UTF-8 into native byte encoding.
 Unlike C<bytes_to_utf8>, this over-writes the original string, and
 updates len to contain the new length.
 Returns zero on failure, setting C<len> to -1.
@@ -810,12 +819,13 @@ Perl_utf8_to_bytes(pTHX_ U8 *s, STRLEN *len)
 /*
 =for apidoc bytes_from_utf8
 
-Converts a string C<s> of length C<len> from UTF-8 into byte encoding.
+Converts a string C<s> of length C<len> from UTF-8 into native byte encoding.
 Unlike C<utf8_to_bytes> but like C<bytes_to_utf8>, returns a pointer to
 the newly-created string, and updates C<len> to contain the new
 length.  Returns the original string if no conversion occurs, C<len>
 is unchanged. Do nothing if C<is_utf8> points to 0. Sets C<is_utf8> to
-0 if C<s> is converted or contains all 7bit characters.
+0 if C<s> is converted or consisted entirely of characters that are invariant
+in utf8 (i.e., US-ASCII on non-EBCDIC machines).
 
 =cut
 */
@@ -867,11 +877,14 @@ Perl_bytes_from_utf8(pTHX_ const U8 *s, STRLEN *len, bool *is_utf8)
 /*
 =for apidoc bytes_to_utf8
 
-Converts a string C<s> of length C<len> from ASCII into UTF-8 encoding.
+Converts a string C<s> of length C<len> from the native encoding into UTF-8.
 Returns a pointer to the newly-created string, and sets C<len> to
 reflect the new length.
 
-If you want to convert to UTF-8 from other encodings than ASCII,
+A NUL character will be written after the end of the string.
+
+If you want to convert to UTF-8 from encodings other than
+the native (Latin1 or EBCDIC),
 see sv_recode_to_utf8().
 
 =cut
@@ -1487,7 +1500,7 @@ Perl_to_utf8_case(pTHX_ const U8 *p, U8* ustrp, STRLEN *lenp,
     if (special && (uv1 == 0xDF || uv1 > 0xFF)) {
          /* It might be "special" (sometimes, but not always,
          * a multicharacter mapping) */
-        HV * const hv = get_hv(special, FALSE);
+        HV * const hv = get_hv(special, 0);
         SV **svp;
 
         if (hv &&