From: Gurusamy Sarathy Date: Tue, 1 Feb 2000 20:29:30 +0000 (+0000) Subject: HINT_UTF8 is not propagated to the op tree anymore; add a X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=393fec973b1b95a178b4b9600173880d9f93debf;p=p5sagit%2Fp5-mst-13.2.git HINT_UTF8 is not propagated to the op tree anymore; add a perlunicode.pod that reflects changes to unicode support so far p4raw-id: //depot/perl@4941 --- diff --git a/MANIFEST b/MANIFEST index 7d248e5..a6506d8 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1104,6 +1104,7 @@ pod/perltodo.pod Todo list explained pod/perltoot.pod Tom's object-oriented tutorial pod/perltootc.pod Tom's object-oriented tutorial (more on class data) pod/perltrap.pod Trap info +pod/perlunicode.pod Unicode support info pod/perlvar.pod Variable info pod/perlxs.pod XS api info pod/perlxstut.pod XS tutorial diff --git a/lib/byte.pm b/lib/byte.pm index cc23b40..33ffb76 100644 --- a/lib/byte.pm +++ b/lib/byte.pm @@ -20,7 +20,7 @@ __END__ =head1 NAME -byte - Perl pragma to turn force treating strings as bytes not UNICODE +byte - Perl pragma to force byte semantics rather than character semantics =head1 SYNOPSIS @@ -29,5 +29,24 @@ byte - Perl pragma to turn force treating strings as bytes not UNICODE =head1 DESCRIPTION +WARNING: The implementation of Unicode support in Perl is incomplete. +Expect sudden and unannounced changes! + +The C pragma disables character semantics for the rest of the +lexical scope in which it appears. C can be used to reverse +the effect of C within the current lexical scope. + +Perl normally assumes character semantics in the presence of +character data (i.e. data that has come from a source that has +been marked as being of a particular character encoding) or when +the global $^U flag is enabled. [XXX: implement -C command line +switch and mention that instead of $^U?] + +To understand the implications and differences between character +semantics and byte semantics, see L. + +=head1 SEE ALSO + +L, L =cut diff --git a/lib/byte_heavy.pl b/lib/byte_heavy.pl index 07c908a..ec05585 100644 --- a/lib/byte_heavy.pl +++ b/lib/byte_heavy.pl @@ -1,8 +1,8 @@ package byte; -sub length ($) -{ - return CORE::length($_[0]); +sub length ($) { + BEGIN { byte::import() } + return CORE::length($_[0]); } 1; diff --git a/lib/utf8.pm b/lib/utf8.pm index 691de0d..be7cc0b 100644 --- a/lib/utf8.pm +++ b/lib/utf8.pm @@ -1,14 +1,15 @@ package utf8; -$^U = 1; +$^U = 1 if caller and caller eq 'main'; # they are unicode aware + # XXX split this out? sub import { - $^H |= 0x00000008; + $^H |= 0x00800000; $enc{caller()} = $_[1] if $_[1]; } sub unimport { - $^H &= ~0x00000008; + $^H &= ~0x00800000; } sub AUTOLOAD { @@ -21,7 +22,7 @@ __END__ =head1 NAME -utf8 - Perl pragma to turn on UTF-8 and Unicode support +utf8 - Perl pragma to enable/disable UTF-8 in source code =head1 SYNOPSIS @@ -30,154 +31,60 @@ utf8 - Perl pragma to turn on UTF-8 and Unicode support =head1 DESCRIPTION -The utf8 pragma tells Perl to use UTF-8 as its internal string -representation for the rest of the enclosing block. (The "no utf8" -pragma tells Perl to switch back to ordinary byte-oriented processing -for the rest of the enclosing block.) Under utf8, many operations that -formerly operated on bytes change to operating on characters. For -ASCII data this makes no difference, because UTF-8 stores ASCII in -single bytes, but for any character greater than C, the -character is stored in a sequence of two or more bytes, all of which -have the high bit set. But by and large, the user need not worry about -this, because the utf8 pragma hides it from the user. A character -under utf8 is logically just a number ranging from 0 to 2**32 or so. -Larger characters encode to longer sequences of bytes, but again, this -is hidden. +WARNING: The implementation of Unicode support in Perl is incomplete. +Expect sudden and unannounced changes! -Use of the utf8 pragma has the following effects: +The C pragma tells the Perl parser to allow UTF-8 in the +program text in the current lexical scope. The C pragma +tells Perl to switch back to treating the source text as literal +bytes in the current lexical scope. -=over 4 +This pragma is primarily a compatibility device. Perl versions +earlier than 5.6 allowed arbitrary bytes in source code, whereas +in future we would like to standardize on the UTF-8 encoding for +source text. Until UTF-8 becomes the default format for source +text, this pragma should be used to recognize UTF-8 in the source. +When UTF-8 becomes the standard source format, this pragma will +effectively become a no-op. -=item * - -Strings and patterns may contain characters that have an ordinal value -larger than 255. Presuming you use a Unicode editor to edit your -program, these will typically occur directly within the literal strings -as UTF-8 characters, but you can also specify a particular character -with an extension of the C<\x> notation. UTF-8 characters are -specified by putting the hexadecimal code within curlies after the -C<\x>. For instance, a Unicode smiley face is C<\x{263A}>. A -character in the Latin-1 range (128..255) should be written C<\x{ab}> -rather than C<\xab>, since the former will turn into a two-byte UTF-8 -code, while the latter will continue to be interpreted as generating a -8-bit byte rather than a character. In fact, if C<-w> is turned on, it will -produce a warning that you might be generating invalid UTF-8. - -=item * +Enabling the C pragma has the following effects: -Identifiers within the Perl script may contain Unicode alphanumeric -characters, including ideographs. (You are currently on your own when -it comes to using the canonical forms of characters--Perl doesn't (yet) -attempt to canonicalize variable names for you.) +=over =item * -Regular expressions match characters instead of bytes. For instance, -"." matches a character instead of a byte. (However, the C<\C> pattern -is provided to force a match a single byte ("C" in C, hence -C<\C>).) +Bytes in the source text that have their high-bit set will be treated +as being part of a literal UTF-8 character. This includes most literals +such as identifiers, string constants, constant regular expression patterns +and package names. =item * -Character classes in regular expressions match characters instead of -bytes, and match against the character properties specified in the -Unicode properties database. So C<\w> can be used to match an ideograph, -for instance. +As a side effect, when this pragma is used within the main package, +it also enables Unicode character semantics for the entire program. +See L for more on that. -=item * - -Named Unicode properties and block ranges make be used as character -classes via the new C<\p{}> (matches property) and C<\P{}> (doesn't -match property) constructs. For instance, C<\p{Lu}> matches any -character with the Unicode uppercase property, while C<\p{M}> matches -any mark character. Single letter properties may omit the brackets, so -that can be written C<\pM> also. Many predefined character classes are -available, such as C<\p{IsMirrored}> and C<\p{InTibetan}>. +[XXX: split this out into separate "pragma" and/or -C command-line +switch?] =item * -The special pattern C<\X> match matches any extended Unicode sequence -(a "combining character sequence" in Standardese), where the first -character is a base character and subsequent characters are mark -characters that apply to the base character. It is equivalent to -C<(?:\PM\pM*)>. - -=item * +In the absence of inputs marked as UTF-8, regular expressions within the +scope of this pragma will default to using character semantics instead +of byte semantics. -The C operator translates characters instead of bytes. It can also -be forced to translate between 8-bit codes and UTF-8 regardless of the -surrounding utf8 state. For instance, if you know your input in Latin-1, -you can say: - - use utf8; - while (<>) { - tr/\0-\xff//CU; # latin1 char to utf8 - ... + @bytes_or_chars = split //, $data; # may split to bytes if data + # $data isn't UTF-8 + { + use utf8; # force char semantics + @chars = split //, $data; # splits characters } -Similarly you could translate your output with - - tr/\0-\x{ff}//UC; # utf8 to latin1 char - -No, C doesn't take /U or /C (yet?). - -=item * - -Case translation operators use the Unicode case translation tables. -Note that C translates to uppercase, while C translates -to titlecase (for languages that make the distinction). Naturally -the corresponding backslash sequences have the same semantics. - -=item * - -Most operators that deal with positions or lengths in the string will -automatically switch to using character positions, including C, -C, C, C, C, C, -C, and C. Operators that specifically don't switch -include C, C, and C. Operators that really -don't care include C, as well as any other operator that -treats a string as a bucket of bits, such as C, and the -operators dealing with filenames. - -=item * - -The C/C letters "C" and "C" do I change, -since they're often used for byte-oriented formats. (Again, think -"C" in the C language.) However, there is a new "C" specifier -that will convert between UTF-8 characters and integers. (It works -outside of the utf8 pragma too.) - -=item * - -The C and C functions work on characters. This is like -C and C, not like C and -C. In fact, the latter are how you now emulate -byte-oriented C and C under utf8. - -=item * - -And finally, C reverses by character rather than by byte. - -=back - -=head1 CAVEATS - -As of yet, there is no method for automatically coercing input and -output to some encoding other than UTF-8. This is planned in the near -future, however. +[XXX: Should this should be enabled like chr()/sprintf("%c") by looking +at $^U instead?] -In any event, you'll need to keep track of whether interfaces to other -modules expect UTF-8 data or something else. The utf8 pragma does not -magically mark strings for you in order to remember their encoding, nor -will any automatic coercion happen (other than that eventually planned -for I/O). If you want such automatic coercion, you can build yourself -a set of pretty object-oriented modules. Expect it to run considerably -slower than than this low-level support. +=head1 SEE ALSO -Use of locales with utf8 may lead to odd results. Currently there is -some attempt to apply 8-bit locale info to characters in the range -0..255, but this is demonstrably incorrect for locales that use -characters above that range (when mapped into Unicode). It will also -tend to run slower. Avoidance of locales is strongly encouraged. +L, L =cut diff --git a/op.c b/op.c index 6bb7876..953ee1c 100644 --- a/op.c +++ b/op.c @@ -2867,7 +2867,7 @@ Perl_pmruntime(pTHX_ OP *o, OP *expr, OP *repl) p = SvPV(pat, plen); pm->op_pmflags |= PMf_SKIPWHITE; } - else if (DO_UTF8(pat)) + if ((PL_hints & HINT_UTF8) || (SvUTF8(pat) && !(PL_hints & HINT_BYTE))) pm->op_pmdynflags |= PMdf_UTF8; pm->op_pmregexp = CALLREGCOMP(aTHX_ p, p + plen, pm); if (strEQ("\\s+", pm->op_pmregexp->precomp)) @@ -2875,6 +2875,8 @@ Perl_pmruntime(pTHX_ OP *o, OP *expr, OP *repl) op_free(expr); } else { + if (PL_hints & HINT_UTF8) + pm->op_pmdynflags |= PMdf_UTF8; if (pm->op_pmflags & PMf_KEEP || !(PL_hints & HINT_RE_EVAL)) expr = newUNOP((!(PL_hints & HINT_RE_EVAL) ? OP_REGCRESET @@ -3401,7 +3403,7 @@ Perl_newSTATEOP(pTHX_ I32 flags, char *label, OP *o) cop->op_ppaddr = PL_ppaddr[ OP_NEXTSTATE ]; } cop->op_flags = flags; - cop->op_private = (PL_hints & (HINT_UTF8|HINT_BYTE)); + cop->op_private = (PL_hints & HINT_BYTE); #ifdef NATIVE_HINTS cop->op_private |= NATIVE_HINTS; #endif diff --git a/perl.h b/perl.h index 518f4b3..d89f3a8 100644 --- a/perl.h +++ b/perl.h @@ -2467,8 +2467,8 @@ enum { /* pass one of these to get_vtbl */ #define HINT_INTEGER 0x00000001 #define HINT_STRICT_REFS 0x00000002 /* #define HINT_notused4 0x00000004 */ -#define HINT_UTF8 0x00000008 -#define HINT_BYTE 0x00000010 +#define HINT_BYTE 0x00000008 +/* #define HINT_notused10 0x00000010 */ /* Note: 20,40,80 used for NATIVE_HINTS */ #define HINT_BLOCK_SCOPE 0x00000100 @@ -2487,6 +2487,7 @@ enum { /* pass one of these to get_vtbl */ #define HINT_RE_EVAL 0x00200000 #define HINT_FILETEST_ACCESS 0x00400000 +#define HINT_UTF8 0x00800000 /* Various states of an input record separator SV (rs, nrs) */ #define RsSNARF(sv) (! SvOK(sv)) diff --git a/pod/perldelta.pod b/pod/perldelta.pod index cba167f..2b5e4e7 100644 --- a/pod/perldelta.pod +++ b/pod/perldelta.pod @@ -378,8 +378,9 @@ building and installing from source, the defaults should be fine. =head2 Unicode and UTF-8 support Perl can optionally use UTF-8 as its internal representation for character -strings. The C pragma enables this support in the current lexical -scope. See L for more information. +strings. The C and C pragmas are used to control this support +in the current lexical scope. See L, L and L for +more information. =head2 Interpreter cloning, threads, and concurrency @@ -1653,6 +1654,10 @@ A tutorial that introduces the essentials of references. A tutorial on managing class data for object modules. +=item perlunicode.pod + +An introduction to Unicode support features in Perl. + =back =head1 New or Changed Diagnostics diff --git a/pod/perlunicode.pod b/pod/perlunicode.pod new file mode 100644 index 0000000..b0efcca --- /dev/null +++ b/pod/perlunicode.pod @@ -0,0 +1,202 @@ +=head1 NAME + +perlunicode - Unicode support in Perl + +=head1 DESCRIPTION + +WARNING: The implementation of Unicode support in Perl is incomplete. +Expect sudden and unannounced changes! + +Beginning with version 5.6, Perl uses logically wide characters to +represent strings internally. This internal representation of strings +uses the UTF-8 encoding. + +In future, Perl-level operations will expect to work with characters +rather than bytes, in general. + +However, Perl v5.6 aims to provide a safe migration path from byte +semantics to character semantics for programs. To preserve compatibility +with earlier versions of Perl which allowed byte semantics in Perl +operations (owing to the fact that the internal representation for +characters was in bytes) byte semantics will continue to be in effect +until a the C pragma is used in the C
package, or the C<$^U> +global flag is explicitly set. + +Under character semantics, many operations that formerly operated on +bytes change to operating on characters. For ASCII data this makes +no difference, because UTF-8 stores ASCII in single bytes, but for +any character greater than C, the character is stored in +a sequence of two or more bytes, all of which have the high bit set. +But by and large, the user need not worry about this, because Perl +hides it from the user. A character in Perl is logically just a number +ranging from 0 to 2**32 or so. Larger characters encode to longer +sequences of bytes internally, but again, this is just an internal +detail which is hidden at the Perl level. + +The C pragma can be used to force byte semantics in a particular +lexical scope. See L. + +The C pragma is a compatibility device to enables recognition +of UTF-8 in literals encountered by the parser. It is also used +for enabling some experimental Unicode support features. Note that +this pragma is only required until a future version of Perl in which +character semantics will become the default. This pragma may then +become a no-op. See L. + +Character semantics have the following effects: + +=over 4 + +=item * + +Strings and patterns may contain characters that have an ordinal value +larger than 255. In Perl v5.6, this is only enabled if the lexical +scope has a C declaration (due to compatibility needs) but +future versions may enable this by default. + +Presuming you use a Unicode editor to edit your program, such characters +will typically occur directly within the literal strings as UTF-8 +characters, but you can also specify a particular character with an +extension of the C<\x> notation. UTF-8 characters are specified by +putting the hexadecimal code within curlies after the C<\x>. For instance, +a Unicode smiley face is C<\x{263A}>. A character in the Latin-1 range +(128..255) should be written C<\x{ab}> rather than C<\xab>, since the +former will turn into a two-byte UTF-8 code, while the latter will +continue to be interpreted as generating a 8-bit byte rather than a +character. In fact, if C<-w> is turned on, it will produce a warning +that you might be generating invalid UTF-8. + +=item * + +Identifiers within the Perl script may contain Unicode alphanumeric +characters, including ideographs. (You are currently on your own when +it comes to using the canonical forms of characters--Perl doesn't (yet) +attempt to canonicalize variable names for you.) + +This also needs C currently. [XXX: Why? High-bit chars were +syntax errors when they occurred within identifiers in previous versions, +so this should be enabled by default.] + +=item * + +Regular expressions match characters instead of bytes. For instance, +"." matches a character instead of a byte. (However, the C<\C> pattern +is provided to force a match a single byte ("C" in C, hence +C<\C>).) + +Unicode support in regular expressions needs C currently. +[XXX: Because the SWASH routines need to be loaded. And the RE engine +appears to need an overhaul to Unicode by default anyway.] + +=item * + +Character classes in regular expressions match characters instead of +bytes, and match against the character properties specified in the +Unicode properties database. So C<\w> can be used to match an ideograph, +for instance. + +C is needed to enable this. See above. + +=item * + +Named Unicode properties and block ranges make be used as character +classes via the new C<\p{}> (matches property) and C<\P{}> (doesn't +match property) constructs. For instance, C<\p{Lu}> matches any +character with the Unicode uppercase property, while C<\p{M}> matches +any mark character. Single letter properties may omit the brackets, so +that can be written C<\pM> also. Many predefined character classes are +available, such as C<\p{IsMirrored}> and C<\p{InTibetan}>. + +C is needed to enable this. See above. + +=item * + +The special pattern C<\X> match matches any extended Unicode sequence +(a "combining character sequence" in Standardese), where the first +character is a base character and subsequent characters are mark +characters that apply to the base character. It is equivalent to +C<(?:\PM\pM*)>. + +C is needed to enable this. See above. + +=item * + +The C operator translates characters instead of bytes. It can also +be forced to translate between 8-bit codes and UTF-8 regardless of the +surrounding utf8 state. For instance, if you know your input in Latin-1, +you can say: + + use utf8; + while (<>) { + tr/\0-\xff//CU; # latin1 char to utf8 + ... + } + +Similarly you could translate your output with + + tr/\0-\x{ff}//UC; # utf8 to latin1 char + +No, C doesn't take /U or /C (yet?). + +C is needed to enable this. See above. + +=item * + +Case translation operators use the Unicode case translation tables +when provided character input. Note that C translates to +uppercase, while C translates to titlecase (for languages +that make the distinction). Naturally the corresponding backslash +sequences have the same semantics. + +=item * + +Most operators that deal with positions or lengths in the string will +automatically switch to using character positions, including C, +C, C, C, C, C, +C, and C. Operators that specifically don't switch +include C, C, and C. Operators that really +don't care include C, as well as any other operator that +treats a string as a bucket of bits, such as C, and the +operators dealing with filenames. + +=item * + +The C/C letters "C" and "C" do I change, +since they're often used for byte-oriented formats. (Again, think +"C" in the C language.) However, there is a new "C" specifier +that will convert between UTF-8 characters and integers. (It works +outside of the utf8 pragma too.) + +=item * + +The C and C functions work on characters. This is like +C and C, not like C and +C. In fact, the latter are how you now emulate +byte-oriented C and C under utf8. + +=item * + +And finally, C reverses by character rather than by byte. + +=back + +=head1 CAVEATS + +As of yet, there is no method for automatically coercing input and +output to some encoding other than UTF-8. This is planned in the near +future, however. + +Whether a piece of data will be treated as "characters" or "bytes" +by internal operations cannot be divined at the current time. + +Use of locales with utf8 may lead to odd results. Currently there is +some attempt to apply 8-bit locale info to characters in the range +0..255, but this is demonstrably incorrect for locales that use +characters above that range (when mapped into Unicode). It will also +tend to run slower. Avoidance of locales is strongly encouraged. + +=head1 SEE ALSO + +L, L, L + +=cut diff --git a/pp_ctl.c b/pp_ctl.c index fd725a3..716be5e 100644 --- a/pp_ctl.c +++ b/pp_ctl.c @@ -116,8 +116,6 @@ PP(pp_regcomp) pm->op_pmflags = pm->op_pmpermflags; /* reset case sensitivity */ if (DO_UTF8(tmpstr)) pm->op_pmdynflags |= PMdf_UTF8; - else - pm->op_pmdynflags &= ~PMdf_UTF8; pm->op_pmregexp = CALLREGCOMP(aTHX_ t, t + len, pm); PL_reginterp_cnt = 0; /* XXXX Be extra paranoid - needed inside tie/overload accessors. */ diff --git a/regcomp.c b/regcomp.c index 5972724..bf08761 100644 --- a/regcomp.c +++ b/regcomp.c @@ -1339,9 +1339,8 @@ Perl_pregcomp(pTHX_ char *exp, char *xend, PMOP *pm) if (exp == NULL) FAIL("NULL regexp argument"); - if (PL_curcop == &PL_compiling ? (PL_hints & HINT_UTF8) : IN_UTF8) { + if (pm->op_pmdynflags & PMdf_UTF8) { PL_reg_flags |= RF_utf8; - pm->op_pmdynflags |= PMdf_UTF8; } else PL_reg_flags = 0; diff --git a/regexec.c b/regexec.c index acdbd89..106c5d1 100644 --- a/regexec.c +++ b/regexec.c @@ -2335,6 +2335,7 @@ S_regmatch(pTHX_ regnode *prog) I32 onpar = PL_regnpar; pm.op_pmflags = 0; + pm.op_pmdynflags = (UTF||DO_UTF8(ret) ? PMdf_UTF8 : 0); re = CALLREGCOMP(aTHX_ t, t + len, &pm); if (!(SvFLAGS(ret) & (SVs_TEMP | SVs_PADTMP | SVf_READONLY))) diff --git a/utf8.h b/utf8.h index 578abc7..96ae621 100644 --- a/utf8.h +++ b/utf8.h @@ -26,7 +26,7 @@ EXTCONST unsigned char PL_utf8skip[]; END_EXTERN_C -#define IN_UTF8 (PL_curcop->op_private & HINT_UTF8) +/*#define IN_UTF8 (PL_curcop->op_private & HINT_UTF8)*/ #define IN_BYTE (PL_curcop->op_private & HINT_BYTE) #define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTE)