Document the "Unicode in package/sub names" problem;

a microfix in toke.c towards the final goal (the fix
removes the need to have quotes around Unicode package
names when calling a method on them)

p4raw-id: //depot/perl@17286

diff --git a/lib/utf8.pm b/lib/utf8.pm
index e0c4ac1..5a37aec 100644 (file)
--- a/lib/utf8.pm
+++ b/lib/utf8.pm
@@ -57,9 +57,10 @@ Enabling the C<utf8> pragma has the following effect:
 Bytes in the source text that have their high-bit set will be treated
 as being part of a literal UTF-8 character.  This includes most
 literals such as identifier names, string constants, and constant
-regular expression patterns.  On EBCDIC platforms characters in
-the Latin 1 character set are treated as being part of a literal
-UTF-EBCDIC character.
+regular expression patterns.
+
+On EBCDIC platforms characters in the Latin 1 character set are
+treated as being part of a literal UTF-EBCDIC character.
 
 =back
 
@@ -131,6 +132,19 @@ functions utf8::valid, utf8::encode, utf8::decode, utf8::upgrade,
 and utf8::downgrade are always available, without a C<require utf8>
 statement-- this may change in future releases.
 
+=head1 BUGS
+
+One can have Unicode in identifier names, but not in package/class or
+subroutine names.  While some limited functionality towards this does
+exist as of Perl 5.8.0, that is more accidental than designed; use of
+Unicode for the said purposes is unsupported.
+
+One reason of this unfinishedness is its (currently) inherent
+unportability: since both package names and subroutine names may need
+to be mapped to file and directory names, the Unicode capability of
+the filesystem becomes important-- and there unfortunately aren't
+portable answers.
+
 =head1 SEE ALSO
 
 L<perlunicode>, L<bytes>

diff --git a/pod/perldelta.pod b/pod/perldelta.pod
index e6a4fe3..67806a9 100644 (file)
--- a/pod/perldelta.pod
+++ b/pod/perldelta.pod
@@ -3284,6 +3284,19 @@ In case of failure please try running them manually, for example
 
     ./perl -Ilib ext/Time/HiRes/HiRes.t
 
+=head2 Unicode in package/class and subroutine names does not work
+
+One can have Unicode in identifier names, but not in package/class or
+subroutine names.  While some limited functionality towards this does
+exist as of Perl 5.8.0, that is more accidental than designed; use of
+Unicode for the said purposes is unsupported.
+
+One reason of this unfinishedness is its (currently) inherent
+unportability: since both package names and subroutine names may
+need to be mapped to file and directory names, the Unicode capability
+of the filesystem becomes important-- and there unfortunately aren't
+portable answers.
+
 =head2 UNICOS/mk
 
 =over 4

diff --git a/pod/perltodo.pod b/pod/perltodo.pod
index 0e5d8a7..5f9a6cf 100644 (file)
--- a/pod/perltodo.pod
+++ b/pod/perltodo.pod
@@ -98,7 +98,25 @@ UTF-8 identifier names should probably be canonicalized: NFC?
 
 UTF-8 in package names and sub names?  The first is problematic
 because of the mapping to pathnames, ditto for the second one if
-one does autosplitting, for example.
+one does autosplitting, for example.  Some of this works already
+in 5.8.0, but essentially it is unsupported.  Constructs to consider,
+at the very least:
+
+       use utf8;
+       package UnicodePackage;
+       sub new { bless {}, shift };
+       sub UnicodeMethod1 { ... $_[0]->UnicodeMethod2(...) ... }
+       sub UnicodeMethod2 { ... } # in here caller(0) should contain Unicode
+       ...
+       package main;
+       my $x = UnicodePackage->new;
+       print ref $x, "\n";  # should be Unicode
+       $x->UnicodeMethod1(...);
+       my $y = UnicodeMethod3 UnicodePackage ...;
+
+In the above all I<UnicodeXxx> contain (identifier-worthy) characters
+beyond the code point 255, for example 256.  Wherever package/class or
+subroutine names can be returned needs to be checked for Unicodeness.
 
 =back
 

diff --git a/pod/perlunicode.pod b/pod/perlunicode.pod
index 25d512e..0aec6fe 100644 (file)
--- a/pod/perlunicode.pod
+++ b/pod/perlunicode.pod
@@ -37,7 +37,7 @@ included to enable recognition of UTF-8 in the Perl scripts themselves
 (in string or regular expression literals, or in identifier names) on
 ASCII-based machines or to recognize UTF-EBCDIC on EBCDIC-based
 machines.  B<These are the only times when an explicit C<use utf8>
-is needed.>
+is needed.>  See L<utf8>.
 
 You can also use the C<encoding> pragma to change the default encoding
 of the data in your script; see L<encoding>.

diff --git a/pod/perluniintro.pod b/pod/perluniintro.pod
index 1468ae3..cc11dde 100644 (file)
--- a/pod/perluniintro.pod
+++ b/pod/perluniintro.pod
@@ -132,7 +132,7 @@ operations.  Only one case remains where an explicit C<use utf8> is
 needed: if your Perl script itself is encoded in UTF-8, you can use
 UTF-8 in your identifier names, and in string and regular expression
 literals, by saying C<use utf8>.  This is not the default because
-scripts with legacy 8-bit data in them would break.
+scripts with legacy 8-bit data in them would break.  See L<utf8>.
 
 =head2 Perl's Unicode Model
 

diff --git a/toke.c b/toke.c
index 7cd0983..4077c60 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -1870,7 +1870,7 @@ S_intuit_more(pTHX_ register char *s)
  * Method if it's "foo $bar"
  * Not a method if it's really "print foo $bar"
  * Method if it's really "foo package::" (interpreted as package->foo)
- * Not a method if bar is known to be a subroutne ("sub bar; foo bar")
+ * Not a method if bar is known to be a subroutine ("sub bar; foo bar")
  * Not a method if bar is a filehandle or package, but is quoted with
  *   =>
  */
@@ -3913,6 +3913,10 @@ Perl_yylex(pTHX)
                CLINE;
                yylval.opval = (OP*)newSVOP(OP_CONST, 0, sv);
                yylval.opval->op_private = OPpCONST_BARE;
+               /* UTF-8 package name? */
+               if (UTF && !IN_BYTES &&
+                   is_utf8_string((U8*)SvPVX(sv), SvCUR(sv)))
+                   SvUTF8_on(sv);
 
                /* And if "Foo::", then that's what it certainly is. */