package Encode;
use strict;
-our $VERSION = do { my @r = (q$Revision: 0.96 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r };
+our $VERSION = do { my @r = (q$Revision: 1.26 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r };
our $DEBUG = 0;
require DynaLoader;
use Carp;
+our $ON_EBCDIC = (ord("A") == 193);
+
use Encode::Alias;
# Make a %Encoding package variable to allow a certain amount of cheating
our %Encoding;
+our %ExtModule;
+
+my @codepages = qw(
+ 37 424 437 500 737 775 850 852 855
+ 856 857 860 861 862 863 864 865 866
+ 869 874 875 932 936 949 950 1006 1026
+ 1047 1250 1251 1252 1253 1254 1255 1256 1257
+ 1258
+ );
+
+my @macintosh = qw(
+ CentralEurRoman Croatian Cyrillic Greek
+ Iceland Roman Rumanian Sami
+ Thai Turkish Ukrainian
+ );
-our %ExtModule =
- (
- viscii => 'Encode/Byte.pm',
+for my $k (2..11,13..16){
+ $ExtModule{"iso-8859-$k"} = 'Encode/Byte.pm';
+}
+
+for my $k (@codepages){
+ $ExtModule{"cp$k"} = 'Encode/Byte.pm';
+}
+
+for my $k (@macintosh)
+{
+ $ExtModule{"mac$k"} = 'Encode/Byte.pm';
+}
+
+%ExtModule =
+ (%ExtModule,
'koi8-r' => 'Encode/Byte.pm',
- cp1047 => 'Encode/EBCDIC.pm',
- cp37 => 'Encode/EBCDIC.pm',
'posix-bc' => 'Encode/EBCDIC.pm',
- symbol => 'Encode/Symbol.pm',
+ cp037 => 'Encode/EBCDIC.pm',
+ cp1026 => 'Encode/EBCDIC.pm',
+ cp1047 => 'Encode/EBCDIC.pm',
+ cp500 => 'Encode/EBCDIC.pm',
+ cp875 => 'Encode/EBCDIC.pm',
dingbats => 'Encode/Symbol.pm',
+ macDingbats => 'Encode/Symbol.pm',
+ macSymbol => 'Encode/Symbol.pm',
+ symbol => 'Encode/Symbol.pm',
+ viscii => 'Encode/Byte.pm',
+);
+
+unless ($ON_EBCDIC) { # CJK added to autoload unless EBCDIC env
+%ExtModule =
+ (%ExtModule,
+
+ 'cp936' => 'Encode/CN.pm',
'euc-cn' => 'Encode/CN.pm',
- gb2312 => 'Encode/CN.pm',
- gb12345 => 'Encode/CN.pm',
- gbk => 'Encode/CN.pm',
- cp936 => 'Encode/CN.pm',
+ 'gb12345-raw' => 'Encode/CN.pm',
+ 'gb2312-raw' => 'Encode/CN.pm',
+ 'gbk' => 'Encode/CN.pm',
'iso-ir-165' => 'Encode/CN.pm',
+
+ '7bit-jis' => 'Encode/JP.pm',
+ 'cp932' => 'Encode/JP.pm',
'euc-jp' => 'Encode/JP.pm',
'iso-2022-jp' => 'Encode/JP.pm',
- '7bit-jis' => 'Encode/JP.pm',
- shiftjis => 'Encode/JP.pm',
- macjapan => 'Encode/JP.pm',
- cp932 => 'Encode/JP.pm',
+ 'iso-2022-jp-1' => 'Encode/JP.pm',
+ 'jis0201-raw' => 'Encode/JP.pm',
+ 'jis0208-raw' => 'Encode/JP.pm',
+ 'jis0212-raw' => 'Encode/JP.pm',
+ 'macJapanese' => 'Encode/JP.pm',
+ 'shiftjis' => 'Encode/JP.pm',
+
+ 'cp949' => 'Encode/KR.pm',
'euc-kr' => 'Encode/KR.pm',
- ksc5601 => 'Encode/KR.pm',
- cp949 => 'Encode/KR.pm',
- big5 => 'Encode/TW.pm',
+ 'ksc5601' => 'Encode/KR.pm',
+ 'macKorean' => 'Encode/KR.pm',
+
+ 'big5' => 'Encode/TW.pm',
'big5-hkscs' => 'Encode/TW.pm',
- cp950 => 'Encode/TW.pm',
- gb18030 => 'Encode/HanExtra.pm',
- big5plus => 'Encode/HanExtra.pm',
+ 'cp950' => 'Encode/TW.pm',
+
+ 'big5plus' => 'Encode/HanExtra.pm',
'euc-tw' => 'Encode/HanExtra.pm',
+ 'gb18030' => 'Encode/HanExtra.pm',
);
-
-for my $k (2..11,13..16){
- $ExtModule{"iso-8859-$k"} = 'Encode/Byte.pm';
}
-for my $k (1250..1258){
- $ExtModule{"cp$k"} = 'Encode/Byte.pm';
-}
-
-for my $k (qw(centeuro croatian cyrillic dingbats greek
- iceland roman rumanian sami
- thai turkish ukraine))
-{
- $ExtModule{"mac$k"} = 'Encode/Byte.pm';
-}
-
-
sub encodings
{
my $class = shift;
- my @modules = ($_[0] eq ":all") ? values %ExtModule : @_;
+ my @modules = (@_ and $_[0] eq ":all") ? values %ExtModule : @_;
for my $m (@modules)
{
$DEBUG and warn "about to require $m;";
croak("Unknown encoding '$to'") unless defined $t;
my $uni = $f->decode($string,$check);
return undef if ($check && length($string));
- $string = $t->encode($uni,$check);
+ $string = $t->encode($uni,$check);
return undef if ($check && length($uni));
- return length($_[0] = $string);
+ return defined($_[0] = $string) ? length($string) : undef ;
}
sub encode_utf8
return $str;
}
+predefine_encodings();
+
+#
+# This is to restore %Encoding if really needed;
+#
+sub predefine_encodings{
+ if ($ON_EBCDIC) {
+ # was in Encode::UTF_EBCDIC
+ package Encode::UTF_EBCDIC;
+ *name = sub{ shift->{'Name'} };
+ *new_sequence = sub{ return $_[0] };
+ *decode = sub{
+ my ($obj,$str,$chk) = @_;
+ my $res = '';
+ for (my $i = 0; $i < length($str); $i++) {
+ $res .=
+ chr(utf8::unicode_to_native(ord(substr($str,$i,1))));
+ }
+ $_[1] = '' if $chk;
+ return $res;
+ };
+ *encode = sub{
+ my ($obj,$str,$chk) = @_;
+ my $res = '';
+ for (my $i = 0; $i < length($str); $i++) {
+ $res .=
+ chr(utf8::native_to_unicode(ord(substr($str,$i,1))));
+ }
+ $_[1] = '' if $chk;
+ return $res;
+ };
+ $Encode::Encoding{Unicode} =
+ bless {Name => "UTF_EBCDIC"}, "Encode::UTF_EBCDIC";
+ } else {
+ # was in Encode::UTF_EBCDIC
+ package Encode::Internal;
+ *name = sub{ shift->{'Name'} };
+ *new_sequence = sub{ return $_[0] };
+ *decode = sub{
+ my ($obj,$str,$chk) = @_;
+ utf8::upgrade($str);
+ $_[1] = '' if $chk;
+ return $str;
+ };
+ *encode = \&decode;
+ $Encode::Encoding{Unicode} =
+ bless {Name => "Internal"}, "Encode::Internal";
+ }
+
+ {
+ # was in Encode::utf8
+ package Encode::utf8;
+ *name = sub{ shift->{'Name'} };
+ *new_sequence = sub{ return $_[0] };
+ *decode = sub{
+ my ($obj,$octets,$chk) = @_;
+ my $str = Encode::decode_utf8($octets);
+ if (defined $str) {
+ $_[1] = '' if $chk;
+ return $str;
+ }
+ return undef;
+ };
+ *encode = sub {
+ my ($obj,$string,$chk) = @_;
+ my $octets = Encode::encode_utf8($string);
+ $_[1] = '' if $chk;
+ return $octets;
+ };
+ $Encode::Encoding{utf8} =
+ bless {Name => "utf8"}, "Encode::utf8";
+ }
+ # do externals if necessary
+ require File::Basename;
+ require File::Spec;
+ for my $ext (qw(Unicode)){
+ my $pm =
+ File::Spec->catfile(File::Basename::dirname($INC{'Encode.pm'}),
+ "Encode", "$ext.pm");
+ do $pm;
+ }
+}
+
require Encode::Encoding;
require Encode::XS;
-require Encode::Internal;
-require Encode::Unicode;
-require Encode::utf8;
-require Encode::iso10646_1;
-require Encode::ucs2_le;
1;
use Encode;
+
+=head2 Table of Contents
+
+Encode consists of a collection of modules which details are too big
+to fit in one document. This POD itself explains the top-level APIs
+and general topics at a glance. For other topics and more details,
+see the PODs below;
+
+ Name Description
+ --------------------------------------------------------
+ Encode::Alias Alias defintions to encodings
+ Encode::Encoding Encode Implementation Base Class
+ Encode::Supported List of Supported Encodings
+ Encode::CN Simplified Chinese Encodings
+ Encode::JP Japanese Encodings
+ Encode::KR Korean Encodings
+ Encode::TW Traditional Chinese Encodings
+ --------------------------------------------------------
+
=head1 DESCRIPTION
The C<Encode> module provides the interfaces between Perl's strings
-and the rest of the system. Perl strings are sequences of B<characters>.
+and the rest of the system. Perl strings are sequences of
+B<characters>.
+
+The repertoire of characters that Perl can represent is at least that
+defined by the Unicode Consortium. On most platforms the ordinal
+values of the characters (as returned by C<ord(ch)>) is the "Unicode
+codepoint" for the character (the exceptions are those platforms where
+the legacy encoding is some variant of EBCDIC rather than a super-set
+of ASCII - see L<perlebcdic>).
+
+Traditionally computer data has been moved around in 8-bit chunks
+often called "bytes". These chunks are also known as "octets" in
+networking standards. Perl is widely used to manipulate data of many
+types - not only strings of characters representing human or computer
+languages but also "binary" data being the machines representation of
+numbers, pixels in an image - or just about anything.
+
+When Perl is processing "binary data" the programmer wants Perl to
+process "sequences of bytes". This is not a problem for Perl - as a
+byte has 256 possible values it easily fits in Perl's much larger
+"logical character".
+
+=head2 TERMINOLOGY
-To find more about character encodings, please consult
-L<Encode::Details> . This document focuses on programming references.
+=over 4
-=head1 PERL ENCODING API
+=item *
-=head2 Generic Encoding Interface
+I<character>: a character in the range 0..(2**32-1) (or more).
+(What Perl's strings are made of.)
-=over 4
+=item *
+
+I<byte>: a character in the range 0..255
+(A special case of a Perl character.)
=item *
- $bytes = encode(ENCODING, $string[, CHECK])
+I<octet>: 8 bits of data, with ordinal values 0..255
+(Term for bytes passed to or from a non-Perl context, e.g. disk file.)
-Encodes string from Perl's internal form into I<ENCODING> and returns
-a sequence of octets. For CHECK see L</"Handling Malformed Data">.
+=back
-For example to convert (internally UTF-8 encoded) Unicode data
-to octets:
+The marker [INTERNAL] marks Internal Implementation Details, in
+general meant only for those who think they know what they are doing,
+and such details may change in future releases.
- $octets = encode("utf8", $unicode);
+=head1 PERL ENCODING API
-=item *
+=over 4
- $string = decode(ENCODING, $bytes[, CHECK])
+=item $octets = encode(ENCODING, $string[, CHECK])
+
+Encodes string from Perl's internal form into I<ENCODING> and returns
+a sequence of octets. ENCODING can be either a canonical name or
+alias. For encoding names and aliases, see L</"Defining Aliases">.
+For CHECK see L</"Handling Malformed Data">.
+
+For example to convert (internally UTF-8 encoded) Unicode string to
+iso-8859-1 (also known as Latin1),
+
+ $octets = encode("iso-8859-1", $unicode);
+
+=item $string = decode(ENCODING, $octets[, CHECK])
Decode sequence of octets assumed to be in I<ENCODING> into Perl's
-internal form and returns the resulting string. For CHECK see
+internal form and returns the resulting string. as in encode(),
+ENCODING can be either a canonical name or alias. For encoding names
+and aliases, see L</"Defining Aliases">. For CHECK see
L</"Handling Malformed Data">.
For example to convert ISO-8859-1 data to UTF-8:
- $utf8 = decode("latin1", $latin1);
-
-=item *
+ $utf8 = decode("iso-8859-1", $latin1);
- from_to($string, FROM_ENCODING, TO_ENCODING[, CHECK])
+=item [$length =] from_to($string, FROM_ENCODING, TO_ENCODING [,CHECK])
Convert B<in-place> the data between two encodings. How did the data
in $string originally get to be in FROM_ENCODING? Either using
-encode() or through PerlIO: See L</"Encoding and IO">. For CHECK
-see L</"Handling Malformed Data">.
+encode() or through PerlIO: See L</"Encoding and IO">.
+For encoding names and aliases, see L</"Defining Aliases">.
+For CHECK see L</"Handling Malformed Data">.
For example to convert ISO-8859-1 data to UTF-8:
Note that because the conversion happens in place, the data to be
converted cannot be a string constant, it must be a scalar variable.
-=back
-
-=head2 Handling Malformed Data
-
-If CHECK is not set, C<undef> is returned. If the data is supposed to
-be UTF-8, an optional lexical warning (category utf8) is given. If
-CHECK is true but not a code reference, dies.
-
-It would desirable to have a way to indicate that transform should use
-the encodings "replacement character" - no such mechanism is defined yet.
-
-It is also planned to allow I<CHECK> to be a code reference.
-
-This is not yet implemented as there are design issues with what its
-arguments should be and how it returns its results.
-
-=over 4
-
-=item Scheme 1
-
-Passed remaining fragment of string being processed.
-Modifies it in place to remove bytes/characters it can understand
-and returns a string used to represent them.
-e.g.
-
- sub fixup {
- my $ch = substr($_[0],0,1,'');
- return sprintf("\x{%02X}",ord($ch);
- }
-
-This scheme is close to how underlying C code for Encode works, but gives
-the fixup routine very little context.
-
-=item Scheme 2
-
-Passed original string, and an index into it of the problem area, and
-output string so far. Appends what it will to output string and
-returns new index into original string. For example:
-
- sub fixup {
- # my ($s,$i,$d) = @_;
- my $ch = substr($_[0],$_[1],1);
- $_[2] .= sprintf("\x{%02X}",ord($ch);
- return $_[1]+1;
- }
-
-This scheme gives maximal control to the fixup routine but is more
-complicated to code, and may need internals of Encode to be tweaked to
-keep original string intact.
-
-=item Other Schemes
-
-Hybrids of above.
-
-Multiple return values rather than in-place modifications.
-
-Index into the string could be pos($str) allowing s/\G...//.
+from_to() return the length of the converted string on success, undef
+otherwise.
=back
=head2 UTF-8 / utf8
The Unicode consortium defines the UTF-8 standard as a way of encoding
-the entire Unicode repertiore as sequences of octets. This encoding is
-expected to become very widespread. Perl can use this form internaly
+the entire Unicode repertoire as sequences of octets. This encoding is
+expected to become very widespread. Perl can use this form internally
to represent strings, so conversions to and from this form are
particularly efficient (as octets in memory do not have to change,
just the meta-data that tells Perl how to treat them).
=over 4
-=item *
-
- $bytes = encode_utf8($string);
+=item $octets = encode_utf8($string);
The characters that comprise string are encoded in Perl's superset of UTF-8
and the resulting octets returned as a sequence of bytes. All possible
characters have a UTF-8 representation so this function cannot fail.
-=item *
-
- $string = decode_utf8($bytes [,CHECK]);
+=item $string = decode_utf8($octets [, CHECK]);
-The sequence of octets represented by $bytes is decoded from UTF-8
+The sequence of octets represented by $octets is decoded from UTF-8
into a sequence of logical characters. Not all sequences of octets
form valid UTF-8 encodings, so it is possible for this call to fail.
For CHECK see L</"Handling Malformed Data">.
@with_jp = Encode->encodings("Encode/JP.pm");
-Note in this case you have to say "Encode/JP.pm instead of Encode::JP.
+Note in this case you have to say C<"Encode/JP.pm"> instead of
+C<"Encode::JP">.
-To find which encodings are suppoted by this package in details,
+To find which encodings are supported by this package in details,
see L<Encode::Supported>.
=head2 Defining Aliases
+To add new alias to a given encoding, Use;
+
use Encode;
use Encode::Alias;
- define_alias( newName => ENCODING);
+ define_alias(newName => ENCODING);
-Allows newName to be used as am alias for ENCODING. ENCODING may be
-either the name of an encoding or and encoding object (as above).
+After that, newName can be used as an alias for ENCODING.
+ENCODING may be either the name of an encoding or an
+I<encoding object>
See L<Encode::Alias> on details.
-=head1 Defining Encodings
-
- use Encode qw(define_alias);
- define_encoding( $object, 'canonicalName' [,alias...]);
-
-Causes I<canonicalName> to be associated with I<$object>. The object
-should provide the interface described in L<Encode::Encoding>
-below. If more than two arguments are provided then additional
-arguments are taken as aliases for I<$object> as for C<define_alias>.
-
=head1 Encoding and IO
It is very common to want to do encoding transformations when
See also L<encoding> for how to change the default encoding of the
data in your script.
+=head1 Handling Malformed Data
+
+If I<CHECK> is not set, (en|de)code will put I<substitution character> in
+place of the malformed character. for UCM-based encodings,
+E<lt>subcharE<gt> will be used. For Unicode, \xFFFD is used. If the
+data is supposed to be UTF-8, an optional lexical warning (category
+utf8) is given.
+
+If I<CHECK> is true but not a code reference, dies with an error message.
+
+In future you will be able to use a code reference to a callback
+function for the value of I<CHECK> but its API is still undecided.
+
+=head1 Defining Encodings
+
+To define a new encoding, use:
+
+ use Encode qw(define_alias);
+ define_encoding($object, 'canonicalName' [, alias...]);
+
+I<canonicalName> will be associated with I<$object>. The object
+should provide the interface described in L<Encode::Encoding>
+If more than two arguments are provided then additional
+arguments are taken as aliases for I<$object> as for C<define_alias>.
+
+See L<Encode::Encoding> for more details.
+
=head1 Messing with Perl's Internals
The following API uses parts of Perl's internals in the current
=over 4
-=item * is_utf8(STRING [, CHECK])
+=item is_utf8(STRING [, CHECK])
[INTERNAL] Test whether the UTF-8 flag is turned on in the STRING.
If CHECK is true, also checks the data in STRING for being well-formed
UTF-8. Returns true if successful, false otherwise.
-=item *
-
- _utf8_on(STRING)
+=item _utf8_on(STRING)
[INTERNAL] Turn on the UTF-8 flag in STRING. The data in STRING is
B<not> checked for being well-formed UTF-8. Do not use unless you
state of the UTF-8 flag (so please don't test the return value as
I<not> success or failure), or C<undef> if STRING is not a string.
-=item *
-
- _utf8_off(STRING)
+=item _utf8_off(STRING)
[INTERNAL] Turn off the UTF-8 flag in STRING. Do not use frivolously.
Returns the previous state of the UTF-8 flag (so please don't test the
=head1 SEE ALSO
-L<Encode::Details>,
L<Encode::Encoding>,
L<Encode::Supported>,
L<PerlIO>,
projects / p5sagit/p5-mst-13.2.git / blobdiff