X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=blobdiff_plain;f=ext%2FEncode%2FEncode.pm;h=fde3891b6fe55575c5502e673c06b0c0c22d0f7a;hb=7d85a32c7dc09903975590ebedb298bcbd436874;hp=57677e690bc816de0a70943ceac338d6175c0ece;hpb=2c674647e4cd32dc05dd268c2de5090815265695;p=p5sagit%2Fp5-mst-13.2.git diff --git a/ext/Encode/Encode.pm b/ext/Encode/Encode.pm index 57677e6..fde3891 100644 --- a/ext/Encode/Encode.pm +++ b/ext/Encode/Encode.pm @@ -1,47 +1,402 @@ package Encode; +use strict; -$VERSION = 0.01; +our $VERSION = 0.02; require DynaLoader; require Exporter; -@ISA = qw(Exporter DynaLoader); +our @ISA = qw(Exporter DynaLoader); -@EXPORT_OK = +# Public, encouraged API is exported by default +our @EXPORT = qw ( + encode + decode + encode_utf8 + decode_utf8 + find_encoding + encodings +); + +our @EXPORT_OK = qw( - bytes_to_utf8 - utf8_to_bytes - chars_to_utf8 - utf8_to_chars - utf8_to_chars_check - bytes_to_chars - chars_to_bytes + define_encoding + define_alias from_to is_utf8 - on_utf8 - off_utf8 - utf_to_utf + is_8bit + is_16bit + utf8_upgrade + utf8_downgrade + _utf8_on + _utf8_off ); bootstrap Encode (); -=pod +# Documentation moved after __END__ for speed - NI-S + +use Carp; + +# Make a %encoding package variable to allow a certain amount of cheating +our %encoding; +my @alias; # ordered matching list +my %alias; # cached known aliases + +sub encodings +{ + my ($class) = @_; + return keys %encoding; +} + +sub findAlias +{ + my $class = shift; + local $_ = shift; + unless (exists $alias{$_}) + { + for (my $i=0; $i < @alias; $i += 2) + { + my $alias = $alias[$i]; + my $val = $alias[$i+1]; + my $new; + + if (ref($alias) eq 'Regexp' && $_ =~ $alias) + { + $new = eval $val; + } + elsif (ref($alias) eq 'CODE') + { + $new = &{$alias}($val) + } + elsif (lc($_) eq lc($alias)) + { + $new = $val; + } + if (defined($new)) + { + next if $new eq $_; # avoid (direct) recursion on bugs + my $enc = (ref($new)) ? $new : find_encoding($new); + if ($enc) + { + $alias{$_} = $enc; + last; + } + } + } + } + return $alias{$_}; +} + +sub define_alias +{ + while (@_) + { + my ($alias,$name) = splice(@_,0,2); + push(@alias, $alias => $name); + } +} + +# Allow variants of iso-8859-1 etc. +define_alias( qr/^iso[-_]?(\d+)[-_](\d+)$/i => '"iso-$1-$2"' ); + +# Allow latin-1 style names as well + # 0 1 2 3 4 5 6 7 8 9 10 +my @latin2iso_num = ( 0, 1, 2, 3, 4, 9, 10, 13, 14, 15, 16 ); +define_alias( qr/^latin[-_]?(\d+)$/i => '"iso-8859-$latin2iso_num[$1]"' ); + +# Common names for non-latin prefered MIME names +define_alias( 'ascii' => 'US-ascii', + 'cyrillic' => 'iso-8859-5', + 'arabic' => 'iso-8859-6', + 'greek' => 'iso-8859-7', + 'hebrew' => 'iso-8859-8'); + +define_alias( 'ibm-1047' => 'cp1047'); + +# Map white space and _ to '-' +define_alias( qr/^(\S+)[\s_]+(.*)$/i => '"$1-$2"' ); + +sub define_encoding +{ + my $obj = shift; + my $name = shift; + $encoding{$name} = $obj; + my $lc = lc($name); + define_alias($lc => $obj) unless $lc eq $name; + while (@_) + { + my $alias = shift; + define_alias($alias,$obj); + } + return $obj; +} + +sub getEncoding +{ + my ($class,$name) = @_; + my $enc; + if (exists $encoding{$name}) + { + return $encoding{$name}; + } + else + { + return $class->findAlias($name); + } +} + +sub find_encoding +{ + my ($name) = @_; + return __PACKAGE__->getEncoding($name); +} + +sub encode +{ + my ($name,$string,$check) = @_; + my $enc = find_encoding($name); + croak("Unknown encoding '$name'") unless defined $enc; + my $octets = $enc->encode($string,$check); + return undef if ($check && length($string)); + return $octets; +} + +sub decode +{ + my ($name,$octets,$check) = @_; + my $enc = find_encoding($name); + croak("Unknown encoding '$name'") unless defined $enc; + my $string = $enc->decode($octets,$check); + return undef if ($check && length($octets)); + return $string; +} + +sub from_to +{ + my ($string,$from,$to,$check) = @_; + my $f = find_encoding($from); + croak("Unknown encoding '$from'") unless defined $f; + my $t = find_encoding($to); + croak("Unknown encoding '$to'") unless defined $t; + my $uni = $f->decode($string,$check); + return undef if ($check && length($string)); + $string = $t->encode($uni,$check); + return undef if ($check && length($uni)); + return length($_[0] = $string); +} + +sub encode_utf8 +{ + my ($str) = @_; + utf8::encode($str); + return $str; +} + +sub decode_utf8 +{ + my ($str) = @_; + return undef unless utf8::decode($str); + return $str; +} + +package Encode::Encoding; +# Base class for classes which implement encodings + +sub Define +{ + my $obj = shift; + my $canonical = shift; + $obj = bless { Name => $canonical },$obj unless ref $obj; + # warn "$canonical => $obj\n"; + Encode::define_encoding($obj, $canonical, @_); +} + +sub name { shift->{'Name'} } + +# Temporary legacy methods +sub toUnicode { shift->decode(@_) } +sub fromUnicode { shift->encode(@_) } + +sub new_sequence { return $_[0] } + +package Encode::XS; +use base 'Encode::Encoding'; + +package Encode::Internal; +use base 'Encode::Encoding'; + +# Dummy package that provides the encode interface but leaves data +# as UTF-X encoded. It is here so that from_to() works. + +__PACKAGE__->Define('Internal'); + +Encode::define_alias( 'Unicode' => 'Internal' ) if ord('A') == 65; + +sub decode +{ + my ($obj,$str,$chk) = @_; + utf8::upgrade($str); + $_[1] = '' if $chk; + return $str; +} + +*encode = \&decode; + +package Encoding::Unicode; +use base 'Encode::Encoding'; + +__PACKAGE__->Define('Unicode') unless ord('A') == 65; + +sub decode +{ + my ($obj,$str,$chk) = @_; + my $res = ''; + for (my $i = 0; $i < length($str); $i++) + { + $res .= chr(utf8::unicode_to_native(ord(substr($str,$i,1)))); + } + $_[1] = '' if $chk; + return $res; +} + +sub encode +{ + my ($obj,$str,$chk) = @_; + my $res = ''; + for (my $i = 0; $i < length($str); $i++) + { + $res .= chr(utf8::native_to_unicode(ord(substr($str,$i,1)))); + } + $_[1] = '' if $chk; + return $res; +} + + +package Encode::utf8; +use base 'Encode::Encoding'; +# package to allow long-hand +# $octets = encode( utf8 => $string ); +# + +__PACKAGE__->Define(qw(UTF-8 utf8)); + +sub decode +{ + my ($obj,$octets,$chk) = @_; + my $str = Encode::decode_utf8($octets); + if (defined $str) + { + $_[1] = '' if $chk; + return $str; + } + return undef; +} + +sub encode +{ + my ($obj,$string,$chk) = @_; + my $octets = Encode::encode_utf8($string); + $_[1] = '' if $chk; + return $octets; +} + +package Encode::iso10646_1; +use base 'Encode::Encoding'; +# Encoding is 16-bit network order Unicode (no surogates) +# Used for X font encodings + +__PACKAGE__->Define(qw(UCS-2 iso-10646-1)); + +sub decode +{ + my ($obj,$str,$chk) = @_; + my $uni = ''; + while (length($str)) + { + my $code = unpack('n',substr($str,0,2,'')) & 0xffff; + $uni .= chr($code); + } + $_[1] = $str if $chk; + utf8::upgrade($uni); + return $uni; +} + +sub encode +{ + my ($obj,$uni,$chk) = @_; + my $str = ''; + while (length($uni)) + { + my $ch = substr($uni,0,1,''); + my $x = ord($ch); + unless ($x < 32768) + { + last if ($chk); + $x = 0; + } + $str .= pack('n',$x); + } + $_[1] = $uni if $chk; + return $str; +} + +# switch back to Encode package in case we ever add AutoLoader +package Encode; + +1; + +__END__ =head1 NAME Encode - character encodings +=head1 SYNOPSIS + + use Encode; + +=head1 DESCRIPTION + +The C module provides the interfaces between perl's strings +and the rest of the system. Perl strings are sequences of B. + +The repertoire of characters that Perl can represent is at least that +defined by the Unicode Consortium. On most platforms the ordinal values +of the characters (as returned by C) is the "Unicode codepoint" for +the character (the exceptions are those platforms where the legacy +encoding is some variant of EBCDIC rather than a super-set of ASCII +- see L). + +Traditionaly computer data has been moved around in 8-bit chunks +often called "bytes". These chunks are also known as "octets" in +networking standards. Perl is widely used to manipulate data of +many types - not only strings of characters representing human or +computer languages but also "binary" data being the machines representation +of numbers, pixels in an image - or just about anything. + +When perl is processing "binary data" the programmer wants perl to process +"sequences of bytes". This is not a problem for perl - as a byte has 256 +possible values it easily fits in perl's much larger "logical character". + =head2 TERMINOLOGY -=over +=over 4 =item * -I: a character in the range 0..maxint (at least 2**32-1) +I: a character in the range 0..(2**32-1) (or more). +(What perl's strings are made of.) =item * I: a character in the range 0..255 +(A special case of a perl character.) + +=item * + +I: 8 bits of data, with ordinal values 0..255 +(Term for bytes passed to or from a non-perl context, e.g. disk file.) =back @@ -49,167 +404,369 @@ The marker [INTERNAL] marks Internal Implementation Details, in general meant only for those who think they know what they are doing, and such details may change in future releases. -=head2 bytes +=head1 ENCODINGS + +=head2 Characteristics of an Encoding + +An encoding has a "repertoire" of characters that it can represent, +and for each representable character there is at least one sequence of +octets that represents it. + +=head2 Types of Encodings + +Encodings can be divided into the following types: =over 4 -=item * +=item * Fixed length 8-bit (or less) encodings. - bytes_to_utf8(STRING [, FROM]) +Each character is a single octet so may have a repertoire of up to +256 characters. ASCII and iso-8859-* are typical examples. -The bytes in STRING are recoded in-place into UTF-8. If no FROM is -specified the bytes are expected to be encoded in US-ASCII or ISO -8859-1 (Latin 1). Returns the new size of STRING, or C if -there's a failure. +=item * Fixed length 16-bit encodings -[INTERNAL] Also the UTF-8 flag of STRING is turned on. +Each character is two octets so may have a repertoire of up to +65,536 characters. Unicode's UCS-2 is an example. Also used for +encodings for East Asian languages. -=item * +=item * Fixed length 32-bit encodings. - utf8_to_bytes(STRING [, TO [, CHECK]]) +Not really very "encoded" encodings. The Unicode code points +are just represented as 4-octet integers. None the less because +different architectures use different representations of integers +(so called "endian") there at least two disctinct encodings. -The UTF-8 in STRING is decoded in-place into bytes. If no TO encoding -is specified the bytes are expected to be encoded in US-ASCII or ISO -8859-1 (Latin 1). Returns the new size of STRING, or C if -there's a failure. +=item * Multi-byte encodings -What if there are characters > 255? What if the UTF-8 in STRING is -malformed? See L. +The number of octets needed to represent a character varies. +UTF-8 is a particularly complex but regular case of a multi-byte +encoding. Several East Asian countries use a multi-byte encoding +where 1-octet is used to cover western roman characters and Asian +characters get 2-octets. +(UTF-16 is strictly a multi-byte encoding taking either 2 or 4 octets +to represent a Unicode code point.) -[INTERNAL] The UTF-8 flag of STRING is not checked. +=item * "Escape" encodings. + +These encodings embed "escape sequences" into the octet sequence +which describe how the following octets are to be interpreted. +The iso-2022-* family is typical. Following the escape sequence +octets are encoded by an "embedded" encoding (which will be one +of the above types) until another escape sequence switches to +a different "embedded" encoding. + +These schemes are very flexible and can handle mixed languages but are +very complex to process (and have state). +No escape encodings are implemented for perl yet. =back -=head2 chars +=head2 Specifying Encodings + +Encodings can be specified to the API described below in two ways: =over 4 -=item * +=item 1. By name + +Encoding names are strings with characters taken from a restricted repertoire. +See L. - chars_to_utf8(STRING) +=item 2. As an object -The chars in STRING are encoded in-place into UTF-8. Returns the new -size of STRING, or C if there's a failure. +Encoding objects are returned by C. + +=back -No assumptions are made on the encoding of the chars. If you want to -assume that the chars are Unicode and to trap illegal Unicode -characters, you must use C. +=head2 Encoding Names -[INTERNAL] Also the UTF-8 flag of STRING is turned on. +Encoding names are case insensitive. White space in names is ignored. +In addition an encoding may have aliases. Each encoding has one "canonical" name. +The "canonical" name is chosen from the names of the encoding by picking +the first in the following sequence: =over 4 -=item * +=item * The MIME name as defined in IETF RFC-XXXX. + +=item * The name in the IANA registry. - utf8_to_chars(STRING) +=item * The name used by the the organization that defined it. -The UTF-8 in STRING is decoded in-place into chars. Returns the new -size of STRING, or C if there's a failure. +=back -If the UTF-8 in STRING is malformed C is returned, and also an -optional lexical warning (category utf8) is given. +Because of all the alias issues, and because in the general case +encodings have state C uses the encoding object internally +once an operation is in progress. -[INTERNAL] The UTF-8 flag of STRING is not checked. +=head1 PERL ENCODING API + +=head2 Generic Encoding Interface + +=over 4 =item * - utf8_to_chars_check(STRING [, CHECK]) + $bytes = encode(ENCODING, $string[, CHECK]) -(Note that special naming of this interface since a two-argument -utf8_to_chars() has different semantics.) +Encodes string from perl's internal form into I and returns a +sequence of octets. +See L. -The UTF-8 in STRING is decoded in-place into chars. Returns the new -size of STRING, or C if there is a failure. +=item * -If the UTF-8 in STRING is malformed? See L. + $string = decode(ENCODING, $bytes[, CHECK]) -[INTERNAL] The UTF-8 flag of STRING is not checked. +Decode sequence of octets assumed to be in I into perls internal +form and returns the resuting string. +See L. =back -=head2 chars With Encoding +=head2 Handling Malformed Data -=over 4 +If CHECK is not set, C is returned. If the data is supposed to +be UTF-8, an optional lexical warning (category utf8) is given. +If CHECK is true but not a code reference, dies. -=item * +It would desirable to have a way to indicate that transform should use the +encodings "replacement character" - no such mechanism is defined yet. - chars_to_utf8(STRING, FROM [, CHECK]) +It is also planned to allow I to be a code reference. -The chars in STRING encoded in FROM are recoded in-place into UTF-8. -Returns the new size of STRING, or C if there's a failure. +This is not yet implemented as there are design issues with what its arguments +should be and how it returns its results. -No assumptions are made on the encoding of the chars. If you want to -assume that the chars are Unicode and to trap illegal Unicode -characters, you must use C. +=over 4 -[INTERNAL] Also the UTF-8 flag of STRING is turned on. +=item Scheme 1 -=item * +Passed remaining fragment of string being processed. +Modifies it in place to remove bytes/characters it can understand +and returns a string used to represent them. +e.g. - utf8_to_chars(STRING, TO [, CHECK]) + sub fixup { + my $ch = substr($_[0],0,1,''); + return sprintf("\x{%02X}",ord($ch); + } -The UTF-8 in STRING is decoded in-place into chars encoded in TO. -Returns the new size of STRING, or C if there's a failure. +This scheme is close to how underlying C code for Encode works, but gives +the fixup routine very little context. -If the UTF-8 in STRING is malformed? See L. +=item Scheme 2 -[INTERNAL] The UTF-8 flag of STRING is not checked. +Passed original string, and an index into it of the problem area, +and output string so far. +Appends what it will to output string and returns new index into +original string. +e.g. -=item * + sub fixup { + # my ($s,$i,$d) = @_; + my $ch = substr($_[0],$_[1],1); + $_[2] .= sprintf("\x{%02X}",ord($ch); + return $_[1]+1; + } - bytes_to_chars(STRING, FROM [, CHECK]) +This scheme gives maximal control to the fixup routine but is more complicated +to code, and may need internals of Encode to be tweaked to keep original +string intact. -The bytes in STRING encoded in FROM are recoded in-place into chars. -Returns the new size of STRING in bytes, or C if there's a -failure. +=item Other Schemes -If the mapping is impossible? See L. +Hybrids of above. -=item * +Multiple return values rather than in-place modifications. + +Index into the string could be pos($str) allowing s/\G...//. + +=back - chars_to_bytes(STRING, TO [, CHECK]) +=head2 UTF-8 / utf8 -The chars in STRING are recoded in-place to bytes encoded in TO. -Returns the new size of STRING in bytes, or C if there's a -failure. +The Unicode consortium defines the UTF-8 standard as a way of encoding +the entire Unicode repertiore as sequences of octets. This encoding +is expected to become very widespread. Perl can use this form internaly +to represent strings, so conversions to and from this form are particularly +efficient (as octets in memory do not have to change, just the meta-data +that tells perl how to treat them). -If the mapping is impossible? See L. +=over 4 =item * - from_to(STRING, FROM, TO [, CHECK]) + $bytes = encode_utf8($string); -The chars in STRING encoded in FROM are recoded in-place into TO. -Returns the new size of STRING, or C if there's a failure. +The characters that comprise string are encoded in perl's superset of UTF-8 +and the resulting octets returned as a sequence of bytes. All possible +characters have a UTF-8 representation so this function cannot fail. -If mapping between the encodings is impossible? +=item * + + $string = decode_utf8($bytes [,CHECK]); + +The sequence of octets represented by $bytes is decoded from UTF-8 into +a sequence of logical characters. Not all sequences of octets form valid +UTF-8 encodings, so it is possible for this call to fail. See L. -[INTERNAL] If TO is UTF-8, also the UTF-8 flag of STRING is turned on. +=back + +=head2 Other Encodings of Unicode + +UTF-16 is similar to UCS-2, 16 bit or 2-byte chunks. +UCS-2 can only represent 0..0xFFFF, while UTF-16 has a "surogate pair" +scheme which allows it to cover the whole Unicode range. + +Encode implements big-endian UCS-2 aliased to "iso-10646-1" as that +happens to be the name used by that representation when used with X11 fonts. + +UTF-32 or UCS-4 is 32-bit or 4-byte chunks. Perl's logical characters +can be considered as being in this form without encoding. An encoding +to transfer strings in this form (e.g. to write them to a file) would need to + + pack('L',map(chr($_),split(//,$string))); # native + or + pack('V',map(chr($_),split(//,$string))); # little-endian + or + pack('N',map(chr($_),split(//,$string))); # big-endian + +depending on the endian required. + +No UTF-32 encodings are implemented yet. + +Both UCS-2 and UCS-4 style encodings can have "byte order marks" by representing +the code point 0xFFFE as the very first thing in a file. + +=head2 Listing available encodings + + use Encode qw(encodings); + @list = encodings(); + +Returns a list of the canonical names of the available encodings. + +=head2 Defining Aliases + + use Encode qw(define_alias); + define_alias( newName => ENCODING); + +Allows newName to be used as am alias for ENCODING. ENCODING may be either the +name of an encoding or and encoding object (as above). + +Currently I can be specified in the following ways: + +=over 4 + +=item As a simple string. + +=item As a qr// compiled regular expression, e.g.: + + define_alias( qr/^iso8859-(\d+)$/i => '"iso-8859-$1"' ); + +In this case if I is not a reference it is C-ed to allow +C<$1> etc. to be subsituted. +The example is one way to names as used in X11 font names to alias the MIME names for the +iso-8859-* family. + +=item As a code reference, e.g.: + + define_alias( sub { return /^iso8859-(\d+)$/i ? "iso-8859-$1" : undef } , ''); + +In this case C<$_> will be set to the name that is being looked up and +I is passed to the sub as its first argument. +The example is another way to names as used in X11 font names to alias the MIME names for +the iso-8859-* family. =back -=head2 Testing For UTF-8 +=head2 Defining Encodings + + use Encode qw(define_alias); + define_encoding( $object, 'canonicalName' [,alias...]); + +Causes I to be associated with I<$object>. +The object should provide the interface described in L below. +If more than two arguments are provided then additional arguments are taken +as aliases for I<$object> as for C. + +=head1 Encoding and IO + +It is very common to want to do encoding transformations when +reading or writing files, network connections, pipes etc. +If perl is configured to use the new 'perlio' IO system then +C provides a "layer" (See L) which can transform +data as it is read or written. + + open(my $ilyad,'>:encoding(iso-8859-7)','ilyad.greek'); + print $ilyad @epic; + +In addition the new IO system can also be configured to read/write +UTF-8 encoded characters (as noted above this is efficient): + + open(my $fh,'>:utf8','anything'); + print $fh "Any \x{0021} string \N{SMILEY FACE}\n"; + +Either of the above forms of "layer" specifications can be made the default +for a lexical scope with the C pragma. See L. + +Once a handle is open is layers can be altered using C. + +Without any such configuration, or if perl itself is built using +system's own IO, then write operations assume that file handle accepts +only I and will C if a character larger than 255 is +written to the handle. When reading, each octet from the handle +becomes a byte-in-a-character. Note that this default is the same +behaviour as bytes-only languages (including perl before v5.6) would have, +and is sufficient to handle native 8-bit encodings e.g. iso-8859-1, +EBCDIC etc. and any legacy mechanisms for handling other encodings +and binary data. + +In other cases it is the programs responsibility +to transform characters into bytes using the API above before +doing writes, and to transform the bytes read from a handle into characters +before doing "character operations" (e.g. C, C, ...). + +=head1 Encoding How to ... + +To do: =over 4 -=item * +=item * IO with mixed content (faking iso-2020-*) - is_utf8(STRING [, CHECK]) +=item * MIME's Content-Length: -[INTERNAL] Test whether the UTF-8 flag is turned on in the STRING. -If CHECK is true, also checks the data in STRING for being -well-formed UTF-8. Returns true if successful, false otherwise. +=item * UTF-8 strings in binary data. + +=item * perl/Encode wrappers on non-Unicode XS modules. =back -=head2 Toggling UTF-8-ness +=head1 Messing with Perl's Internals + +The following API uses parts of perl's internals in the current implementation. +As such they are efficient, but may change. =over 4 +=item * is_utf8(STRING [, CHECK]) + +[INTERNAL] Test whether the UTF-8 flag is turned on in the STRING. +If CHECK is true, also checks the data in STRING for being +well-formed UTF-8. Returns true if successful, false otherwise. + +=item * valid_utf8(STRING) + +[INTERNAL] Test whether STRING is in a consistent state. +Will return true if string is held as bytes, or is well-formed UTF-8 +and has the UTF-8 flag on. +Main reason for this routine is to allow perl's testsuite to check +that operations have left strings in a consistent state. + =item * - on_utf8(STRING) + _utf8_on(STRING) [INTERNAL] Turn on the UTF-8 flag in STRING. The data in STRING is B checked for being well-formed UTF-8. Do not use unless you @@ -219,7 +776,7 @@ I success or failure), or C if STRING is not a string. =item * - off_utf8(STRING) + _utf8_off(STRING) [INTERNAL] Turn off the UTF-8 flag in STRING. Do not use frivolously. Returns the previous state of the UTF-8 flag (so please don't test the @@ -228,102 +785,162 @@ not a string. =back -=head2 UTF-16 and UTF-32 Encodings +=head1 IMPLEMENTATION CLASSES + +As mentioned above encodings are (in the current implementation at least) +defined by objects. The mapping of encoding name to object is via the +C<%encodings> hash. + +The values of the hash can currently be either strings or objects. +The string form may go away in the future. The string form occurs +when C has scanned C<@INC> for loadable encodings but has +not actually loaded the encoding in question. This is because the +current "loading" process is all perl and a bit slow. + +Once an encoding is loaded then value of the hash is object which implements +the encoding. The object should provide the following interface: =over 4 -=item * +=item -Ename + +Should return the string representing the canonical name of the encoding. + +=item -Enew_sequence - utf_to_utf(STRING, FROM, TO [, CHECK]) +This is a placeholder for encodings with state. It should return an object +which implements this interface, all current implementations return the +original object. -The data in STRING is converted from Unicode Transfer Encoding FROM to -Unicode Transfer Encoding TO. Both FROM and TO may be any of the -following tags (case-insensitive, with or without 'utf' or 'utf-' prefix): +=item -Eencode($string,$check) - tag meaning +Should return the octet sequence representing I<$string>. If I<$check> is true +it should modify I<$string> in place to remove the converted part (i.e. +the whole string unless there is an error). +If an error occurs it should return the octet sequence for the +fragment of string that has been converted, and modify $string in-place +to remove the converted part leaving it starting with the problem fragment. - '7' UTF-7 - '8' UTF-8 - '16be' UTF-16 big-endian - '16le' UTF-16 little-endian - '16' UTF-16 native-endian - '32be' UTF-32 big-endian - '32le' UTF-32 little-endian - '32' UTF-32 native-endian +If check is is false then C should make a "best effort" to convert +the string - for example by using a replacement character. -UTF-16 is also known as UCS-2, 16 bit or 2-byte chunks, and UTF-32 as -UCS-4, 32-bit or 4-byte chunks. Returns the new size of STRING, or -C is there's a failure. +=item -Edecode($octets,$check) -If FROM is UTF-8 and the UTF-8 in STRING is malformed? See -L. +Should return the string that I<$octets> represents. If I<$check> is true +it should modify I<$octets> in place to remove the converted part (i.e. +the whole sequence unless there is an error). +If an error occurs it should return the fragment of string +that has been converted, and modify $octets in-place to remove the converted part +leaving it starting with the problem fragment. -[INTERNAL] Even if CHECK is true and FROM is UTF-8, the UTF-8 flag of -STRING is not checked. If TO is UTF-8, also the UTF-8 flag of STRING is -turned on. Identical FROM and TO are fine. +If check is is false then C should make a "best effort" to convert +the string - for example by using Unicode's "\x{FFFD}" as a replacement character. =back -=head2 Handling Malformed Data +It should be noted that the check behaviour is different from the outer +public API. The logic is that the "unchecked" case is useful when +encoding is part of a stream which may be reporting errors (e.g. STDERR). +In such cases it is desirable to get everything through somehow without +causing additional errors which obscure the original one. Also the encoding +is best placed to know what the correct replacement character is, so if that +is the desired behaviour then letting low level code do it is the most efficient. -If CHECK is not set, C is returned. If the data is supposed to -be UTF-8, an optional lexical warning (category utf8) is given. If -CHECK is true but not a code reference, dies. If CHECK is a code -reference, it is called with the arguments +In contrast if check is true, the scheme above allows the encoding to do as +much as it can and tell layer above how much that was. What is lacking +at present is a mechanism to report what went wrong. The most likely interface +will be an additional method call to the object, or perhaps +(to avoid forcing per-stream objects on otherwise stateless encodings) +and additional parameter. - (MALFORMED_STRING, STRING_FROM_SO_FAR, STRING_TO_SO_FAR) +It is also highly desirable that encoding classes inherit from C +as a base class. This allows that class to define additional behaviour for +all encoding objects. For example built in Unicode, UCS-2 and UTF-8 classes +use : -Two return values are expected from the call: the string to be used in -the result string in place of the malformed section, and the length of -the malformed section in bytes. + package Encode::MyEncoding; + use base qw(Encode::Encoding); -=cut + __PACKAGE__->Define(qw(myCanonical myAlias)); -sub bytes_to_utf8 { - &_bytes_to_utf8; -} +To create an object with bless {Name => ...},$class, and call define_encoding. +They inherit their C method from C. -sub utf8_to_bytes { - &_utf8_to_bytes; -} +=head2 Compiled Encodings -sub chars_to_utf8 { - &C_to_utf8; -} +F provides a class C which provides the interface described +above. It calls a generic octet-sequence to octet-sequence "engine" that is +driven by tables (defined in F). The same engine is used for both +encode and decode. C's C forces perl's characters to their UTF-8 form +and then treats them as just another multibyte encoding. C's C transforms +the sequence and then turns the UTF-8-ness flag as that is the form that the tables +are defined to produce. For details of the engine see the comments in F. -sub utf8_to_chars { - &_utf8_to_chars; -} +The tables are produced by the perl script F (the name needs to change so +we can eventually install it somewhere). F can currently read two formats: -sub utf8_to_chars_check { - &_utf8_to_chars_check; -} +=over 4 -sub bytes_to_chars { - &_bytes_to_chars; -} +=item *.enc -sub chars_to_bytes { - &_chars_to_bytes; -} +This is a coined format used by Tcl. It is documented in Encode/EncodeFormat.pod. -sub from_to { - &_from_to; -} +=item *.ucm -sub is_utf8 { - &_is_utf8; -} +This is the semi-standard format used by IBM's ICU package. -sub on_utf8 { - &_on_utf8; -} +=back + +F can write the following forms: + +=over 4 + +=item *.ucm + +See above - the F files provided with the distribution have +been created from the original Tcl .enc files using this approach. + +=item *.c + +Produces tables as C data structures - this is used to build in encodings +into F/F. + +=item *.xs + +In theory this allows encodings to be stand-alone loadable perl extensions. +The process has not yet been tested. The plan is to use this approach +for large East Asian encodings. + +=back + +The set of encodings built-in to F/F is determined by +F. The current set is as follows: + +=over 4 + +=item ascii and iso-8859-* + +That is all the common 8-bit "western" encodings. + +=item IBM-1047 and two other variants of EBCDIC. + +These are the same variants that are supported by EBCDIC perl as "native" encodings. +They are included to prove "reversibility" of some constructs in EBCDIC perl. + +=item symbol and dingbats as used by Tk on X11. + +(The reason Encode got started was to support perl/Tk.) + +=back + +That set is rather ad. hoc. and has been driven by the needs of the tests rather +than the needs of typical applications. It is likely to be rationalized. + +=head1 SEE ALSO + +L, L, L + +=cut -sub off_utf8 { - &_off_utf8; -} -sub utf_to_utf { - &_utf_to_utf; -}