8 our @ISA = qw(Exporter DynaLoader);
10 # Public, encouraged API is exported by default
36 # Documentation moved after __END__ for speed - NI-S
40 # Make a %encoding package variable to allow a certain amount of cheating
42 my @alias; # ordered matching list
43 my %alias; # cached known aliases
45 # 0 1 2 3 4 5 6 7 8 9 10
46 our @latin2iso_num = ( 0, 1, 2, 3, 4, 9, 10, 13, 14, 15, 16 );
65 sort { $a->[1] cmp $b->[1] }
67 grep { $_ ne 'Internal' }
75 # print "# findAlias $_\n";
76 unless (exists $alias{$_})
78 for (my $i=0; $i < @alias; $i += 2)
80 my $alias = $alias[$i];
81 my $val = $alias[$i+1];
83 if (ref($alias) eq 'Regexp' && $_ =~ $alias)
87 elsif (ref($alias) eq 'CODE')
89 $new = &{$alias}($val)
91 elsif (lc($_) eq lc($alias))
97 next if $new eq $_; # avoid (direct) recursion on bugs
98 my $enc = (ref($new)) ? $new : find_encoding($new);
114 my ($alias,$name) = splice(@_,0,2);
115 push(@alias, $alias => $name);
119 # Allow variants of iso-8859-1 etc.
120 define_alias( qr/^iso[-_]?(\d+)[-_](\d+)$/i => '"iso-$1-$2"' );
122 # At least HP-UX has these.
123 define_alias( qr/^iso8859(\d+)$/i => '"iso-8859-$1"' );
126 define_alias( qr/^(?:hp-)?(arabic|greek|hebrew|kana|roman|thai|turkish)8$/i => '"${1}8"' );
128 # The Official name of ASCII.
129 define_alias( qr/^ANSI[-_]?X3\.4[-_]?1968$/i => '"ascii"' );
131 # This is a font issue, not an encoding issue.
132 # (The currency symbol of the Latin 1 upper half
133 # has been redefined as the euro symbol.)
134 define_alias( qr/^(.+)\@euro$/i => '"$1"' );
136 # Allow latin-1 style names as well
137 define_alias( qr/^(?:iso[-_]?)?latin[-_]?(\d+)$/i => '"iso-8859-$latin2iso_num[$1]"' );
139 # Allow winlatin1 style names as well
140 define_alias( qr/^win(latin[12]|cyrillic|baltic|greek|turkish|hebrew|arabic|baltic|vietnamese)$/i => '"cp$winlatin2cp{\u$1}"' );
142 # Common names for non-latin prefered MIME names
143 define_alias( 'ascii' => 'US-ascii',
144 'cyrillic' => 'iso-8859-5',
145 'arabic' => 'iso-8859-6',
146 'greek' => 'iso-8859-7',
147 'hebrew' => 'iso-8859-8',
148 'thai' => 'iso-8859-11',
149 'tis620' => 'iso-8859-11',
152 # At least AIX has IBM-NNN (surprisingly...) instead of cpNNN.
153 # And Microsoft has their own naming (again, surprisingly).
154 define_alias( qr/^(?:ibm|ms)[-_]?(\d\d\d\d?)$/i => '"cp$1"');
156 # Sometimes seen with a leading zero.
157 define_alias( qr/^cp037$/i => '"cp37"');
160 define_alias( qr/^macRomanian$/i => '"macRumanian"');
162 # Standardize on the dashed versions.
163 define_alias( qr/^utf8$/i => 'utf-8' );
164 define_alias( qr/^koi8r$/i => 'koi8-r' );
165 define_alias( qr/^koi8u$/i => 'koi8-u' );
167 # Seen in some Linuxes.
168 define_alias( qr/^ujis$/i => 'euc-jp' );
170 # TODO: HP-UX '8' encodings arabic8 greek8 hebrew8 kana8 thai8 turkish8
171 # TODO: HP-UX '15' encodings japanese15 korean15 roi15
172 # TODO: Cyrillic encoding ISO-IR-111 (useful?)
173 # TODO: Chinese encodings GB18030 EUC-TW HZ
174 # TODO: Armenian encoding ARMSCII-8
175 # TODO: Hebrew encoding ISO-8859-8-1
176 # TODO: Thai encoding TCVN
177 # TODO: Korean encoding Johab
178 # TODO: Vietnamese encodings VPS
179 # TODO: Japanese encoding JIS (not the same as SJIS)
180 # TODO: Mac Asian+African encodings: Arabic Armenian Bengali Burmese
181 # ChineseSimp ChineseTrad Devanagari Ethiopic ExtArabic
182 # Farsi Georgian Gujarati Gurmukhi Hebrew Japanese
183 # Kannada Khmer Korean Laotian Malayalam Mongolian
184 # Oriya Sinhalese Symbol Tamil Telugu Tibetan Vietnamese
186 # Map white space and _ to '-'
187 define_alias( qr/^(\S+)[\s_]+(.*)$/i => '"$1-$2"' );
193 $encoding{$name} = $obj;
195 define_alias($lc => $obj) unless $lc eq $name;
199 define_alias($alias,$obj);
206 my ($class,$name) = @_;
208 if (ref($name) && $name->can('new_sequence'))
213 if (exists $encoding{$name})
215 return $encoding{$name};
217 if (exists $encoding{$lc})
219 return $encoding{$lc};
222 my $oc = $class->findAlias($name);
223 return $oc if defined $oc;
224 return $class->findAlias($lc) if $lc ne $name;
232 return __PACKAGE__->getEncoding($name);
237 my ($name,$string,$check) = @_;
238 my $enc = find_encoding($name);
239 croak("Unknown encoding '$name'") unless defined $enc;
240 my $octets = $enc->encode($string,$check);
241 return undef if ($check && length($string));
247 my ($name,$octets,$check) = @_;
248 my $enc = find_encoding($name);
249 croak("Unknown encoding '$name'") unless defined $enc;
250 my $string = $enc->decode($octets,$check);
251 $_[1] = $octets if $check;
257 my ($string,$from,$to,$check) = @_;
258 my $f = find_encoding($from);
259 croak("Unknown encoding '$from'") unless defined $f;
260 my $t = find_encoding($to);
261 croak("Unknown encoding '$to'") unless defined $t;
262 my $uni = $f->decode($string,$check);
263 return undef if ($check && length($string));
264 $string = $t->encode($uni,$check);
265 return undef if ($check && length($uni));
266 return length($_[0] = $string);
279 return undef unless utf8::decode($str);
283 require Encode::Encoding;
285 require Encode::Internal;
286 require Encode::Unicode;
287 require Encode::utf8;
288 require Encode::iso10646_1;
289 require Encode::ucs2_le;
297 Encode - character encodings
305 The C<Encode> module provides the interfaces between Perl's strings
306 and the rest of the system. Perl strings are sequences of B<characters>.
308 The repertoire of characters that Perl can represent is at least that
309 defined by the Unicode Consortium. On most platforms the ordinal
310 values of the characters (as returned by C<ord(ch)>) is the "Unicode
311 codepoint" for the character (the exceptions are those platforms where
312 the legacy encoding is some variant of EBCDIC rather than a super-set
313 of ASCII - see L<perlebcdic>).
315 Traditionaly computer data has been moved around in 8-bit chunks
316 often called "bytes". These chunks are also known as "octets" in
317 networking standards. Perl is widely used to manipulate data of
318 many types - not only strings of characters representing human or
319 computer languages but also "binary" data being the machines representation
320 of numbers, pixels in an image - or just about anything.
322 When Perl is processing "binary data" the programmer wants Perl to process
323 "sequences of bytes". This is not a problem for Perl - as a byte has 256
324 possible values it easily fits in Perl's much larger "logical character".
332 I<character>: a character in the range 0..(2**32-1) (or more).
333 (What Perl's strings are made of.)
337 I<byte>: a character in the range 0..255
338 (A special case of a Perl character.)
342 I<octet>: 8 bits of data, with ordinal values 0..255
343 (Term for bytes passed to or from a non-Perl context, e.g. disk file.)
347 The marker [INTERNAL] marks Internal Implementation Details, in
348 general meant only for those who think they know what they are doing,
349 and such details may change in future releases.
353 =head2 Characteristics of an Encoding
355 An encoding has a "repertoire" of characters that it can represent,
356 and for each representable character there is at least one sequence of
357 octets that represents it.
359 =head2 Types of Encodings
361 Encodings can be divided into the following types:
365 =item * Fixed length 8-bit (or less) encodings.
367 Each character is a single octet so may have a repertoire of up to
368 256 characters. ASCII and iso-8859-* are typical examples.
370 =item * Fixed length 16-bit encodings
372 Each character is two octets so may have a repertoire of up to
373 65 536 characters. Unicode's UCS-2 is an example. Also used for
374 encodings for East Asian languages.
376 =item * Fixed length 32-bit encodings.
378 Not really very "encoded" encodings. The Unicode code points
379 are just represented as 4-octet integers. None the less because
380 different architectures use different representations of integers
381 (so called "endian") there at least two disctinct encodings.
383 =item * Multi-byte encodings
385 The number of octets needed to represent a character varies.
386 UTF-8 is a particularly complex but regular case of a multi-byte
387 encoding. Several East Asian countries use a multi-byte encoding
388 where 1-octet is used to cover western roman characters and Asian
389 characters get 2-octets.
390 (UTF-16 is strictly a multi-byte encoding taking either 2 or 4 octets
391 to represent a Unicode code point.)
393 =item * "Escape" encodings.
395 These encodings embed "escape sequences" into the octet sequence
396 which describe how the following octets are to be interpreted.
397 The iso-2022-* family is typical. Following the escape sequence
398 octets are encoded by an "embedded" encoding (which will be one
399 of the above types) until another escape sequence switches to
400 a different "embedded" encoding.
402 These schemes are very flexible and can handle mixed languages but are
403 very complex to process (and have state). No escape encodings are
404 implemented for Perl yet.
408 =head2 Specifying Encodings
410 Encodings can be specified to the API described below in two ways:
416 Encoding names are strings with characters taken from a restricted
417 repertoire. See L</"Encoding Names">.
419 =item 2. As an object
421 Encoding objects are returned by C<find_encoding($name)>.
425 =head2 Encoding Names
427 Encoding names are case insensitive. White space in names is ignored.
428 In addition an encoding may have aliases. Each encoding has one
429 "canonical" name. The "canonical" name is chosen from the names of
430 the encoding by picking the first in the following sequence:
434 =item * The MIME name as defined in IETF RFCs.
436 =item * The name in the IANA registry.
438 =item * The name used by the organization that defined it.
442 Because of all the alias issues, and because in the general case
443 encodings have state C<Encode> uses the encoding object internally
444 once an operation is in progress.
446 As of Perl 5.8.0, at least the following encodings are recognized
447 (the => marks aliases):
461 The ISO 8859 and KOI:
463 ISO 8859-1 ISO 8859-6 ISO 8859-11 KOI8-F
464 ISO 8859-2 ISO 8859-7 (12 doesn't exist) KOI8-R
465 ISO 8859-3 ISO 8859-8 ISO 8859-13 KOI8-U
466 ISO 8859-4 ISO 8859-9 ISO 8859-14
467 ISO 8859-5 ISO 8859-10 ISO 8859-15
470 Latin1 => 8859-1 Latin6 => 8859-10
471 Latin2 => 8859-2 Latin7 => 8859-13
472 Latin3 => 8859-3 Latin8 => 8859-14
473 Latin4 => 8859-4 Latin9 => 8859-15
474 Latin5 => 8859-9 Latin10 => 8859-16
483 The CJKV: Chinese, Japanese, Korean, Vietnamese:
485 ISO 2022 ISO 2022 JP-1 JIS 0201 GB 1988 Big5 EUC-CN
486 ISO 2022 CN ISO 2022 JP-2 JIS 0208 GB 2312 HZ EUC-JP
487 ISO 2022 JP ISO 2022 KR JIS 0210 GB 12345 CNS 11643 EUC-JP-0212
488 Shift-JIS GBK Big5-HKSCS EUC-KR
491 (Due to size concerns, GB 18030 and EUC-TW are available separately on CPAN,
492 under the name L<Encode::HanExtra>.)
496 CP37 CP852 CP861 CP866 CP949 CP1251 CP1256
497 CP424 CP855 CP862 CP869 CP950 CP1252 CP1257
498 CP737 CP856 CP863 CP874 CP1006 CP1253 CP1258
499 CP775 CP857 CP864 CP932 CP1047 CP1254
500 CP850 CP860 CP865 CP936 CP1250 CP1255
504 WinCyrillic => CP1251
506 WinTurkiskh => CP1254
510 WinVietnamese => CP1258
512 (All the CPI<NNN...> are available also as IBMI<NNN...>.)
516 MacCentralEuropean MacJapanese
518 MacCyrillic MacRomanian
521 MacIcelandic MacTurkish
532 =head1 PERL ENCODING API
534 =head2 Generic Encoding Interface
540 $bytes = encode(ENCODING, $string[, CHECK])
542 Encodes string from Perl's internal form into I<ENCODING> and returns
543 a sequence of octets. For CHECK see L</"Handling Malformed Data">.
545 For example to convert (internally UTF-8 encoded) Unicode data
548 $octets = encode("utf8", $unicode);
552 $string = decode(ENCODING, $bytes[, CHECK])
554 Decode sequence of octets assumed to be in I<ENCODING> into Perl's
555 internal form and returns the resulting string. For CHECK see
556 L</"Handling Malformed Data">.
558 For example to convert ISO 8859-1 data to UTF-8:
560 $utf8 = decode("latin1", $latin1);
564 from_to($string, FROM_ENCODING, TO_ENCODING[, CHECK])
566 Convert B<in-place> the data between two encodings. How did the data
567 in $string originally get to be in FROM_ENCODING? Either using
568 encode() or through PerlIO: See L</"Encoding and IO">. For CHECK
569 see L</"Handling Malformed Data">.
571 For example to convert ISO 8859-1 data to UTF-8:
573 from_to($data, "iso-8859-1", "utf-8");
575 and to convert it back:
577 from_to($data, "utf-8", "iso-8859-1");
579 Note that because the conversion happens in place, the data to be
580 converted cannot be a string constant, it must be a scalar variable.
584 =head2 Handling Malformed Data
586 If CHECK is not set, C<undef> is returned. If the data is supposed to
587 be UTF-8, an optional lexical warning (category utf8) is given. If
588 CHECK is true but not a code reference, dies.
590 It would desirable to have a way to indicate that transform should use
591 the encodings "replacement character" - no such mechanism is defined yet.
593 It is also planned to allow I<CHECK> to be a code reference.
595 This is not yet implemented as there are design issues with what its
596 arguments should be and how it returns its results.
602 Passed remaining fragment of string being processed.
603 Modifies it in place to remove bytes/characters it can understand
604 and returns a string used to represent them.
608 my $ch = substr($_[0],0,1,'');
609 return sprintf("\x{%02X}",ord($ch);
612 This scheme is close to how underlying C code for Encode works, but gives
613 the fixup routine very little context.
617 Passed original string, and an index into it of the problem area, and
618 output string so far. Appends what it will to output string and
619 returns new index into original string. For example:
622 # my ($s,$i,$d) = @_;
623 my $ch = substr($_[0],$_[1],1);
624 $_[2] .= sprintf("\x{%02X}",ord($ch);
628 This scheme gives maximal control to the fixup routine but is more
629 complicated to code, and may need internals of Encode to be tweaked to
630 keep original string intact.
636 Multiple return values rather than in-place modifications.
638 Index into the string could be pos($str) allowing s/\G...//.
644 The Unicode consortium defines the UTF-8 standard as a way of encoding
645 the entire Unicode repertiore as sequences of octets. This encoding is
646 expected to become very widespread. Perl can use this form internaly
647 to represent strings, so conversions to and from this form are
648 particularly efficient (as octets in memory do not have to change,
649 just the meta-data that tells Perl how to treat them).
655 $bytes = encode_utf8($string);
657 The characters that comprise string are encoded in Perl's superset of UTF-8
658 and the resulting octets returned as a sequence of bytes. All possible
659 characters have a UTF-8 representation so this function cannot fail.
663 $string = decode_utf8($bytes [,CHECK]);
665 The sequence of octets represented by $bytes is decoded from UTF-8
666 into a sequence of logical characters. Not all sequences of octets
667 form valid UTF-8 encodings, so it is possible for this call to fail.
668 For CHECK see L</"Handling Malformed Data">.
672 =head2 Other Encodings of Unicode
674 UTF-16 is similar to UCS-2, 16 bit or 2-byte chunks. UCS-2 can only
675 represent 0..0xFFFF, while UTF-16 has a I<surrogate pair> scheme which
676 allows it to cover the whole Unicode range.
678 Surrogates are code points set aside to encode the 0x01000..0x10FFFF
679 range of Unicode code points in pairs of 16-bit units. The I<high
680 surrogates> are the range 0xD800..0xDBFF, and the I<low surrogates>
681 are the range 0xDC00..0xDFFFF. The surrogate encoding is
683 $hi = ($uni - 0x10000) / 0x400 + 0xD800;
684 $lo = ($uni - 0x10000) % 0x400 + 0xDC00;
688 $uni = 0x10000 + ($hi - 0xD8000) * 0x400 + ($lo - 0xDC00);
690 Encode implements big-endian UCS-2 aliased to "iso-10646-1" as that
691 happens to be the name used by that representation when used with X11
694 UTF-32 or UCS-4 is 32-bit or 4-byte chunks. Perl's logical characters
695 can be considered as being in this form without encoding. An encoding
696 to transfer strings in this form (e.g. to write them to a file) would
699 pack('L*', unpack('U*', $string)); # native
701 pack('V*', unpack('U*', $string)); # little-endian
703 pack('N*', unpack('U*', $string)); # big-endian
705 depending on the endianness required.
707 No UTF-32 encodings are implemented yet.
709 Both UCS-2 and UCS-4 style encodings can have "byte order marks" by
710 representing the code point 0xFFFE as the very first thing in a file.
712 =head2 Listing available encodings
714 use Encode qw(encodings);
717 Returns a list of the canonical names of the available encodings.
719 =head2 Defining Aliases
721 use Encode qw(define_alias);
722 define_alias( newName => ENCODING);
724 Allows newName to be used as am alias for ENCODING. ENCODING may be
725 either the name of an encoding or and encoding object (as above).
727 Currently I<newName> can be specified in the following ways:
731 =item As a simple string.
733 =item As a qr// compiled regular expression, e.g.:
735 define_alias( qr/^iso8859-(\d+)$/i => '"iso-8859-$1"' );
737 In this case if I<ENCODING> is not a reference it is C<eval>-ed to
738 allow C<$1> etc. to be subsituted. The example is one way to names as
739 used in X11 font names to alias the MIME names for the iso-8859-*
742 =item As a code reference, e.g.:
744 define_alias( sub { return /^iso8859-(\d+)$/i ? "iso-8859-$1" : undef } , '');
746 In this case C<$_> will be set to the name that is being looked up and
747 I<ENCODING> is passed to the sub as its first argument. The example
748 is another way to names as used in X11 font names to alias the MIME
749 names for the iso-8859-* family.
753 =head2 Defining Encodings
755 use Encode qw(define_alias);
756 define_encoding( $object, 'canonicalName' [,alias...]);
758 Causes I<canonicalName> to be associated with I<$object>. The object
759 should provide the interface described in L</"IMPLEMENTATION CLASSES">
760 below. If more than two arguments are provided then additional
761 arguments are taken as aliases for I<$object> as for C<define_alias>.
763 =head1 Encoding and IO
765 It is very common to want to do encoding transformations when
766 reading or writing files, network connections, pipes etc.
767 If Perl is configured to use the new 'perlio' IO system then
768 C<Encode> provides a "layer" (See L<perliol>) which can transform
769 data as it is read or written.
771 Here is how the blind poet would modernise the encoding:
774 open(my $iliad,'<:encoding(iso-8859-7)','iliad.greek');
775 open(my $utf8,'>:utf8','iliad.utf8');
781 In addition the new IO system can also be configured to read/write
782 UTF-8 encoded characters (as noted above this is efficient):
784 open(my $fh,'>:utf8','anything');
785 print $fh "Any \x{0021} string \N{SMILEY FACE}\n";
787 Either of the above forms of "layer" specifications can be made the default
788 for a lexical scope with the C<use open ...> pragma. See L<open>.
790 Once a handle is open is layers can be altered using C<binmode>.
792 Without any such configuration, or if Perl itself is built using
793 system's own IO, then write operations assume that file handle accepts
794 only I<bytes> and will C<die> if a character larger than 255 is
795 written to the handle. When reading, each octet from the handle
796 becomes a byte-in-a-character. Note that this default is the same
797 behaviour as bytes-only languages (including Perl before v5.6) would
798 have, and is sufficient to handle native 8-bit encodings
799 e.g. iso-8859-1, EBCDIC etc. and any legacy mechanisms for handling
800 other encodings and binary data.
802 In other cases it is the programs responsibility to transform
803 characters into bytes using the API above before doing writes, and to
804 transform the bytes read from a handle into characters before doing
805 "character operations" (e.g. C<lc>, C</\W+/>, ...).
807 You can also use PerlIO to convert larger amounts of data you don't
808 want to bring into memory. For example to convert between ISO 8859-1
809 (Latin 1) and UTF-8 (or UTF-EBCDIC in EBCDIC machines):
811 open(F, "<:encoding(iso-8859-1)", "data.txt") or die $!;
812 open(G, ">:utf8", "data.utf") or die $!;
813 while (<F>) { print G }
815 # Could also do "print G <F>" but that would pull
816 # the whole file into memory just to write it out again.
820 open(my $f, "<:encoding(cp1252)")
821 open(my $g, ">:encoding(iso-8859-2)")
822 open(my $h, ">:encoding(latin9)") # iso-8859-15
824 See L<PerlIO> for more information.
826 See also L<encoding> for how to change the default encoding of the
829 =head1 Encoding How to ...
835 =item * IO with mixed content (faking iso-2020-*)
837 =item * MIME's Content-Length:
839 =item * UTF-8 strings in binary data.
841 =item * Perl/Encode wrappers on non-Unicode XS modules.
845 =head1 Messing with Perl's Internals
847 The following API uses parts of Perl's internals in the current
848 implementation. As such they are efficient, but may change.
852 =item * is_utf8(STRING [, CHECK])
854 [INTERNAL] Test whether the UTF-8 flag is turned on in the STRING.
855 If CHECK is true, also checks the data in STRING for being well-formed
856 UTF-8. Returns true if successful, false otherwise.
858 =item * valid_utf8(STRING)
860 [INTERNAL] Test whether STRING is in a consistent state. Will return
861 true if string is held as bytes, or is well-formed UTF-8 and has the
862 UTF-8 flag on. Main reason for this routine is to allow Perl's
863 testsuite to check that operations have left strings in a consistent
870 [INTERNAL] Turn on the UTF-8 flag in STRING. The data in STRING is
871 B<not> checked for being well-formed UTF-8. Do not use unless you
872 B<know> that the STRING is well-formed UTF-8. Returns the previous
873 state of the UTF-8 flag (so please don't test the return value as
874 I<not> success or failure), or C<undef> if STRING is not a string.
880 [INTERNAL] Turn off the UTF-8 flag in STRING. Do not use frivolously.
881 Returns the previous state of the UTF-8 flag (so please don't test the
882 return value as I<not> success or failure), or C<undef> if STRING is
887 =head1 IMPLEMENTATION CLASSES
889 As mentioned above encodings are (in the current implementation at least)
890 defined by objects. The mapping of encoding name to object is via the
893 The values of the hash can currently be either strings or objects.
894 The string form may go away in the future. The string form occurs
895 when C<encodings()> has scanned C<@INC> for loadable encodings but has
896 not actually loaded the encoding in question. This is because the
897 current "loading" process is all Perl and a bit slow.
899 Once an encoding is loaded then value of the hash is object which
900 implements the encoding. The object should provide the following
907 Should return the string representing the canonical name of the encoding.
909 =item -E<gt>new_sequence
911 This is a placeholder for encodings with state. It should return an
912 object which implements this interface, all current implementations
913 return the original object.
915 =item -E<gt>encode($string,$check)
917 Should return the octet sequence representing I<$string>. If I<$check>
918 is true it should modify I<$string> in place to remove the converted
919 part (i.e. the whole string unless there is an error). If an error
920 occurs it should return the octet sequence for the fragment of string
921 that has been converted, and modify $string in-place to remove the
922 converted part leaving it starting with the problem fragment.
924 If check is is false then C<encode> should make a "best effort" to
925 convert the string - for example by using a replacement character.
927 =item -E<gt>decode($octets,$check)
929 Should return the string that I<$octets> represents. If I<$check> is
930 true it should modify I<$octets> in place to remove the converted part
931 (i.e. the whole sequence unless there is an error). If an error
932 occurs it should return the fragment of string that has been
933 converted, and modify $octets in-place to remove the converted part
934 leaving it starting with the problem fragment.
936 If check is is false then C<decode> should make a "best effort" to
937 convert the string - for example by using Unicode's "\x{FFFD}" as a
938 replacement character.
942 It should be noted that the check behaviour is different from the
943 outer public API. The logic is that the "unchecked" case is useful
944 when encoding is part of a stream which may be reporting errors
945 (e.g. STDERR). In such cases it is desirable to get everything
946 through somehow without causing additional errors which obscure the
947 original one. Also the encoding is best placed to know what the
948 correct replacement character is, so if that is the desired behaviour
949 then letting low level code do it is the most efficient.
951 In contrast if check is true, the scheme above allows the encoding to
952 do as much as it can and tell layer above how much that was. What is
953 lacking at present is a mechanism to report what went wrong. The most
954 likely interface will be an additional method call to the object, or
955 perhaps (to avoid forcing per-stream objects on otherwise stateless
956 encodings) and additional parameter.
958 It is also highly desirable that encoding classes inherit from
959 C<Encode::Encoding> as a base class. This allows that class to define
960 additional behaviour for all encoding objects. For example built in
961 Unicode, UCS-2 and UTF-8 classes use :
963 package Encode::MyEncoding;
964 use base qw(Encode::Encoding);
966 __PACKAGE__->Define(qw(myCanonical myAlias));
968 To create an object with bless {Name => ...},$class, and call
969 define_encoding. They inherit their C<name> method from
972 =head2 Compiled Encodings
974 F<Encode.xs> provides a class C<Encode::XS> which provides the
975 interface described above. It calls a generic octet-sequence to
976 octet-sequence "engine" that is driven by tables (defined in
977 F<encengine.c>). The same engine is used for both encode and
978 decode. C<Encode:XS>'s C<encode> forces Perl's characters to their
979 UTF-8 form and then treats them as just another multibyte
980 encoding. C<Encode:XS>'s C<decode> transforms the sequence and then
981 turns the UTF-8-ness flag as that is the form that the tables are
982 defined to produce. For details of the engine see the comments in
985 The tables are produced by the Perl script F<compile> (the name needs
986 to change so we can eventually install it somewhere). F<compile> can
987 currently read two formats:
993 This is a coined format used by Tcl. It is documented in
994 Encode/EncodeFormat.pod.
998 This is the semi-standard format used by IBM's ICU package.
1002 F<compile> can write the following forms:
1008 See above - the F<Encode/*.ucm> files provided with the distribution have
1009 been created from the original Tcl .enc files using this approach.
1013 Produces tables as C data structures - this is used to build in encodings
1014 into F<Encode.so>/F<Encode.dll>.
1018 In theory this allows encodings to be stand-alone loadable Perl
1019 extensions. The process has not yet been tested. The plan is to use
1020 this approach for large East Asian encodings.
1024 The set of encodings built-in to F<Encode.so>/F<Encode.dll> is
1025 determined by F<Makefile.PL>. The current set is as follows:
1029 =item ascii and iso-8859-*
1031 That is all the common 8-bit "western" encodings.
1033 =item IBM-1047 and two other variants of EBCDIC.
1035 These are the same variants that are supported by EBCDIC Perl as
1036 "native" encodings. They are included to prove "reversibility" of
1037 some constructs in EBCDIC Perl.
1039 =item symbol and dingbats as used by Tk on X11.
1041 (The reason Encode got started was to support Perl/Tk.)
1045 That set is rather ad hoc and has been driven by the needs of the
1046 tests rather than the needs of typical applications. It is likely
1051 L<perlunicode>, L<perlebcdic>, L<perlfunc/open>, L<PerlIO>, L<encoding>