Remove GDBM_File from the dynamic extensions for UTS,
[p5sagit/p5-mst-13.2.git] / ext / Encode / Encode.pm
CommitLineData
2c674647 1package Encode;
51ef4e11 2use strict;
2c674647 3
b8a524e9 4our $VERSION = '0.02';
2c674647 5
6require DynaLoader;
7require Exporter;
8
51ef4e11 9our @ISA = qw(Exporter DynaLoader);
2c674647 10
4411f3b6 11# Public, encouraged API is exported by default
51ef4e11 12our @EXPORT = qw (
4411f3b6 13 encode
14 decode
15 encode_utf8
16 decode_utf8
17 find_encoding
51ef4e11 18 encodings
4411f3b6 19);
20
51ef4e11 21our @EXPORT_OK =
2c674647 22 qw(
51ef4e11 23 define_encoding
24 define_alias
2c674647 25 from_to
26 is_utf8
4411f3b6 27 is_8bit
28 is_16bit
a12c0f56 29 utf8_upgrade
30 utf8_downgrade
4411f3b6 31 _utf8_on
32 _utf8_off
2c674647 33 );
34
35bootstrap Encode ();
36
4411f3b6 37# Documentation moved after __END__ for speed - NI-S
2c674647 38
bf230f3d 39use Carp;
40
51ef4e11 41# Make a %encoding package variable to allow a certain amount of cheating
42our %encoding;
43my @alias; # ordered matching list
44my %alias; # cached known aliases
6d6a7c8d 45 # 0 1 2 3 4 5 6 7 8 9 10
46our @latin2iso_num = ( 0, 1, 2, 3, 4, 9, 10, 13, 14, 15, 16 );
47
5345d506 48
656753f8 49sub encodings
50{
51 my ($class) = @_;
51ef4e11 52 return keys %encoding;
53}
54
55sub findAlias
56{
57 my $class = shift;
58 local $_ = shift;
59 unless (exists $alias{$_})
656753f8 60 {
51ef4e11 61 for (my $i=0; $i < @alias; $i += 2)
656753f8 62 {
51ef4e11 63 my $alias = $alias[$i];
64 my $val = $alias[$i+1];
65 my $new;
66 if (ref($alias) eq 'Regexp' && $_ =~ $alias)
5345d506 67 {
51ef4e11 68 $new = eval $val;
69 }
70 elsif (ref($alias) eq 'CODE')
71 {
72 $new = &{$alias}($val)
73 }
5ad8ef52 74 elsif (lc($_) eq lc($alias))
51ef4e11 75 {
76 $new = $val;
77 }
78 if (defined($new))
79 {
80 next if $new eq $_; # avoid (direct) recursion on bugs
81 my $enc = (ref($new)) ? $new : find_encoding($new);
82 if ($enc)
5345d506 83 {
51ef4e11 84 $alias{$_} = $enc;
85 last;
5345d506 86 }
87 }
656753f8 88 }
5345d506 89 }
51ef4e11 90 return $alias{$_};
5345d506 91}
92
51ef4e11 93sub define_alias
5345d506 94{
51ef4e11 95 while (@_)
5345d506 96 {
51ef4e11 97 my ($alias,$name) = splice(@_,0,2);
98 push(@alias, $alias => $name);
656753f8 99 }
51ef4e11 100}
101
016cb72c 102# Allow variants of iso-8859-1 etc.
d6089a2a 103define_alias( qr/^iso[-_]?(\d+)[-_](\d+)$/i => '"iso-$1-$2"' );
016cb72c 104
105# Allow latin-1 style names as well
016cb72c 106define_alias( qr/^latin[-_]?(\d+)$/i => '"iso-8859-$latin2iso_num[$1]"' );
107
108# Common names for non-latin prefered MIME names
109define_alias( 'ascii' => 'US-ascii',
110 'cyrillic' => 'iso-8859-5',
111 'arabic' => 'iso-8859-6',
112 'greek' => 'iso-8859-7',
113 'hebrew' => 'iso-8859-8');
114
51ef4e11 115define_alias( 'ibm-1047' => 'cp1047');
116
016cb72c 117# Map white space and _ to '-'
118define_alias( qr/^(\S+)[\s_]+(.*)$/i => '"$1-$2"' );
119
51ef4e11 120sub define_encoding
121{
122 my $obj = shift;
123 my $name = shift;
124 $encoding{$name} = $obj;
125 my $lc = lc($name);
126 define_alias($lc => $obj) unless $lc eq $name;
127 while (@_)
656753f8 128 {
51ef4e11 129 my $alias = shift;
130 define_alias($alias,$obj);
656753f8 131 }
51ef4e11 132 return $obj;
656753f8 133}
134
656753f8 135sub getEncoding
136{
137 my ($class,$name) = @_;
5345d506 138 my $enc;
0f43fc90 139 if (ref($name) && $name->can('new_sequence'))
140 {
141 return $name;
142 }
51ef4e11 143 if (exists $encoding{$name})
656753f8 144 {
51ef4e11 145 return $encoding{$name};
146 }
147 else
148 {
149 return $class->findAlias($name);
656753f8 150 }
656753f8 151}
152
4411f3b6 153sub find_encoding
154{
155 my ($name) = @_;
156 return __PACKAGE__->getEncoding($name);
157}
158
159sub encode
160{
161 my ($name,$string,$check) = @_;
162 my $enc = find_encoding($name);
163 croak("Unknown encoding '$name'") unless defined $enc;
50d26985 164 my $octets = $enc->encode($string,$check);
4411f3b6 165 return undef if ($check && length($string));
166 return $octets;
167}
168
169sub decode
170{
171 my ($name,$octets,$check) = @_;
172 my $enc = find_encoding($name);
173 croak("Unknown encoding '$name'") unless defined $enc;
50d26985 174 my $string = $enc->decode($octets,$check);
4411f3b6 175 return undef if ($check && length($octets));
176 return $string;
177}
178
179sub from_to
180{
181 my ($string,$from,$to,$check) = @_;
182 my $f = find_encoding($from);
183 croak("Unknown encoding '$from'") unless defined $f;
184 my $t = find_encoding($to);
185 croak("Unknown encoding '$to'") unless defined $t;
50d26985 186 my $uni = $f->decode($string,$check);
4411f3b6 187 return undef if ($check && length($string));
50d26985 188 $string = $t->encode($uni,$check);
4411f3b6 189 return undef if ($check && length($uni));
190 return length($_[0] = $string);
191}
192
193sub encode_utf8
194{
195 my ($str) = @_;
1b026014 196 utf8::encode($str);
4411f3b6 197 return $str;
198}
199
200sub decode_utf8
201{
202 my ($str) = @_;
1b026014 203 return undef unless utf8::decode($str);
4411f3b6 204 return $str;
205}
206
50d26985 207package Encode::Encoding;
208# Base class for classes which implement encodings
4edaa979 209
51ef4e11 210sub Define
211{
212 my $obj = shift;
213 my $canonical = shift;
214 $obj = bless { Name => $canonical },$obj unless ref $obj;
215 # warn "$canonical => $obj\n";
216 Encode::define_encoding($obj, $canonical, @_);
217}
218
219sub name { shift->{'Name'} }
220
50d26985 221# Temporary legacy methods
4edaa979 222sub toUnicode { shift->decode(@_) }
223sub fromUnicode { shift->encode(@_) }
224
225sub new_sequence { return $_[0] }
50d26985 226
227package Encode::XS;
228use base 'Encode::Encoding';
229
5ad8ef52 230package Encode::Internal;
50d26985 231use base 'Encode::Encoding';
656753f8 232
9b37254d 233# Dummy package that provides the encode interface but leaves data
1b026014 234# as UTF-X encoded. It is here so that from_to() works.
656753f8 235
5ad8ef52 236__PACKAGE__->Define('Internal');
237
238Encode::define_alias( 'Unicode' => 'Internal' ) if ord('A') == 65;
656753f8 239
50d26985 240sub decode
a12c0f56 241{
242 my ($obj,$str,$chk) = @_;
1b026014 243 utf8::upgrade($str);
a12c0f56 244 $_[1] = '' if $chk;
245 return $str;
246}
656753f8 247
50d26985 248*encode = \&decode;
656753f8 249
5ad8ef52 250package Encoding::Unicode;
251use base 'Encode::Encoding';
252
253__PACKAGE__->Define('Unicode') unless ord('A') == 65;
254
255sub decode
256{
257 my ($obj,$str,$chk) = @_;
258 my $res = '';
259 for (my $i = 0; $i < length($str); $i++)
260 {
261 $res .= chr(utf8::unicode_to_native(ord(substr($str,$i,1))));
262 }
263 $_[1] = '' if $chk;
264 return $res;
265}
266
267sub encode
268{
269 my ($obj,$str,$chk) = @_;
270 my $res = '';
271 for (my $i = 0; $i < length($str); $i++)
272 {
273 $res .= chr(utf8::native_to_unicode(ord(substr($str,$i,1))));
274 }
275 $_[1] = '' if $chk;
276 return $res;
277}
278
279
4411f3b6 280package Encode::utf8;
50d26985 281use base 'Encode::Encoding';
4411f3b6 282# package to allow long-hand
283# $octets = encode( utf8 => $string );
284#
285
51ef4e11 286__PACKAGE__->Define(qw(UTF-8 utf8));
4411f3b6 287
50d26985 288sub decode
4411f3b6 289{
290 my ($obj,$octets,$chk) = @_;
2a936312 291 my $str = Encode::decode_utf8($octets);
4411f3b6 292 if (defined $str)
293 {
294 $_[1] = '' if $chk;
295 return $str;
296 }
297 return undef;
298}
299
50d26985 300sub encode
4411f3b6 301{
302 my ($obj,$string,$chk) = @_;
2a936312 303 my $octets = Encode::encode_utf8($string);
4411f3b6 304 $_[1] = '' if $chk;
305 return $octets;
4411f3b6 306}
307
9b37254d 308package Encode::iso10646_1;
50d26985 309use base 'Encode::Encoding';
51ef4e11 310# Encoding is 16-bit network order Unicode (no surogates)
9b37254d 311# Used for X font encodings
87714904 312
8040349a 313__PACKAGE__->Define(qw(UCS-2 iso-10646-1));
87714904 314
50d26985 315sub decode
87714904 316{
317 my ($obj,$str,$chk) = @_;
318 my $uni = '';
319 while (length($str))
320 {
5dcbab34 321 my $code = unpack('n',substr($str,0,2,'')) & 0xffff;
87714904 322 $uni .= chr($code);
323 }
324 $_[1] = $str if $chk;
8040349a 325 utf8::upgrade($uni);
87714904 326 return $uni;
327}
328
50d26985 329sub encode
87714904 330{
331 my ($obj,$uni,$chk) = @_;
332 my $str = '';
333 while (length($uni))
334 {
335 my $ch = substr($uni,0,1,'');
336 my $x = ord($ch);
337 unless ($x < 32768)
338 {
339 last if ($chk);
340 $x = 0;
341 }
5dcbab34 342 $str .= pack('n',$x);
656753f8 343 }
bf230f3d 344 $_[1] = $uni if $chk;
656753f8 345 return $str;
346}
347
4411f3b6 348# switch back to Encode package in case we ever add AutoLoader
349package Encode;
350
656753f8 3511;
352
2a936312 353__END__
354
4411f3b6 355=head1 NAME
356
357Encode - character encodings
358
359=head1 SYNOPSIS
360
361 use Encode;
362
363=head1 DESCRIPTION
364
47bfe92f 365The C<Encode> module provides the interfaces between Perl's strings
366and the rest of the system. Perl strings are sequences of B<characters>.
4411f3b6 367
368The repertoire of characters that Perl can represent is at least that
47bfe92f 369defined by the Unicode Consortium. On most platforms the ordinal
370values of the characters (as returned by C<ord(ch)>) is the "Unicode
371codepoint" for the character (the exceptions are those platforms where
372the legacy encoding is some variant of EBCDIC rather than a super-set
373of ASCII - see L<perlebcdic>).
4411f3b6 374
375Traditionaly computer data has been moved around in 8-bit chunks
376often called "bytes". These chunks are also known as "octets" in
377networking standards. Perl is widely used to manipulate data of
378many types - not only strings of characters representing human or
379computer languages but also "binary" data being the machines representation
380of numbers, pixels in an image - or just about anything.
381
47bfe92f 382When Perl is processing "binary data" the programmer wants Perl to process
383"sequences of bytes". This is not a problem for Perl - as a byte has 256
384possible values it easily fits in Perl's much larger "logical character".
4411f3b6 385
386=head2 TERMINOLOGY
387
4ac9195f 388=over 4
4411f3b6 389
390=item *
391
392I<character>: a character in the range 0..(2**32-1) (or more).
47bfe92f 393(What Perl's strings are made of.)
4411f3b6 394
395=item *
396
397I<byte>: a character in the range 0..255
47bfe92f 398(A special case of a Perl character.)
4411f3b6 399
400=item *
401
402I<octet>: 8 bits of data, with ordinal values 0..255
47bfe92f 403(Term for bytes passed to or from a non-Perl context, e.g. disk file.)
4411f3b6 404
405=back
406
407The marker [INTERNAL] marks Internal Implementation Details, in
408general meant only for those who think they know what they are doing,
409and such details may change in future releases.
410
411=head1 ENCODINGS
412
413=head2 Characteristics of an Encoding
414
415An encoding has a "repertoire" of characters that it can represent,
416and for each representable character there is at least one sequence of
417octets that represents it.
418
419=head2 Types of Encodings
420
421Encodings can be divided into the following types:
422
423=over 4
424
425=item * Fixed length 8-bit (or less) encodings.
426
427Each character is a single octet so may have a repertoire of up to
428256 characters. ASCII and iso-8859-* are typical examples.
429
430=item * Fixed length 16-bit encodings
431
432Each character is two octets so may have a repertoire of up to
47bfe92f 43365 536 characters. Unicode's UCS-2 is an example. Also used for
4411f3b6 434encodings for East Asian languages.
435
436=item * Fixed length 32-bit encodings.
437
438Not really very "encoded" encodings. The Unicode code points
439are just represented as 4-octet integers. None the less because
440different architectures use different representations of integers
441(so called "endian") there at least two disctinct encodings.
442
443=item * Multi-byte encodings
444
445The number of octets needed to represent a character varies.
446UTF-8 is a particularly complex but regular case of a multi-byte
447encoding. Several East Asian countries use a multi-byte encoding
448where 1-octet is used to cover western roman characters and Asian
449characters get 2-octets.
450(UTF-16 is strictly a multi-byte encoding taking either 2 or 4 octets
451to represent a Unicode code point.)
452
453=item * "Escape" encodings.
454
455These encodings embed "escape sequences" into the octet sequence
456which describe how the following octets are to be interpreted.
457The iso-2022-* family is typical. Following the escape sequence
458octets are encoded by an "embedded" encoding (which will be one
459of the above types) until another escape sequence switches to
460a different "embedded" encoding.
461
462These schemes are very flexible and can handle mixed languages but are
47bfe92f 463very complex to process (and have state). No escape encodings are
464implemented for Perl yet.
4411f3b6 465
466=back
467
468=head2 Specifying Encodings
469
470Encodings can be specified to the API described below in two ways:
471
472=over 4
473
474=item 1. By name
475
47bfe92f 476Encoding names are strings with characters taken from a restricted
477repertoire. See L</"Encoding Names">.
4411f3b6 478
479=item 2. As an object
480
481Encoding objects are returned by C<find_encoding($name)>.
482
483=back
484
485=head2 Encoding Names
486
487Encoding names are case insensitive. White space in names is ignored.
47bfe92f 488In addition an encoding may have aliases. Each encoding has one
489"canonical" name. The "canonical" name is chosen from the names of
490the encoding by picking the first in the following sequence:
4411f3b6 491
492=over 4
493
494=item * The MIME name as defined in IETF RFC-XXXX.
495
496=item * The name in the IANA registry.
497
498=item * The name used by the the organization that defined it.
499
500=back
501
502Because of all the alias issues, and because in the general case
503encodings have state C<Encode> uses the encoding object internally
504once an operation is in progress.
505
4411f3b6 506=head1 PERL ENCODING API
507
508=head2 Generic Encoding Interface
509
510=over 4
511
512=item *
513
514 $bytes = encode(ENCODING, $string[, CHECK])
515
47bfe92f 516Encodes string from Perl's internal form into I<ENCODING> and returns
517a sequence of octets. For CHECK see L</"Handling Malformed Data">.
4411f3b6 518
519=item *
520
521 $string = decode(ENCODING, $bytes[, CHECK])
522
47bfe92f 523Decode sequence of octets assumed to be in I<ENCODING> into Perl's
524internal form and returns the resulting string. For CHECK see
525L</"Handling Malformed Data">.
526
527=item *
528
529 from_to($string, FROM_ENCODING, TO_ENCODING[, CHECK])
530
2b106fbe 531Convert B<in-place> the data between two encodings. How did the data
532in $string originally get to be in FROM_ENCODING? Either using
533encode() or through PerlIO: See L</"Encode and PerlIO">. For CHECK
534see L</"Handling Malformed Data">.
535
536For example to convert ISO 8859-1 data to UTF-8:
537
538 from_to($data, "iso-8859-1", "utf-8");
539
540and to convert it back:
541
542 from_to($data, "utf-8", "iso-8859-1");
4411f3b6 543
ab97ca19 544Note that because the conversion happens in place, the data to be
545converted cannot be a string constant, it must be a scalar variable.
546
4411f3b6 547=back
548
549=head2 Handling Malformed Data
550
551If CHECK is not set, C<undef> is returned. If the data is supposed to
47bfe92f 552be UTF-8, an optional lexical warning (category utf8) is given. If
553CHECK is true but not a code reference, dies.
4411f3b6 554
47bfe92f 555It would desirable to have a way to indicate that transform should use
556the encodings "replacement character" - no such mechanism is defined yet.
4411f3b6 557
558It is also planned to allow I<CHECK> to be a code reference.
559
47bfe92f 560This is not yet implemented as there are design issues with what its
561arguments should be and how it returns its results.
4411f3b6 562
563=over 4
564
565=item Scheme 1
566
567Passed remaining fragment of string being processed.
568Modifies it in place to remove bytes/characters it can understand
569and returns a string used to represent them.
570e.g.
571
572 sub fixup {
573 my $ch = substr($_[0],0,1,'');
574 return sprintf("\x{%02X}",ord($ch);
575 }
576
577This scheme is close to how underlying C code for Encode works, but gives
578the fixup routine very little context.
579
580=item Scheme 2
581
47bfe92f 582Passed original string, and an index into it of the problem area, and
583output string so far. Appends what it will to output string and
584returns new index into original string. For example:
4411f3b6 585
586 sub fixup {
587 # my ($s,$i,$d) = @_;
588 my $ch = substr($_[0],$_[1],1);
589 $_[2] .= sprintf("\x{%02X}",ord($ch);
590 return $_[1]+1;
591 }
592
47bfe92f 593This scheme gives maximal control to the fixup routine but is more
594complicated to code, and may need internals of Encode to be tweaked to
595keep original string intact.
4411f3b6 596
597=item Other Schemes
598
599Hybrids of above.
600
601Multiple return values rather than in-place modifications.
602
603Index into the string could be pos($str) allowing s/\G...//.
604
605=back
606
607=head2 UTF-8 / utf8
608
609The Unicode consortium defines the UTF-8 standard as a way of encoding
47bfe92f 610the entire Unicode repertiore as sequences of octets. This encoding is
611expected to become very widespread. Perl can use this form internaly
612to represent strings, so conversions to and from this form are
613particularly efficient (as octets in memory do not have to change,
614just the meta-data that tells Perl how to treat them).
4411f3b6 615
616=over 4
617
618=item *
619
620 $bytes = encode_utf8($string);
621
47bfe92f 622The characters that comprise string are encoded in Perl's superset of UTF-8
4411f3b6 623and the resulting octets returned as a sequence of bytes. All possible
624characters have a UTF-8 representation so this function cannot fail.
625
626=item *
627
628 $string = decode_utf8($bytes [,CHECK]);
629
47bfe92f 630The sequence of octets represented by $bytes is decoded from UTF-8
631into a sequence of logical characters. Not all sequences of octets
632form valid UTF-8 encodings, so it is possible for this call to fail.
633For CHECK see L</"Handling Malformed Data">.
4411f3b6 634
635=back
636
637=head2 Other Encodings of Unicode
638
47bfe92f 639UTF-16 is similar to UCS-2, 16 bit or 2-byte chunks. UCS-2 can only
640represent 0..0xFFFF, while UTF-16 has a "surrogate pair" scheme which
641allows it to cover the whole Unicode range.
4411f3b6 642
8040349a 643Encode implements big-endian UCS-2 aliased to "iso-10646-1" as that
47bfe92f 644happens to be the name used by that representation when used with X11
645fonts.
4411f3b6 646
647UTF-32 or UCS-4 is 32-bit or 4-byte chunks. Perl's logical characters
648can be considered as being in this form without encoding. An encoding
47bfe92f 649to transfer strings in this form (e.g. to write them to a file) would
650need to
4411f3b6 651
652 pack('L',map(chr($_),split(//,$string))); # native
653 or
654 pack('V',map(chr($_),split(//,$string))); # little-endian
655 or
656 pack('N',map(chr($_),split(//,$string))); # big-endian
657
658depending on the endian required.
659
51ef4e11 660No UTF-32 encodings are implemented yet.
4411f3b6 661
47bfe92f 662Both UCS-2 and UCS-4 style encodings can have "byte order marks" by
663representing the code point 0xFFFE as the very first thing in a file.
4411f3b6 664
51ef4e11 665=head2 Listing available encodings
666
667 use Encode qw(encodings);
668 @list = encodings();
669
670Returns a list of the canonical names of the available encodings.
671
672=head2 Defining Aliases
673
674 use Encode qw(define_alias);
675 define_alias( newName => ENCODING);
676
47bfe92f 677Allows newName to be used as am alias for ENCODING. ENCODING may be
678either the name of an encoding or and encoding object (as above).
51ef4e11 679
680Currently I<newName> can be specified in the following ways:
681
682=over 4
683
684=item As a simple string.
685
686=item As a qr// compiled regular expression, e.g.:
687
688 define_alias( qr/^iso8859-(\d+)$/i => '"iso-8859-$1"' );
689
47bfe92f 690In this case if I<ENCODING> is not a reference it is C<eval>-ed to
691allow C<$1> etc. to be subsituted. The example is one way to names as
692used in X11 font names to alias the MIME names for the iso-8859-*
693family.
51ef4e11 694
695=item As a code reference, e.g.:
696
697 define_alias( sub { return /^iso8859-(\d+)$/i ? "iso-8859-$1" : undef } , '');
698
699In this case C<$_> will be set to the name that is being looked up and
47bfe92f 700I<ENCODING> is passed to the sub as its first argument. The example
701is another way to names as used in X11 font names to alias the MIME
702names for the iso-8859-* family.
51ef4e11 703
704=back
705
706=head2 Defining Encodings
707
708 use Encode qw(define_alias);
709 define_encoding( $object, 'canonicalName' [,alias...]);
710
47bfe92f 711Causes I<canonicalName> to be associated with I<$object>. The object
712should provide the interface described in L</"IMPLEMENTATION CLASSES">
713below. If more than two arguments are provided then additional
714arguments are taken as aliases for I<$object> as for C<define_alias>.
51ef4e11 715
4411f3b6 716=head1 Encoding and IO
717
718It is very common to want to do encoding transformations when
719reading or writing files, network connections, pipes etc.
47bfe92f 720If Perl is configured to use the new 'perlio' IO system then
4411f3b6 721C<Encode> provides a "layer" (See L<perliol>) which can transform
722data as it is read or written.
723
51ef4e11 724 open(my $ilyad,'>:encoding(iso-8859-7)','ilyad.greek');
4411f3b6 725 print $ilyad @epic;
726
727In addition the new IO system can also be configured to read/write
728UTF-8 encoded characters (as noted above this is efficient):
729
730 open(my $fh,'>:utf8','anything');
731 print $fh "Any \x{0021} string \N{SMILEY FACE}\n";
732
733Either of the above forms of "layer" specifications can be made the default
734for a lexical scope with the C<use open ...> pragma. See L<open>.
735
736Once a handle is open is layers can be altered using C<binmode>.
737
47bfe92f 738Without any such configuration, or if Perl itself is built using
4411f3b6 739system's own IO, then write operations assume that file handle accepts
740only I<bytes> and will C<die> if a character larger than 255 is
741written to the handle. When reading, each octet from the handle
742becomes a byte-in-a-character. Note that this default is the same
47bfe92f 743behaviour as bytes-only languages (including Perl before v5.6) would
744have, and is sufficient to handle native 8-bit encodings
745e.g. iso-8859-1, EBCDIC etc. and any legacy mechanisms for handling
746other encodings and binary data.
747
748In other cases it is the programs responsibility to transform
749characters into bytes using the API above before doing writes, and to
750transform the bytes read from a handle into characters before doing
751"character operations" (e.g. C<lc>, C</\W+/>, ...).
752
753=head1 Encode and PerlIO
754
755The PerlIO layer (new since Perl 5.7) can be used to automatically
756convert the data being read in or written out to be converted from
757some encoding into Perl's internal encoding or from Perl's internal
758encoding into some other encoding.
759
760Examples:
4411f3b6 761
47bfe92f 762 open(my $f, "<:encoding(cp1252)")
763
764 open(my $g, ">:encoding(iso-8859-1)")
765
766You can also use PerlIO to convert larger amounts of data you don't
767want to bring into memory. For example to convert between ISO 8859-1
768(Latin 1) and UTF-8 (or UTF-EBCDIC in EBCDIC machines):
769
770 open(F, "<:encoding(iso-8859-1)", "data.txt") or die $!;
771 open(G, ">:utf8", "data.utf") or die $!;
772 while (<F>) { print G }
773
2b106fbe 774 # Could also do "print G <F>" but that would pull
47bfe92f 775 # the whole file into memory just to write it out again.
776
777See L<PerlIO> for more information.
4411f3b6 778
779=head1 Encoding How to ...
780
781To do:
782
783=over 4
784
785=item * IO with mixed content (faking iso-2020-*)
786
787=item * MIME's Content-Length:
788
789=item * UTF-8 strings in binary data.
790
47bfe92f 791=item * Perl/Encode wrappers on non-Unicode XS modules.
4411f3b6 792
793=back
794
795=head1 Messing with Perl's Internals
796
47bfe92f 797The following API uses parts of Perl's internals in the current
798implementation. As such they are efficient, but may change.
4411f3b6 799
800=over 4
801
4411f3b6 802=item * is_utf8(STRING [, CHECK])
803
804[INTERNAL] Test whether the UTF-8 flag is turned on in the STRING.
47bfe92f 805If CHECK is true, also checks the data in STRING for being well-formed
806UTF-8. Returns true if successful, false otherwise.
4411f3b6 807
808=item * valid_utf8(STRING)
809
47bfe92f 810[INTERNAL] Test whether STRING is in a consistent state. Will return
811true if string is held as bytes, or is well-formed UTF-8 and has the
812UTF-8 flag on. Main reason for this routine is to allow Perl's
813testsuite to check that operations have left strings in a consistent
814state.
4411f3b6 815
816=item *
817
818 _utf8_on(STRING)
819
820[INTERNAL] Turn on the UTF-8 flag in STRING. The data in STRING is
821B<not> checked for being well-formed UTF-8. Do not use unless you
822B<know> that the STRING is well-formed UTF-8. Returns the previous
823state of the UTF-8 flag (so please don't test the return value as
824I<not> success or failure), or C<undef> if STRING is not a string.
825
826=item *
827
828 _utf8_off(STRING)
829
830[INTERNAL] Turn off the UTF-8 flag in STRING. Do not use frivolously.
831Returns the previous state of the UTF-8 flag (so please don't test the
832return value as I<not> success or failure), or C<undef> if STRING is
833not a string.
834
835=back
836
4edaa979 837=head1 IMPLEMENTATION CLASSES
838
839As mentioned above encodings are (in the current implementation at least)
840defined by objects. The mapping of encoding name to object is via the
51ef4e11 841C<%encodings> hash.
4edaa979 842
843The values of the hash can currently be either strings or objects.
844The string form may go away in the future. The string form occurs
845when C<encodings()> has scanned C<@INC> for loadable encodings but has
846not actually loaded the encoding in question. This is because the
47bfe92f 847current "loading" process is all Perl and a bit slow.
4edaa979 848
47bfe92f 849Once an encoding is loaded then value of the hash is object which
850implements the encoding. The object should provide the following
851interface:
4edaa979 852
853=over 4
854
855=item -E<gt>name
856
857Should return the string representing the canonical name of the encoding.
858
859=item -E<gt>new_sequence
860
47bfe92f 861This is a placeholder for encodings with state. It should return an
862object which implements this interface, all current implementations
863return the original object.
4edaa979 864
865=item -E<gt>encode($string,$check)
866
47bfe92f 867Should return the octet sequence representing I<$string>. If I<$check>
868is true it should modify I<$string> in place to remove the converted
869part (i.e. the whole string unless there is an error). If an error
870occurs it should return the octet sequence for the fragment of string
871that has been converted, and modify $string in-place to remove the
872converted part leaving it starting with the problem fragment.
4edaa979 873
47bfe92f 874If check is is false then C<encode> should make a "best effort" to
875convert the string - for example by using a replacement character.
4edaa979 876
877=item -E<gt>decode($octets,$check)
878
47bfe92f 879Should return the string that I<$octets> represents. If I<$check> is
880true it should modify I<$octets> in place to remove the converted part
881(i.e. the whole sequence unless there is an error). If an error
882occurs it should return the fragment of string that has been
883converted, and modify $octets in-place to remove the converted part
4edaa979 884leaving it starting with the problem fragment.
885
47bfe92f 886If check is is false then C<decode> should make a "best effort" to
887convert the string - for example by using Unicode's "\x{FFFD}" as a
888replacement character.
4edaa979 889
890=back
891
47bfe92f 892It should be noted that the check behaviour is different from the
893outer public API. The logic is that the "unchecked" case is useful
894when encoding is part of a stream which may be reporting errors
895(e.g. STDERR). In such cases it is desirable to get everything
896through somehow without causing additional errors which obscure the
897original one. Also the encoding is best placed to know what the
898correct replacement character is, so if that is the desired behaviour
899then letting low level code do it is the most efficient.
900
901In contrast if check is true, the scheme above allows the encoding to
902do as much as it can and tell layer above how much that was. What is
903lacking at present is a mechanism to report what went wrong. The most
904likely interface will be an additional method call to the object, or
905perhaps (to avoid forcing per-stream objects on otherwise stateless
906encodings) and additional parameter.
907
908It is also highly desirable that encoding classes inherit from
909C<Encode::Encoding> as a base class. This allows that class to define
910additional behaviour for all encoding objects. For example built in
911Unicode, UCS-2 and UTF-8 classes use :
51ef4e11 912
913 package Encode::MyEncoding;
914 use base qw(Encode::Encoding);
915
916 __PACKAGE__->Define(qw(myCanonical myAlias));
917
47bfe92f 918To create an object with bless {Name => ...},$class, and call
919define_encoding. They inherit their C<name> method from
920C<Encode::Encoding>.
4edaa979 921
922=head2 Compiled Encodings
923
47bfe92f 924F<Encode.xs> provides a class C<Encode::XS> which provides the
925interface described above. It calls a generic octet-sequence to
926octet-sequence "engine" that is driven by tables (defined in
927F<encengine.c>). The same engine is used for both encode and
928decode. C<Encode:XS>'s C<encode> forces Perl's characters to their
929UTF-8 form and then treats them as just another multibyte
930encoding. C<Encode:XS>'s C<decode> transforms the sequence and then
931turns the UTF-8-ness flag as that is the form that the tables are
932defined to produce. For details of the engine see the comments in
933F<encengine.c>.
934
935The tables are produced by the Perl script F<compile> (the name needs
936to change so we can eventually install it somewhere). F<compile> can
937currently read two formats:
4edaa979 938
939=over 4
940
941=item *.enc
942
47bfe92f 943This is a coined format used by Tcl. It is documented in
944Encode/EncodeFormat.pod.
4edaa979 945
946=item *.ucm
947
948This is the semi-standard format used by IBM's ICU package.
949
950=back
951
952F<compile> can write the following forms:
953
954=over 4
955
956=item *.ucm
957
958See above - the F<Encode/*.ucm> files provided with the distribution have
959been created from the original Tcl .enc files using this approach.
960
961=item *.c
962
963Produces tables as C data structures - this is used to build in encodings
964into F<Encode.so>/F<Encode.dll>.
965
966=item *.xs
967
47bfe92f 968In theory this allows encodings to be stand-alone loadable Perl
969extensions. The process has not yet been tested. The plan is to use
970this approach for large East Asian encodings.
4edaa979 971
972=back
973
47bfe92f 974The set of encodings built-in to F<Encode.so>/F<Encode.dll> is
975determined by F<Makefile.PL>. The current set is as follows:
4edaa979 976
977=over 4
978
979=item ascii and iso-8859-*
980
981That is all the common 8-bit "western" encodings.
982
983=item IBM-1047 and two other variants of EBCDIC.
984
47bfe92f 985These are the same variants that are supported by EBCDIC Perl as
986"native" encodings. They are included to prove "reversibility" of
987some constructs in EBCDIC Perl.
4edaa979 988
989=item symbol and dingbats as used by Tk on X11.
990
47bfe92f 991(The reason Encode got started was to support Perl/Tk.)
4edaa979 992
993=back
994
47bfe92f 995That set is rather ad hoc and has been driven by the needs of the
996tests rather than the needs of typical applications. It is likely
997to be rationalized.
4edaa979 998
4411f3b6 999=head1 SEE ALSO
1000
47bfe92f 1001L<perlunicode>, L<perlebcdic>, L<perlfunc/open>, L<PerlIO>
4411f3b6 1002
1003=cut
1004