Better fix from Autrijus.
[p5sagit/p5-mst-13.2.git] / ext / Encode / Encode.pm
CommitLineData
2c674647 1package Encode;
51ef4e11 2use strict;
0e567a6c 3our $VERSION = '0.40';
2c674647 4
5require DynaLoader;
6require Exporter;
7
51ef4e11 8our @ISA = qw(Exporter DynaLoader);
2c674647 9
4411f3b6 10# Public, encouraged API is exported by default
51ef4e11 11our @EXPORT = qw (
4411f3b6 12 encode
13 decode
14 encode_utf8
15 decode_utf8
16 find_encoding
51ef4e11 17 encodings
4411f3b6 18);
19
51ef4e11 20our @EXPORT_OK =
2c674647 21 qw(
51ef4e11 22 define_encoding
23 define_alias
2c674647 24 from_to
25 is_utf8
4411f3b6 26 is_8bit
27 is_16bit
a12c0f56 28 utf8_upgrade
29 utf8_downgrade
4411f3b6 30 _utf8_on
31 _utf8_off
2c674647 32 );
33
34bootstrap Encode ();
35
4411f3b6 36# Documentation moved after __END__ for speed - NI-S
2c674647 37
bf230f3d 38use Carp;
39
51ef4e11 40# Make a %encoding package variable to allow a certain amount of cheating
41our %encoding;
42my @alias; # ordered matching list
43my %alias; # cached known aliases
f7ac3676 44
6d6a7c8d 45 # 0 1 2 3 4 5 6 7 8 9 10
46our @latin2iso_num = ( 0, 1, 2, 3, 4, 9, 10, 13, 14, 15, 16 );
47
f7ac3676 48our %winlatin2cp = (
49 'Latin1' => 1252,
50 'Latin2' => 1250,
51 'Cyrillic' => 1251,
f7ac3676 52 'Greek' => 1253,
53 'Turkish' => 1254,
54 'Hebrew' => 1255,
55 'Arabic' => 1256,
56 'Baltic' => 1257,
57 'Vietnamese' => 1258,
58 );
5345d506 59
2b217bf7 60our %external_tables =
61 (
62 'euc-cn' => 'Encode/CN.pm',
63 gb2312 => 'Encode/CN.pm',
64 gb12345 => 'Encode/CN.pm',
65 gbk => 'Encode/CN.pm',
66 cp936 => 'Encode/CN.pm',
67 'iso-ir-165' => 'Encode/CN.pm',
68 'euc-jp' => 'Encode/JP.pm',
69 shiftjis => 'Encode/JP.pm',
70 macjapan => 'Encode/JP.pm',
71 cp932 => 'Encode/JP.pm',
72 'euc-kr' => 'Encode/KR.pm',
73 ksc5601 => 'Encode/KR.pm',
74 cp949 => 'Encode/KR.pm',
75 big5 => 'Encode/TW.pm',
76 'big5-hkscs' => 'Encode/TW.pm',
77 cp950 => 'Encode/TW.pm',
78 gb18030 => 'Encode/HanExtra.pm',
79 big5plus => 'Encode/HanExtra.pm',
80 'euc-tw' => 'Encode/HanExtra.pm',
81 );
d1ed7747 82
656753f8 83sub encodings
84{
85 my ($class) = @_;
40a073c6 86 return
87 map { $_->[0] }
88 sort { $a->[1] cmp $b->[1] }
89 map { [$_, lc $_] }
90 grep { $_ ne 'Internal' }
91 keys %encoding;
51ef4e11 92}
93
94sub findAlias
95{
18586f54 96 my $class = shift;
97 local $_ = shift;
98 # print "# findAlias $_\n";
99 unless (exists $alias{$_})
656753f8 100 {
18586f54 101 for (my $i=0; $i < @alias; $i += 2)
102 {
103 my $alias = $alias[$i];
104 my $val = $alias[$i+1];
105 my $new;
106 if (ref($alias) eq 'Regexp' && $_ =~ $alias)
107 {
108 $new = eval $val;
109 }
110 elsif (ref($alias) eq 'CODE')
111 {
112 $new = &{$alias}($val)
113 }
114 elsif (lc($_) eq lc($alias))
115 {
116 $new = $val;
117 }
118 if (defined($new))
119 {
120 next if $new eq $_; # avoid (direct) recursion on bugs
121 my $enc = (ref($new)) ? $new : find_encoding($new);
122 if ($enc)
123 {
124 $alias{$_} = $enc;
125 last;
126 }
127 }
128 }
656753f8 129 }
18586f54 130 return $alias{$_};
5345d506 131}
132
51ef4e11 133sub define_alias
5345d506 134{
18586f54 135 while (@_)
136 {
137 my ($alias,$name) = splice(@_,0,2);
138 push(@alias, $alias => $name);
139 }
51ef4e11 140}
141
016cb72c 142# Allow variants of iso-8859-1 etc.
d6089a2a 143define_alias( qr/^iso[-_]?(\d+)[-_](\d+)$/i => '"iso-$1-$2"' );
016cb72c 144
7faf300d 145# At least HP-UX has these.
146define_alias( qr/^iso8859(\d+)$/i => '"iso-8859-$1"' );
147
f7ac3676 148# More HP stuff.
149define_alias( qr/^(?:hp-)?(arabic|greek|hebrew|kana|roman|thai|turkish)8$/i => '"${1}8"' );
150
0b3236bb 151# The Official name of ASCII.
8a361256 152define_alias( qr/^ANSI[-_]?X3\.4[-_]?1968$/i => '"ascii"' );
153
58d53262 154# This is a font issue, not an encoding issue.
155# (The currency symbol of the Latin 1 upper half
156# has been redefined as the euro symbol.)
157define_alias( qr/^(.+)\@euro$/i => '"$1"' );
158
016cb72c 159# Allow latin-1 style names as well
7faf300d 160define_alias( qr/^(?:iso[-_]?)?latin[-_]?(\d+)$/i => '"iso-8859-$latin2iso_num[$1]"' );
016cb72c 161
f7ac3676 162# Allow winlatin1 style names as well
cf91068f 163define_alias( qr/^win(latin[12]|cyrillic|baltic|greek|turkish|hebrew|arabic|baltic|vietnamese)$/i => '"cp$winlatin2cp{\u$1}"' );
f7ac3676 164
016cb72c 165# Common names for non-latin prefered MIME names
166define_alias( 'ascii' => 'US-ascii',
167 'cyrillic' => 'iso-8859-5',
168 'arabic' => 'iso-8859-6',
169 'greek' => 'iso-8859-7',
f7ac3676 170 'hebrew' => 'iso-8859-8',
171 'thai' => 'iso-8859-11',
172 'tis620' => 'iso-8859-11',
173 );
016cb72c 174
7faf300d 175# At least AIX has IBM-NNN (surprisingly...) instead of cpNNN.
1853dd5f 176# And Microsoft has their own naming (again, surprisingly).
177define_alias( qr/^(?:ibm|ms)[-_]?(\d\d\d\d?)$/i => '"cp$1"');
178
179# Sometimes seen with a leading zero.
180define_alias( qr/^cp037$/i => '"cp37"');
181
182# Ououououou.
183define_alias( qr/^macRomanian$/i => '"macRumanian"');
7faf300d 184
58d53262 185# Standardize on the dashed versions.
186define_alias( qr/^utf8$/i => 'utf-8' );
7faf300d 187define_alias( qr/^koi8r$/i => 'koi8-r' );
f7ac3676 188define_alias( qr/^koi8u$/i => 'koi8-u' );
189
1853dd5f 190# Seen in some Linuxes.
191define_alias( qr/^ujis$/i => 'euc-jp' );
192
b2729934 193# CP936 doesn't have vendor-addon for GBK, so they're identical.
194define_alias( qr/^gbk$/i => '"cp936"');
195
f7ac3676 196# TODO: HP-UX '8' encodings arabic8 greek8 hebrew8 kana8 thai8 turkish8
197# TODO: HP-UX '15' encodings japanese15 korean15 roi15
198# TODO: Cyrillic encoding ISO-IR-111 (useful?)
f7ac3676 199# TODO: Armenian encoding ARMSCII-8
200# TODO: Hebrew encoding ISO-8859-8-1
201# TODO: Thai encoding TCVN
202# TODO: Korean encoding Johab
56a543c5 203# TODO: Vietnamese encodings VPS
f7ac3676 204# TODO: Japanese encoding JIS (not the same as SJIS)
205# TODO: Mac Asian+African encodings: Arabic Armenian Bengali Burmese
206# ChineseSimp ChineseTrad Devanagari Ethiopic ExtArabic
207# Farsi Georgian Gujarati Gurmukhi Hebrew Japanese
208# Kannada Khmer Korean Laotian Malayalam Mongolian
209# Oriya Sinhalese Symbol Tamil Telugu Tibetan Vietnamese
18586f54 210
1853dd5f 211# Map white space and _ to '-'
016cb72c 212define_alias( qr/^(\S+)[\s_]+(.*)$/i => '"$1-$2"' );
213
51ef4e11 214sub define_encoding
215{
18586f54 216 my $obj = shift;
217 my $name = shift;
218 $encoding{$name} = $obj;
219 my $lc = lc($name);
220 define_alias($lc => $obj) unless $lc eq $name;
221 while (@_)
222 {
223 my $alias = shift;
224 define_alias($alias,$obj);
225 }
226 return $obj;
656753f8 227}
228
656753f8 229sub getEncoding
230{
18586f54 231 my ($class,$name) = @_;
232 my $enc;
233 if (ref($name) && $name->can('new_sequence'))
234 {
235 return $name;
236 }
237 my $lc = lc $name;
238 if (exists $encoding{$name})
239 {
240 return $encoding{$name};
241 }
242 if (exists $encoding{$lc})
243 {
244 return $encoding{$lc};
245 }
d1ed7747 246 if (exists $external_tables{$lc})
247 {
248 require $external_tables{$lc};
249 return $encoding{$name} if exists $encoding{$name};
250 }
18586f54 251
252 my $oc = $class->findAlias($name);
253 return $oc if defined $oc;
254 return $class->findAlias($lc) if $lc ne $name;
255
256 return;
656753f8 257}
258
4411f3b6 259sub find_encoding
260{
18586f54 261 my ($name) = @_;
262 return __PACKAGE__->getEncoding($name);
4411f3b6 263}
264
265sub encode
266{
18586f54 267 my ($name,$string,$check) = @_;
268 my $enc = find_encoding($name);
269 croak("Unknown encoding '$name'") unless defined $enc;
270 my $octets = $enc->encode($string,$check);
271 return undef if ($check && length($string));
272 return $octets;
4411f3b6 273}
274
275sub decode
276{
18586f54 277 my ($name,$octets,$check) = @_;
278 my $enc = find_encoding($name);
279 croak("Unknown encoding '$name'") unless defined $enc;
280 my $string = $enc->decode($octets,$check);
281 $_[1] = $octets if $check;
282 return $string;
4411f3b6 283}
284
285sub from_to
286{
18586f54 287 my ($string,$from,$to,$check) = @_;
288 my $f = find_encoding($from);
289 croak("Unknown encoding '$from'") unless defined $f;
290 my $t = find_encoding($to);
291 croak("Unknown encoding '$to'") unless defined $t;
292 my $uni = $f->decode($string,$check);
293 return undef if ($check && length($string));
294 $string = $t->encode($uni,$check);
295 return undef if ($check && length($uni));
296 return length($_[0] = $string);
4411f3b6 297}
298
299sub encode_utf8
300{
18586f54 301 my ($str) = @_;
302 utf8::encode($str);
303 return $str;
4411f3b6 304}
305
306sub decode_utf8
307{
18586f54 308 my ($str) = @_;
309 return undef unless utf8::decode($str);
310 return $str;
5ad8ef52 311}
312
18586f54 313require Encode::Encoding;
314require Encode::XS;
315require Encode::Internal;
316require Encode::Unicode;
317require Encode::utf8;
318require Encode::iso10646_1;
319require Encode::ucs2_le;
4411f3b6 320
656753f8 3211;
322
2a936312 323__END__
324
4411f3b6 325=head1 NAME
326
327Encode - character encodings
328
329=head1 SYNOPSIS
330
331 use Encode;
332
333=head1 DESCRIPTION
334
47bfe92f 335The C<Encode> module provides the interfaces between Perl's strings
336and the rest of the system. Perl strings are sequences of B<characters>.
4411f3b6 337
338The repertoire of characters that Perl can represent is at least that
47bfe92f 339defined by the Unicode Consortium. On most platforms the ordinal
340values of the characters (as returned by C<ord(ch)>) is the "Unicode
341codepoint" for the character (the exceptions are those platforms where
342the legacy encoding is some variant of EBCDIC rather than a super-set
343of ASCII - see L<perlebcdic>).
4411f3b6 344
345Traditionaly computer data has been moved around in 8-bit chunks
346often called "bytes". These chunks are also known as "octets" in
347networking standards. Perl is widely used to manipulate data of
348many types - not only strings of characters representing human or
349computer languages but also "binary" data being the machines representation
350of numbers, pixels in an image - or just about anything.
351
47bfe92f 352When Perl is processing "binary data" the programmer wants Perl to process
353"sequences of bytes". This is not a problem for Perl - as a byte has 256
354possible values it easily fits in Perl's much larger "logical character".
4411f3b6 355
d1ed7747 356Due to size concerns, each of B<CJK> (Chinese, Japanese & Korean) modules
357are not loaded in memory until the first time they're used. Although you
358don't have to C<use> the corresponding B<Encode::>(B<TW>|B<CN>|B<JP>|B<KR>)
359modules first, be aware that those encodings will not be in C<%encodings>
360until their module is loaded (either implicitly through using encodings
361contained in the same module, or via an explicit C<use>).
a67efb5b 362
4411f3b6 363=head2 TERMINOLOGY
364
4ac9195f 365=over 4
4411f3b6 366
367=item *
368
369I<character>: a character in the range 0..(2**32-1) (or more).
47bfe92f 370(What Perl's strings are made of.)
4411f3b6 371
372=item *
373
374I<byte>: a character in the range 0..255
47bfe92f 375(A special case of a Perl character.)
4411f3b6 376
377=item *
378
379I<octet>: 8 bits of data, with ordinal values 0..255
47bfe92f 380(Term for bytes passed to or from a non-Perl context, e.g. disk file.)
4411f3b6 381
382=back
383
384The marker [INTERNAL] marks Internal Implementation Details, in
385general meant only for those who think they know what they are doing,
386and such details may change in future releases.
387
388=head1 ENCODINGS
389
390=head2 Characteristics of an Encoding
391
392An encoding has a "repertoire" of characters that it can represent,
393and for each representable character there is at least one sequence of
394octets that represents it.
395
396=head2 Types of Encodings
397
398Encodings can be divided into the following types:
399
400=over 4
401
402=item * Fixed length 8-bit (or less) encodings.
403
404Each character is a single octet so may have a repertoire of up to
405256 characters. ASCII and iso-8859-* are typical examples.
406
407=item * Fixed length 16-bit encodings
408
409Each character is two octets so may have a repertoire of up to
47bfe92f 41065 536 characters. Unicode's UCS-2 is an example. Also used for
4411f3b6 411encodings for East Asian languages.
412
413=item * Fixed length 32-bit encodings.
414
415Not really very "encoded" encodings. The Unicode code points
416are just represented as 4-octet integers. None the less because
417different architectures use different representations of integers
418(so called "endian") there at least two disctinct encodings.
419
420=item * Multi-byte encodings
421
422The number of octets needed to represent a character varies.
423UTF-8 is a particularly complex but regular case of a multi-byte
424encoding. Several East Asian countries use a multi-byte encoding
425where 1-octet is used to cover western roman characters and Asian
426characters get 2-octets.
427(UTF-16 is strictly a multi-byte encoding taking either 2 or 4 octets
428to represent a Unicode code point.)
429
430=item * "Escape" encodings.
431
432These encodings embed "escape sequences" into the octet sequence
433which describe how the following octets are to be interpreted.
434The iso-2022-* family is typical. Following the escape sequence
435octets are encoded by an "embedded" encoding (which will be one
436of the above types) until another escape sequence switches to
437a different "embedded" encoding.
438
439These schemes are very flexible and can handle mixed languages but are
47bfe92f 440very complex to process (and have state). No escape encodings are
441implemented for Perl yet.
4411f3b6 442
443=back
444
445=head2 Specifying Encodings
446
447Encodings can be specified to the API described below in two ways:
448
449=over 4
450
451=item 1. By name
452
47bfe92f 453Encoding names are strings with characters taken from a restricted
454repertoire. See L</"Encoding Names">.
4411f3b6 455
456=item 2. As an object
457
458Encoding objects are returned by C<find_encoding($name)>.
459
460=back
461
462=head2 Encoding Names
463
464Encoding names are case insensitive. White space in names is ignored.
47bfe92f 465In addition an encoding may have aliases. Each encoding has one
466"canonical" name. The "canonical" name is chosen from the names of
467the encoding by picking the first in the following sequence:
4411f3b6 468
469=over 4
470
78255929 471=item * The MIME name as defined in IETF RFCs.
4411f3b6 472
473=item * The name in the IANA registry.
474
d1be9408 475=item * The name used by the organization that defined it.
4411f3b6 476
477=back
478
479Because of all the alias issues, and because in the general case
480encodings have state C<Encode> uses the encoding object internally
481once an operation is in progress.
482
21938dfa 483As of Perl 5.8.0, at least the following encodings are recognized
484(the => marks aliases):
485
486 ASCII
487
488 US-ASCII => ASCII
489
490The Unicode:
491
0b3236bb 492 UTF-8
21938dfa 493 UTF-16
494 UCS-2
495
496 ISO 10646-1 => UCS-2
497
498The ISO 8859 and KOI:
499
500 ISO 8859-1 ISO 8859-6 ISO 8859-11 KOI8-F
501 ISO 8859-2 ISO 8859-7 (12 doesn't exist) KOI8-R
56a543c5 502 ISO 8859-3 ISO 8859-8 ISO 8859-13 KOI8-U
21938dfa 503 ISO 8859-4 ISO 8859-9 ISO 8859-14
504 ISO 8859-5 ISO 8859-10 ISO 8859-15
505 ISO 8859-16
506
507 Latin1 => 8859-1 Latin6 => 8859-10
508 Latin2 => 8859-2 Latin7 => 8859-13
0b3236bb 509 Latin3 => 8859-3 Latin8 => 8859-14
21938dfa 510 Latin4 => 8859-4 Latin9 => 8859-15
511 Latin5 => 8859-9 Latin10 => 8859-16
512
513 Cyrillic => 8859-5
514 Arabic => 8859-6
515 Greek => 8859-7
516 Hebrew => 8859-8
517 Thai => 8859-11
0b3236bb 518 TIS620 => 8859-11
21938dfa 519
520The CJKV: Chinese, Japanese, Korean, Vietnamese:
521
0b3236bb 522 ISO 2022 ISO 2022 JP-1 JIS 0201 GB 1988 Big5 EUC-CN
523 ISO 2022 CN ISO 2022 JP-2 JIS 0208 GB 2312 HZ EUC-JP
21938dfa 524 ISO 2022 JP ISO 2022 KR JIS 0210 GB 12345 CNS 11643 EUC-JP-0212
cb448690 525 Shift-JIS GBK Big5-HKSCS EUC-KR
526 VISCII ISO-IR-165
527
a67efb5b 528(Due to size concerns, additional Chinese encodings including C<GB 18030>,
529C<EUC-TW> and C<BIG5PLUS> are distributed separately on CPAN, under the name
530L<Encode::HanExtra>.)
21938dfa 531
532The PC codepages:
533
534 CP37 CP852 CP861 CP866 CP949 CP1251 CP1256
535 CP424 CP855 CP862 CP869 CP950 CP1252 CP1257
536 CP737 CP856 CP863 CP874 CP1006 CP1253 CP1258
537 CP775 CP857 CP864 CP932 CP1047 CP1254
538 CP850 CP860 CP865 CP936 CP1250 CP1255
539
540 WinLatin1 => CP1252
541 WinLatin2 => CP1250
542 WinCyrillic => CP1251
543 WinGreek => CP1253
544 WinTurkiskh => CP1254
545 WinHebrew => CP1255
546 WinArabic => CP1256
547 WinBaltic => CP1257
548 WinVietnamese => CP1258
549
4a42e14c 550(All the CPI<NNN...> are available also as IBMI<NNN...>.)
21938dfa 551
552The Mac codepages:
553
0b3236bb 554 MacCentralEuropean MacJapanese
555 MacCroatian MacRoman
1853dd5f 556 MacCyrillic MacRomanian
0b3236bb 557 MacDingbats MacSami
558 MacGreek MacThai
559 MacIcelandic MacTurkish
560 MacUkraine
21938dfa 561
562Miscellaneous:
563
564 7bit-greek IR-197
565 7bit-kana NeXTstep
566 7bit-latin1 POSIX-BC
567 DingBats Roman8
568 GSM 0338 Symbol
569
4411f3b6 570=head1 PERL ENCODING API
571
572=head2 Generic Encoding Interface
573
574=over 4
575
576=item *
577
578 $bytes = encode(ENCODING, $string[, CHECK])
579
47bfe92f 580Encodes string from Perl's internal form into I<ENCODING> and returns
581a sequence of octets. For CHECK see L</"Handling Malformed Data">.
4411f3b6 582
681a7c68 583For example to convert (internally UTF-8 encoded) Unicode data
584to octets:
585
586 $octets = encode("utf8", $unicode);
587
4411f3b6 588=item *
589
590 $string = decode(ENCODING, $bytes[, CHECK])
591
47bfe92f 592Decode sequence of octets assumed to be in I<ENCODING> into Perl's
593internal form and returns the resulting string. For CHECK see
594L</"Handling Malformed Data">.
595
681a7c68 596For example to convert ISO 8859-1 data to UTF-8:
597
598 $utf8 = decode("latin1", $latin1);
599
47bfe92f 600=item *
601
602 from_to($string, FROM_ENCODING, TO_ENCODING[, CHECK])
603
2b106fbe 604Convert B<in-place> the data between two encodings. How did the data
605in $string originally get to be in FROM_ENCODING? Either using
e9692b5b 606encode() or through PerlIO: See L</"Encoding and IO">. For CHECK
2b106fbe 607see L</"Handling Malformed Data">.
608
609For example to convert ISO 8859-1 data to UTF-8:
610
611 from_to($data, "iso-8859-1", "utf-8");
612
613and to convert it back:
614
615 from_to($data, "utf-8", "iso-8859-1");
4411f3b6 616
ab97ca19 617Note that because the conversion happens in place, the data to be
618converted cannot be a string constant, it must be a scalar variable.
619
4411f3b6 620=back
621
622=head2 Handling Malformed Data
623
624If CHECK is not set, C<undef> is returned. If the data is supposed to
47bfe92f 625be UTF-8, an optional lexical warning (category utf8) is given. If
626CHECK is true but not a code reference, dies.
4411f3b6 627
47bfe92f 628It would desirable to have a way to indicate that transform should use
629the encodings "replacement character" - no such mechanism is defined yet.
4411f3b6 630
631It is also planned to allow I<CHECK> to be a code reference.
632
47bfe92f 633This is not yet implemented as there are design issues with what its
634arguments should be and how it returns its results.
4411f3b6 635
636=over 4
637
638=item Scheme 1
639
640Passed remaining fragment of string being processed.
641Modifies it in place to remove bytes/characters it can understand
642and returns a string used to represent them.
643e.g.
644
645 sub fixup {
646 my $ch = substr($_[0],0,1,'');
647 return sprintf("\x{%02X}",ord($ch);
648 }
649
650This scheme is close to how underlying C code for Encode works, but gives
651the fixup routine very little context.
652
653=item Scheme 2
654
47bfe92f 655Passed original string, and an index into it of the problem area, and
656output string so far. Appends what it will to output string and
657returns new index into original string. For example:
4411f3b6 658
659 sub fixup {
660 # my ($s,$i,$d) = @_;
661 my $ch = substr($_[0],$_[1],1);
662 $_[2] .= sprintf("\x{%02X}",ord($ch);
663 return $_[1]+1;
664 }
665
47bfe92f 666This scheme gives maximal control to the fixup routine but is more
667complicated to code, and may need internals of Encode to be tweaked to
668keep original string intact.
4411f3b6 669
670=item Other Schemes
671
672Hybrids of above.
673
674Multiple return values rather than in-place modifications.
675
676Index into the string could be pos($str) allowing s/\G...//.
677
678=back
679
680=head2 UTF-8 / utf8
681
682The Unicode consortium defines the UTF-8 standard as a way of encoding
47bfe92f 683the entire Unicode repertiore as sequences of octets. This encoding is
684expected to become very widespread. Perl can use this form internaly
685to represent strings, so conversions to and from this form are
686particularly efficient (as octets in memory do not have to change,
687just the meta-data that tells Perl how to treat them).
4411f3b6 688
689=over 4
690
691=item *
692
693 $bytes = encode_utf8($string);
694
47bfe92f 695The characters that comprise string are encoded in Perl's superset of UTF-8
4411f3b6 696and the resulting octets returned as a sequence of bytes. All possible
697characters have a UTF-8 representation so this function cannot fail.
698
699=item *
700
701 $string = decode_utf8($bytes [,CHECK]);
702
47bfe92f 703The sequence of octets represented by $bytes is decoded from UTF-8
704into a sequence of logical characters. Not all sequences of octets
705form valid UTF-8 encodings, so it is possible for this call to fail.
706For CHECK see L</"Handling Malformed Data">.
4411f3b6 707
708=back
709
710=head2 Other Encodings of Unicode
711
47bfe92f 712UTF-16 is similar to UCS-2, 16 bit or 2-byte chunks. UCS-2 can only
7a4efbb2 713represent 0..0xFFFF, while UTF-16 has a I<surrogate pair> scheme which
47bfe92f 714allows it to cover the whole Unicode range.
4411f3b6 715
7a4efbb2 716Surrogates are code points set aside to encode the 0x01000..0x10FFFF
717range of Unicode code points in pairs of 16-bit units. The I<high
718surrogates> are the range 0xD800..0xDBFF, and the I<low surrogates>
719are the range 0xDC00..0xDFFFF. The surrogate encoding is
720
721 $hi = ($uni - 0x10000) / 0x400 + 0xD800;
722 $lo = ($uni - 0x10000) % 0x400 + 0xDC00;
723
724and the decoding is
725
726 $uni = 0x10000 + ($hi - 0xD8000) * 0x400 + ($lo - 0xDC00);
727
8040349a 728Encode implements big-endian UCS-2 aliased to "iso-10646-1" as that
47bfe92f 729happens to be the name used by that representation when used with X11
730fonts.
4411f3b6 731
732UTF-32 or UCS-4 is 32-bit or 4-byte chunks. Perl's logical characters
733can be considered as being in this form without encoding. An encoding
47bfe92f 734to transfer strings in this form (e.g. to write them to a file) would
735need to
4411f3b6 736
c079d275 737 pack('L*', unpack('U*', $string)); # native
4411f3b6 738 or
c079d275 739 pack('V*', unpack('U*', $string)); # little-endian
4411f3b6 740 or
c079d275 741 pack('N*', unpack('U*', $string)); # big-endian
4411f3b6 742
c079d275 743depending on the endianness required.
4411f3b6 744
51ef4e11 745No UTF-32 encodings are implemented yet.
4411f3b6 746
47bfe92f 747Both UCS-2 and UCS-4 style encodings can have "byte order marks" by
748representing the code point 0xFFFE as the very first thing in a file.
4411f3b6 749
51ef4e11 750=head2 Listing available encodings
751
752 use Encode qw(encodings);
753 @list = encodings();
754
755Returns a list of the canonical names of the available encodings.
756
757=head2 Defining Aliases
758
759 use Encode qw(define_alias);
760 define_alias( newName => ENCODING);
761
47bfe92f 762Allows newName to be used as am alias for ENCODING. ENCODING may be
763either the name of an encoding or and encoding object (as above).
51ef4e11 764
765Currently I<newName> can be specified in the following ways:
766
767=over 4
768
769=item As a simple string.
770
771=item As a qr// compiled regular expression, e.g.:
772
773 define_alias( qr/^iso8859-(\d+)$/i => '"iso-8859-$1"' );
774
47bfe92f 775In this case if I<ENCODING> is not a reference it is C<eval>-ed to
776allow C<$1> etc. to be subsituted. The example is one way to names as
777used in X11 font names to alias the MIME names for the iso-8859-*
778family.
51ef4e11 779
780=item As a code reference, e.g.:
781
782 define_alias( sub { return /^iso8859-(\d+)$/i ? "iso-8859-$1" : undef } , '');
783
784In this case C<$_> will be set to the name that is being looked up and
47bfe92f 785I<ENCODING> is passed to the sub as its first argument. The example
786is another way to names as used in X11 font names to alias the MIME
787names for the iso-8859-* family.
51ef4e11 788
789=back
790
791=head2 Defining Encodings
792
e9692b5b 793 use Encode qw(define_alias);
794 define_encoding( $object, 'canonicalName' [,alias...]);
51ef4e11 795
47bfe92f 796Causes I<canonicalName> to be associated with I<$object>. The object
797should provide the interface described in L</"IMPLEMENTATION CLASSES">
798below. If more than two arguments are provided then additional
799arguments are taken as aliases for I<$object> as for C<define_alias>.
51ef4e11 800
4411f3b6 801=head1 Encoding and IO
802
803It is very common to want to do encoding transformations when
804reading or writing files, network connections, pipes etc.
47bfe92f 805If Perl is configured to use the new 'perlio' IO system then
4411f3b6 806C<Encode> provides a "layer" (See L<perliol>) which can transform
807data as it is read or written.
808
8e86646e 809Here is how the blind poet would modernise the encoding:
810
42234700 811 use Encode;
8e86646e 812 open(my $iliad,'<:encoding(iso-8859-7)','iliad.greek');
813 open(my $utf8,'>:utf8','iliad.utf8');
814 my @epic = <$iliad>;
815 print $utf8 @epic;
816 close($utf8);
817 close($illiad);
4411f3b6 818
819In addition the new IO system can also be configured to read/write
820UTF-8 encoded characters (as noted above this is efficient):
821
e9692b5b 822 open(my $fh,'>:utf8','anything');
823 print $fh "Any \x{0021} string \N{SMILEY FACE}\n";
4411f3b6 824
825Either of the above forms of "layer" specifications can be made the default
826for a lexical scope with the C<use open ...> pragma. See L<open>.
827
828Once a handle is open is layers can be altered using C<binmode>.
829
47bfe92f 830Without any such configuration, or if Perl itself is built using
4411f3b6 831system's own IO, then write operations assume that file handle accepts
832only I<bytes> and will C<die> if a character larger than 255 is
833written to the handle. When reading, each octet from the handle
834becomes a byte-in-a-character. Note that this default is the same
47bfe92f 835behaviour as bytes-only languages (including Perl before v5.6) would
836have, and is sufficient to handle native 8-bit encodings
837e.g. iso-8859-1, EBCDIC etc. and any legacy mechanisms for handling
838other encodings and binary data.
839
840In other cases it is the programs responsibility to transform
841characters into bytes using the API above before doing writes, and to
842transform the bytes read from a handle into characters before doing
843"character operations" (e.g. C<lc>, C</\W+/>, ...).
844
47bfe92f 845You can also use PerlIO to convert larger amounts of data you don't
846want to bring into memory. For example to convert between ISO 8859-1
847(Latin 1) and UTF-8 (or UTF-EBCDIC in EBCDIC machines):
848
e9692b5b 849 open(F, "<:encoding(iso-8859-1)", "data.txt") or die $!;
850 open(G, ">:utf8", "data.utf") or die $!;
851 while (<F>) { print G }
852
853 # Could also do "print G <F>" but that would pull
854 # the whole file into memory just to write it out again.
855
856More examples:
47bfe92f 857
e9692b5b 858 open(my $f, "<:encoding(cp1252)")
859 open(my $g, ">:encoding(iso-8859-2)")
860 open(my $h, ">:encoding(latin9)") # iso-8859-15
47bfe92f 861
862See L<PerlIO> for more information.
4411f3b6 863
1768d7eb 864See also L<encoding> for how to change the default encoding of the
d521382b 865data in your script.
1768d7eb 866
4411f3b6 867=head1 Encoding How to ...
868
869To do:
870
871=over 4
872
873=item * IO with mixed content (faking iso-2020-*)
874
875=item * MIME's Content-Length:
876
877=item * UTF-8 strings in binary data.
878
47bfe92f 879=item * Perl/Encode wrappers on non-Unicode XS modules.
4411f3b6 880
881=back
882
883=head1 Messing with Perl's Internals
884
47bfe92f 885The following API uses parts of Perl's internals in the current
886implementation. As such they are efficient, but may change.
4411f3b6 887
888=over 4
889
4411f3b6 890=item * is_utf8(STRING [, CHECK])
891
892[INTERNAL] Test whether the UTF-8 flag is turned on in the STRING.
47bfe92f 893If CHECK is true, also checks the data in STRING for being well-formed
894UTF-8. Returns true if successful, false otherwise.
4411f3b6 895
896=item * valid_utf8(STRING)
897
47bfe92f 898[INTERNAL] Test whether STRING is in a consistent state. Will return
899true if string is held as bytes, or is well-formed UTF-8 and has the
900UTF-8 flag on. Main reason for this routine is to allow Perl's
901testsuite to check that operations have left strings in a consistent
902state.
4411f3b6 903
904=item *
905
906 _utf8_on(STRING)
907
908[INTERNAL] Turn on the UTF-8 flag in STRING. The data in STRING is
909B<not> checked for being well-formed UTF-8. Do not use unless you
910B<know> that the STRING is well-formed UTF-8. Returns the previous
911state of the UTF-8 flag (so please don't test the return value as
912I<not> success or failure), or C<undef> if STRING is not a string.
913
914=item *
915
916 _utf8_off(STRING)
917
918[INTERNAL] Turn off the UTF-8 flag in STRING. Do not use frivolously.
919Returns the previous state of the UTF-8 flag (so please don't test the
920return value as I<not> success or failure), or C<undef> if STRING is
921not a string.
922
923=back
924
4edaa979 925=head1 IMPLEMENTATION CLASSES
926
927As mentioned above encodings are (in the current implementation at least)
928defined by objects. The mapping of encoding name to object is via the
51ef4e11 929C<%encodings> hash.
4edaa979 930
931The values of the hash can currently be either strings or objects.
932The string form may go away in the future. The string form occurs
933when C<encodings()> has scanned C<@INC> for loadable encodings but has
934not actually loaded the encoding in question. This is because the
47bfe92f 935current "loading" process is all Perl and a bit slow.
4edaa979 936
47bfe92f 937Once an encoding is loaded then value of the hash is object which
938implements the encoding. The object should provide the following
939interface:
4edaa979 940
941=over 4
942
943=item -E<gt>name
944
945Should return the string representing the canonical name of the encoding.
946
947=item -E<gt>new_sequence
948
47bfe92f 949This is a placeholder for encodings with state. It should return an
950object which implements this interface, all current implementations
951return the original object.
4edaa979 952
953=item -E<gt>encode($string,$check)
954
47bfe92f 955Should return the octet sequence representing I<$string>. If I<$check>
956is true it should modify I<$string> in place to remove the converted
957part (i.e. the whole string unless there is an error). If an error
958occurs it should return the octet sequence for the fragment of string
959that has been converted, and modify $string in-place to remove the
960converted part leaving it starting with the problem fragment.
4edaa979 961
47bfe92f 962If check is is false then C<encode> should make a "best effort" to
963convert the string - for example by using a replacement character.
4edaa979 964
965=item -E<gt>decode($octets,$check)
966
47bfe92f 967Should return the string that I<$octets> represents. If I<$check> is
968true it should modify I<$octets> in place to remove the converted part
969(i.e. the whole sequence unless there is an error). If an error
970occurs it should return the fragment of string that has been
971converted, and modify $octets in-place to remove the converted part
4edaa979 972leaving it starting with the problem fragment.
973
47bfe92f 974If check is is false then C<decode> should make a "best effort" to
975convert the string - for example by using Unicode's "\x{FFFD}" as a
976replacement character.
4edaa979 977
978=back
979
47bfe92f 980It should be noted that the check behaviour is different from the
981outer public API. The logic is that the "unchecked" case is useful
982when encoding is part of a stream which may be reporting errors
983(e.g. STDERR). In such cases it is desirable to get everything
984through somehow without causing additional errors which obscure the
985original one. Also the encoding is best placed to know what the
986correct replacement character is, so if that is the desired behaviour
987then letting low level code do it is the most efficient.
988
989In contrast if check is true, the scheme above allows the encoding to
990do as much as it can and tell layer above how much that was. What is
991lacking at present is a mechanism to report what went wrong. The most
992likely interface will be an additional method call to the object, or
993perhaps (to avoid forcing per-stream objects on otherwise stateless
994encodings) and additional parameter.
995
996It is also highly desirable that encoding classes inherit from
997C<Encode::Encoding> as a base class. This allows that class to define
998additional behaviour for all encoding objects. For example built in
999Unicode, UCS-2 and UTF-8 classes use :
51ef4e11 1000
1001 package Encode::MyEncoding;
1002 use base qw(Encode::Encoding);
1003
1004 __PACKAGE__->Define(qw(myCanonical myAlias));
1005
47bfe92f 1006To create an object with bless {Name => ...},$class, and call
1007define_encoding. They inherit their C<name> method from
1008C<Encode::Encoding>.
4edaa979 1009
1010=head2 Compiled Encodings
1011
47bfe92f 1012F<Encode.xs> provides a class C<Encode::XS> which provides the
1013interface described above. It calls a generic octet-sequence to
1014octet-sequence "engine" that is driven by tables (defined in
1015F<encengine.c>). The same engine is used for both encode and
1016decode. C<Encode:XS>'s C<encode> forces Perl's characters to their
1017UTF-8 form and then treats them as just another multibyte
1018encoding. C<Encode:XS>'s C<decode> transforms the sequence and then
1019turns the UTF-8-ness flag as that is the form that the tables are
1020defined to produce. For details of the engine see the comments in
1021F<encengine.c>.
1022
1023The tables are produced by the Perl script F<compile> (the name needs
1024to change so we can eventually install it somewhere). F<compile> can
1025currently read two formats:
4edaa979 1026
1027=over 4
1028
1029=item *.enc
1030
47bfe92f 1031This is a coined format used by Tcl. It is documented in
1032Encode/EncodeFormat.pod.
4edaa979 1033
1034=item *.ucm
1035
1036This is the semi-standard format used by IBM's ICU package.
1037
1038=back
1039
1040F<compile> can write the following forms:
1041
1042=over 4
1043
1044=item *.ucm
1045
1046See above - the F<Encode/*.ucm> files provided with the distribution have
1047been created from the original Tcl .enc files using this approach.
1048
1049=item *.c
1050
1051Produces tables as C data structures - this is used to build in encodings
1052into F<Encode.so>/F<Encode.dll>.
1053
1054=item *.xs
1055
47bfe92f 1056In theory this allows encodings to be stand-alone loadable Perl
1057extensions. The process has not yet been tested. The plan is to use
1058this approach for large East Asian encodings.
4edaa979 1059
1060=back
1061
47bfe92f 1062The set of encodings built-in to F<Encode.so>/F<Encode.dll> is
1063determined by F<Makefile.PL>. The current set is as follows:
4edaa979 1064
1065=over 4
1066
1067=item ascii and iso-8859-*
1068
1069That is all the common 8-bit "western" encodings.
1070
1071=item IBM-1047 and two other variants of EBCDIC.
1072
47bfe92f 1073These are the same variants that are supported by EBCDIC Perl as
1074"native" encodings. They are included to prove "reversibility" of
1075some constructs in EBCDIC Perl.
4edaa979 1076
1077=item symbol and dingbats as used by Tk on X11.
1078
47bfe92f 1079(The reason Encode got started was to support Perl/Tk.)
4edaa979 1080
1081=back
1082
47bfe92f 1083That set is rather ad hoc and has been driven by the needs of the
1084tests rather than the needs of typical applications. It is likely
1085to be rationalized.
4edaa979 1086
4411f3b6 1087=head1 SEE ALSO
1088
1768d7eb 1089L<perlunicode>, L<perlebcdic>, L<perlfunc/open>, L<PerlIO>, L<encoding>
4411f3b6 1090
1091=cut
1092