Commit | Line | Data |
2c674647 |
1 | package Encode; |
51ef4e11 |
2 | use strict; |
ee981de6 |
3 | our $VERSION = do { my @r = (q$Revision: 0.90 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; |
2c674647 |
4 | |
5 | require DynaLoader; |
6 | require Exporter; |
7 | |
51ef4e11 |
8 | our @ISA = qw(Exporter DynaLoader); |
2c674647 |
9 | |
4411f3b6 |
10 | # Public, encouraged API is exported by default |
51ef4e11 |
11 | our @EXPORT = qw ( |
4411f3b6 |
12 | encode |
13 | decode |
14 | encode_utf8 |
15 | decode_utf8 |
16 | find_encoding |
51ef4e11 |
17 | encodings |
4411f3b6 |
18 | ); |
19 | |
51ef4e11 |
20 | our @EXPORT_OK = |
2c674647 |
21 | qw( |
51ef4e11 |
22 | define_encoding |
23 | define_alias |
2c674647 |
24 | from_to |
25 | is_utf8 |
4411f3b6 |
26 | is_8bit |
27 | is_16bit |
a12c0f56 |
28 | utf8_upgrade |
29 | utf8_downgrade |
4411f3b6 |
30 | _utf8_on |
31 | _utf8_off |
2c674647 |
32 | ); |
33 | |
34 | bootstrap Encode (); |
35 | |
4411f3b6 |
36 | # Documentation moved after __END__ for speed - NI-S |
2c674647 |
37 | |
bf230f3d |
38 | use Carp; |
39 | |
51ef4e11 |
40 | # Make a %encoding package variable to allow a certain amount of cheating |
41 | our %encoding; |
42 | my @alias; # ordered matching list |
43 | my %alias; # cached known aliases |
f7ac3676 |
44 | |
6d6a7c8d |
45 | # 0 1 2 3 4 5 6 7 8 9 10 |
46 | our @latin2iso_num = ( 0, 1, 2, 3, 4, 9, 10, 13, 14, 15, 16 ); |
47 | |
f7ac3676 |
48 | our %winlatin2cp = ( |
49 | 'Latin1' => 1252, |
50 | 'Latin2' => 1250, |
51 | 'Cyrillic' => 1251, |
f7ac3676 |
52 | 'Greek' => 1253, |
53 | 'Turkish' => 1254, |
54 | 'Hebrew' => 1255, |
55 | 'Arabic' => 1256, |
56 | 'Baltic' => 1257, |
57 | 'Vietnamese' => 1258, |
58 | ); |
5345d506 |
59 | |
2b217bf7 |
60 | our %external_tables = |
61 | ( |
62 | 'euc-cn' => 'Encode/CN.pm', |
63 | gb2312 => 'Encode/CN.pm', |
64 | gb12345 => 'Encode/CN.pm', |
65 | gbk => 'Encode/CN.pm', |
66 | cp936 => 'Encode/CN.pm', |
67 | 'iso-ir-165' => 'Encode/CN.pm', |
68 | 'euc-jp' => 'Encode/JP.pm', |
ee981de6 |
69 | 'iso-2022-jp' => 'Encode/JP.pm', |
70 | '7bit-jis' => 'Encode/JP.pm', |
2b217bf7 |
71 | shiftjis => 'Encode/JP.pm', |
72 | macjapan => 'Encode/JP.pm', |
73 | cp932 => 'Encode/JP.pm', |
74 | 'euc-kr' => 'Encode/KR.pm', |
75 | ksc5601 => 'Encode/KR.pm', |
76 | cp949 => 'Encode/KR.pm', |
77 | big5 => 'Encode/TW.pm', |
78 | 'big5-hkscs' => 'Encode/TW.pm', |
79 | cp950 => 'Encode/TW.pm', |
80 | gb18030 => 'Encode/HanExtra.pm', |
81 | big5plus => 'Encode/HanExtra.pm', |
82 | 'euc-tw' => 'Encode/HanExtra.pm', |
83 | ); |
d1ed7747 |
84 | |
656753f8 |
85 | sub encodings |
86 | { |
87 | my ($class) = @_; |
40a073c6 |
88 | return |
89 | map { $_->[0] } |
90 | sort { $a->[1] cmp $b->[1] } |
91 | map { [$_, lc $_] } |
92 | grep { $_ ne 'Internal' } |
93 | keys %encoding; |
51ef4e11 |
94 | } |
95 | |
96 | sub findAlias |
97 | { |
18586f54 |
98 | my $class = shift; |
99 | local $_ = shift; |
100 | # print "# findAlias $_\n"; |
101 | unless (exists $alias{$_}) |
656753f8 |
102 | { |
18586f54 |
103 | for (my $i=0; $i < @alias; $i += 2) |
104 | { |
105 | my $alias = $alias[$i]; |
106 | my $val = $alias[$i+1]; |
107 | my $new; |
108 | if (ref($alias) eq 'Regexp' && $_ =~ $alias) |
109 | { |
110 | $new = eval $val; |
111 | } |
112 | elsif (ref($alias) eq 'CODE') |
113 | { |
114 | $new = &{$alias}($val) |
115 | } |
116 | elsif (lc($_) eq lc($alias)) |
117 | { |
118 | $new = $val; |
119 | } |
120 | if (defined($new)) |
121 | { |
122 | next if $new eq $_; # avoid (direct) recursion on bugs |
123 | my $enc = (ref($new)) ? $new : find_encoding($new); |
124 | if ($enc) |
125 | { |
126 | $alias{$_} = $enc; |
127 | last; |
128 | } |
129 | } |
130 | } |
656753f8 |
131 | } |
18586f54 |
132 | return $alias{$_}; |
5345d506 |
133 | } |
134 | |
51ef4e11 |
135 | sub define_alias |
5345d506 |
136 | { |
18586f54 |
137 | while (@_) |
138 | { |
139 | my ($alias,$name) = splice(@_,0,2); |
140 | push(@alias, $alias => $name); |
141 | } |
51ef4e11 |
142 | } |
143 | |
016cb72c |
144 | # Allow variants of iso-8859-1 etc. |
d6089a2a |
145 | define_alias( qr/^iso[-_]?(\d+)[-_](\d+)$/i => '"iso-$1-$2"' ); |
016cb72c |
146 | |
7faf300d |
147 | # At least HP-UX has these. |
148 | define_alias( qr/^iso8859(\d+)$/i => '"iso-8859-$1"' ); |
149 | |
f7ac3676 |
150 | # More HP stuff. |
151 | define_alias( qr/^(?:hp-)?(arabic|greek|hebrew|kana|roman|thai|turkish)8$/i => '"${1}8"' ); |
152 | |
0b3236bb |
153 | # The Official name of ASCII. |
8a361256 |
154 | define_alias( qr/^ANSI[-_]?X3\.4[-_]?1968$/i => '"ascii"' ); |
155 | |
58d53262 |
156 | # This is a font issue, not an encoding issue. |
157 | # (The currency symbol of the Latin 1 upper half |
158 | # has been redefined as the euro symbol.) |
159 | define_alias( qr/^(.+)\@euro$/i => '"$1"' ); |
160 | |
016cb72c |
161 | # Allow latin-1 style names as well |
7faf300d |
162 | define_alias( qr/^(?:iso[-_]?)?latin[-_]?(\d+)$/i => '"iso-8859-$latin2iso_num[$1]"' ); |
016cb72c |
163 | |
f7ac3676 |
164 | # Allow winlatin1 style names as well |
cf91068f |
165 | define_alias( qr/^win(latin[12]|cyrillic|baltic|greek|turkish|hebrew|arabic|baltic|vietnamese)$/i => '"cp$winlatin2cp{\u$1}"' ); |
f7ac3676 |
166 | |
016cb72c |
167 | # Common names for non-latin prefered MIME names |
168 | define_alias( 'ascii' => 'US-ascii', |
169 | 'cyrillic' => 'iso-8859-5', |
170 | 'arabic' => 'iso-8859-6', |
171 | 'greek' => 'iso-8859-7', |
f7ac3676 |
172 | 'hebrew' => 'iso-8859-8', |
173 | 'thai' => 'iso-8859-11', |
174 | 'tis620' => 'iso-8859-11', |
175 | ); |
016cb72c |
176 | |
7faf300d |
177 | # At least AIX has IBM-NNN (surprisingly...) instead of cpNNN. |
1853dd5f |
178 | # And Microsoft has their own naming (again, surprisingly). |
179 | define_alias( qr/^(?:ibm|ms)[-_]?(\d\d\d\d?)$/i => '"cp$1"'); |
180 | |
181 | # Sometimes seen with a leading zero. |
182 | define_alias( qr/^cp037$/i => '"cp37"'); |
183 | |
184 | # Ououououou. |
185 | define_alias( qr/^macRomanian$/i => '"macRumanian"'); |
7faf300d |
186 | |
58d53262 |
187 | # Standardize on the dashed versions. |
188 | define_alias( qr/^utf8$/i => 'utf-8' ); |
7faf300d |
189 | define_alias( qr/^koi8r$/i => 'koi8-r' ); |
f7ac3676 |
190 | define_alias( qr/^koi8u$/i => 'koi8-u' ); |
191 | |
1853dd5f |
192 | # Seen in some Linuxes. |
193 | define_alias( qr/^ujis$/i => 'euc-jp' ); |
194 | |
b2729934 |
195 | # CP936 doesn't have vendor-addon for GBK, so they're identical. |
196 | define_alias( qr/^gbk$/i => '"cp936"'); |
197 | |
f7ac3676 |
198 | # TODO: HP-UX '8' encodings arabic8 greek8 hebrew8 kana8 thai8 turkish8 |
199 | # TODO: HP-UX '15' encodings japanese15 korean15 roi15 |
200 | # TODO: Cyrillic encoding ISO-IR-111 (useful?) |
f7ac3676 |
201 | # TODO: Armenian encoding ARMSCII-8 |
202 | # TODO: Hebrew encoding ISO-8859-8-1 |
203 | # TODO: Thai encoding TCVN |
204 | # TODO: Korean encoding Johab |
56a543c5 |
205 | # TODO: Vietnamese encodings VPS |
f7ac3676 |
206 | # TODO: Japanese encoding JIS (not the same as SJIS) |
207 | # TODO: Mac Asian+African encodings: Arabic Armenian Bengali Burmese |
208 | # ChineseSimp ChineseTrad Devanagari Ethiopic ExtArabic |
209 | # Farsi Georgian Gujarati Gurmukhi Hebrew Japanese |
210 | # Kannada Khmer Korean Laotian Malayalam Mongolian |
211 | # Oriya Sinhalese Symbol Tamil Telugu Tibetan Vietnamese |
18586f54 |
212 | |
1853dd5f |
213 | # Map white space and _ to '-' |
016cb72c |
214 | define_alias( qr/^(\S+)[\s_]+(.*)$/i => '"$1-$2"' ); |
215 | |
51ef4e11 |
216 | sub define_encoding |
217 | { |
18586f54 |
218 | my $obj = shift; |
219 | my $name = shift; |
220 | $encoding{$name} = $obj; |
221 | my $lc = lc($name); |
222 | define_alias($lc => $obj) unless $lc eq $name; |
223 | while (@_) |
224 | { |
225 | my $alias = shift; |
226 | define_alias($alias,$obj); |
227 | } |
228 | return $obj; |
656753f8 |
229 | } |
230 | |
656753f8 |
231 | sub getEncoding |
232 | { |
dd9703c9 |
233 | my ($class,$name,$skip_external) = @_; |
18586f54 |
234 | my $enc; |
235 | if (ref($name) && $name->can('new_sequence')) |
236 | { |
237 | return $name; |
238 | } |
239 | my $lc = lc $name; |
240 | if (exists $encoding{$name}) |
241 | { |
242 | return $encoding{$name}; |
243 | } |
244 | if (exists $encoding{$lc}) |
245 | { |
246 | return $encoding{$lc}; |
247 | } |
c50d192e |
248 | |
249 | my $oc = $class->findAlias($name); |
250 | return $oc if defined $oc; |
251 | |
252 | $oc = $class->findAlias($lc) if $lc ne $name; |
253 | return $oc if defined $oc; |
254 | |
dd9703c9 |
255 | if (!$skip_external and exists $external_tables{$lc}) |
d1ed7747 |
256 | { |
257 | require $external_tables{$lc}; |
258 | return $encoding{$name} if exists $encoding{$name}; |
259 | } |
18586f54 |
260 | |
18586f54 |
261 | return; |
656753f8 |
262 | } |
263 | |
4411f3b6 |
264 | sub find_encoding |
265 | { |
dd9703c9 |
266 | my ($name,$skip_external) = @_; |
267 | return __PACKAGE__->getEncoding($name,$skip_external); |
4411f3b6 |
268 | } |
269 | |
270 | sub encode |
271 | { |
18586f54 |
272 | my ($name,$string,$check) = @_; |
273 | my $enc = find_encoding($name); |
274 | croak("Unknown encoding '$name'") unless defined $enc; |
275 | my $octets = $enc->encode($string,$check); |
276 | return undef if ($check && length($string)); |
277 | return $octets; |
4411f3b6 |
278 | } |
279 | |
280 | sub decode |
281 | { |
18586f54 |
282 | my ($name,$octets,$check) = @_; |
283 | my $enc = find_encoding($name); |
284 | croak("Unknown encoding '$name'") unless defined $enc; |
285 | my $string = $enc->decode($octets,$check); |
286 | $_[1] = $octets if $check; |
287 | return $string; |
4411f3b6 |
288 | } |
289 | |
290 | sub from_to |
291 | { |
18586f54 |
292 | my ($string,$from,$to,$check) = @_; |
293 | my $f = find_encoding($from); |
294 | croak("Unknown encoding '$from'") unless defined $f; |
295 | my $t = find_encoding($to); |
296 | croak("Unknown encoding '$to'") unless defined $t; |
297 | my $uni = $f->decode($string,$check); |
298 | return undef if ($check && length($string)); |
299 | $string = $t->encode($uni,$check); |
300 | return undef if ($check && length($uni)); |
301 | return length($_[0] = $string); |
4411f3b6 |
302 | } |
303 | |
304 | sub encode_utf8 |
305 | { |
18586f54 |
306 | my ($str) = @_; |
307 | utf8::encode($str); |
308 | return $str; |
4411f3b6 |
309 | } |
310 | |
311 | sub decode_utf8 |
312 | { |
18586f54 |
313 | my ($str) = @_; |
314 | return undef unless utf8::decode($str); |
315 | return $str; |
5ad8ef52 |
316 | } |
317 | |
18586f54 |
318 | require Encode::Encoding; |
319 | require Encode::XS; |
320 | require Encode::Internal; |
321 | require Encode::Unicode; |
322 | require Encode::utf8; |
323 | require Encode::iso10646_1; |
324 | require Encode::ucs2_le; |
4411f3b6 |
325 | |
656753f8 |
326 | 1; |
327 | |
2a936312 |
328 | __END__ |
329 | |
4411f3b6 |
330 | =head1 NAME |
331 | |
332 | Encode - character encodings |
333 | |
334 | =head1 SYNOPSIS |
335 | |
336 | use Encode; |
337 | |
338 | =head1 DESCRIPTION |
339 | |
47bfe92f |
340 | The C<Encode> module provides the interfaces between Perl's strings |
341 | and the rest of the system. Perl strings are sequences of B<characters>. |
4411f3b6 |
342 | |
343 | The repertoire of characters that Perl can represent is at least that |
47bfe92f |
344 | defined by the Unicode Consortium. On most platforms the ordinal |
345 | values of the characters (as returned by C<ord(ch)>) is the "Unicode |
346 | codepoint" for the character (the exceptions are those platforms where |
347 | the legacy encoding is some variant of EBCDIC rather than a super-set |
348 | of ASCII - see L<perlebcdic>). |
4411f3b6 |
349 | |
350 | Traditionaly computer data has been moved around in 8-bit chunks |
351 | often called "bytes". These chunks are also known as "octets" in |
352 | networking standards. Perl is widely used to manipulate data of |
353 | many types - not only strings of characters representing human or |
354 | computer languages but also "binary" data being the machines representation |
355 | of numbers, pixels in an image - or just about anything. |
356 | |
47bfe92f |
357 | When Perl is processing "binary data" the programmer wants Perl to process |
358 | "sequences of bytes". This is not a problem for Perl - as a byte has 256 |
359 | possible values it easily fits in Perl's much larger "logical character". |
4411f3b6 |
360 | |
d1ed7747 |
361 | Due to size concerns, each of B<CJK> (Chinese, Japanese & Korean) modules |
362 | are not loaded in memory until the first time they're used. Although you |
363 | don't have to C<use> the corresponding B<Encode::>(B<TW>|B<CN>|B<JP>|B<KR>) |
364 | modules first, be aware that those encodings will not be in C<%encodings> |
365 | until their module is loaded (either implicitly through using encodings |
366 | contained in the same module, or via an explicit C<use>). |
a67efb5b |
367 | |
4411f3b6 |
368 | =head2 TERMINOLOGY |
369 | |
4ac9195f |
370 | =over 4 |
4411f3b6 |
371 | |
372 | =item * |
373 | |
374 | I<character>: a character in the range 0..(2**32-1) (or more). |
47bfe92f |
375 | (What Perl's strings are made of.) |
4411f3b6 |
376 | |
377 | =item * |
378 | |
379 | I<byte>: a character in the range 0..255 |
47bfe92f |
380 | (A special case of a Perl character.) |
4411f3b6 |
381 | |
382 | =item * |
383 | |
384 | I<octet>: 8 bits of data, with ordinal values 0..255 |
47bfe92f |
385 | (Term for bytes passed to or from a non-Perl context, e.g. disk file.) |
4411f3b6 |
386 | |
387 | =back |
388 | |
389 | The marker [INTERNAL] marks Internal Implementation Details, in |
390 | general meant only for those who think they know what they are doing, |
391 | and such details may change in future releases. |
392 | |
393 | =head1 ENCODINGS |
394 | |
395 | =head2 Characteristics of an Encoding |
396 | |
397 | An encoding has a "repertoire" of characters that it can represent, |
398 | and for each representable character there is at least one sequence of |
399 | octets that represents it. |
400 | |
401 | =head2 Types of Encodings |
402 | |
403 | Encodings can be divided into the following types: |
404 | |
405 | =over 4 |
406 | |
407 | =item * Fixed length 8-bit (or less) encodings. |
408 | |
409 | Each character is a single octet so may have a repertoire of up to |
410 | 256 characters. ASCII and iso-8859-* are typical examples. |
411 | |
412 | =item * Fixed length 16-bit encodings |
413 | |
414 | Each character is two octets so may have a repertoire of up to |
47bfe92f |
415 | 65 536 characters. Unicode's UCS-2 is an example. Also used for |
4411f3b6 |
416 | encodings for East Asian languages. |
417 | |
418 | =item * Fixed length 32-bit encodings. |
419 | |
420 | Not really very "encoded" encodings. The Unicode code points |
421 | are just represented as 4-octet integers. None the less because |
422 | different architectures use different representations of integers |
423 | (so called "endian") there at least two disctinct encodings. |
424 | |
425 | =item * Multi-byte encodings |
426 | |
427 | The number of octets needed to represent a character varies. |
428 | UTF-8 is a particularly complex but regular case of a multi-byte |
429 | encoding. Several East Asian countries use a multi-byte encoding |
430 | where 1-octet is used to cover western roman characters and Asian |
431 | characters get 2-octets. |
432 | (UTF-16 is strictly a multi-byte encoding taking either 2 or 4 octets |
433 | to represent a Unicode code point.) |
434 | |
435 | =item * "Escape" encodings. |
436 | |
437 | These encodings embed "escape sequences" into the octet sequence |
438 | which describe how the following octets are to be interpreted. |
439 | The iso-2022-* family is typical. Following the escape sequence |
440 | octets are encoded by an "embedded" encoding (which will be one |
441 | of the above types) until another escape sequence switches to |
442 | a different "embedded" encoding. |
443 | |
444 | These schemes are very flexible and can handle mixed languages but are |
47bfe92f |
445 | very complex to process (and have state). No escape encodings are |
446 | implemented for Perl yet. |
4411f3b6 |
447 | |
448 | =back |
449 | |
450 | =head2 Specifying Encodings |
451 | |
452 | Encodings can be specified to the API described below in two ways: |
453 | |
454 | =over 4 |
455 | |
456 | =item 1. By name |
457 | |
47bfe92f |
458 | Encoding names are strings with characters taken from a restricted |
459 | repertoire. See L</"Encoding Names">. |
4411f3b6 |
460 | |
461 | =item 2. As an object |
462 | |
2659725b |
463 | Encoding objects are returned by C<find_encoding($name, [$skip_external])>. |
464 | If the second parameter is true, Encode will refrain from loading external |
465 | modules for CJK encodings. |
4411f3b6 |
466 | |
467 | =back |
468 | |
469 | =head2 Encoding Names |
470 | |
471 | Encoding names are case insensitive. White space in names is ignored. |
47bfe92f |
472 | In addition an encoding may have aliases. Each encoding has one |
473 | "canonical" name. The "canonical" name is chosen from the names of |
474 | the encoding by picking the first in the following sequence: |
4411f3b6 |
475 | |
476 | =over 4 |
477 | |
78255929 |
478 | =item * The MIME name as defined in IETF RFCs. |
4411f3b6 |
479 | |
480 | =item * The name in the IANA registry. |
481 | |
d1be9408 |
482 | =item * The name used by the organization that defined it. |
4411f3b6 |
483 | |
484 | =back |
485 | |
486 | Because of all the alias issues, and because in the general case |
487 | encodings have state C<Encode> uses the encoding object internally |
488 | once an operation is in progress. |
489 | |
21938dfa |
490 | As of Perl 5.8.0, at least the following encodings are recognized |
491 | (the => marks aliases): |
492 | |
493 | ASCII |
494 | |
495 | US-ASCII => ASCII |
496 | |
497 | The Unicode: |
498 | |
0b3236bb |
499 | UTF-8 |
21938dfa |
500 | UTF-16 |
501 | UCS-2 |
502 | |
503 | ISO 10646-1 => UCS-2 |
504 | |
505 | The ISO 8859 and KOI: |
506 | |
507 | ISO 8859-1 ISO 8859-6 ISO 8859-11 KOI8-F |
508 | ISO 8859-2 ISO 8859-7 (12 doesn't exist) KOI8-R |
56a543c5 |
509 | ISO 8859-3 ISO 8859-8 ISO 8859-13 KOI8-U |
21938dfa |
510 | ISO 8859-4 ISO 8859-9 ISO 8859-14 |
511 | ISO 8859-5 ISO 8859-10 ISO 8859-15 |
512 | ISO 8859-16 |
513 | |
514 | Latin1 => 8859-1 Latin6 => 8859-10 |
515 | Latin2 => 8859-2 Latin7 => 8859-13 |
0b3236bb |
516 | Latin3 => 8859-3 Latin8 => 8859-14 |
21938dfa |
517 | Latin4 => 8859-4 Latin9 => 8859-15 |
518 | Latin5 => 8859-9 Latin10 => 8859-16 |
519 | |
520 | Cyrillic => 8859-5 |
521 | Arabic => 8859-6 |
522 | Greek => 8859-7 |
523 | Hebrew => 8859-8 |
524 | Thai => 8859-11 |
0b3236bb |
525 | TIS620 => 8859-11 |
21938dfa |
526 | |
527 | The CJKV: Chinese, Japanese, Korean, Vietnamese: |
528 | |
0b3236bb |
529 | ISO 2022 ISO 2022 JP-1 JIS 0201 GB 1988 Big5 EUC-CN |
530 | ISO 2022 CN ISO 2022 JP-2 JIS 0208 GB 2312 HZ EUC-JP |
21938dfa |
531 | ISO 2022 JP ISO 2022 KR JIS 0210 GB 12345 CNS 11643 EUC-JP-0212 |
cb448690 |
532 | Shift-JIS GBK Big5-HKSCS EUC-KR |
533 | VISCII ISO-IR-165 |
534 | |
a67efb5b |
535 | (Due to size concerns, additional Chinese encodings including C<GB 18030>, |
536 | C<EUC-TW> and C<BIG5PLUS> are distributed separately on CPAN, under the name |
537 | L<Encode::HanExtra>.) |
21938dfa |
538 | |
539 | The PC codepages: |
540 | |
541 | CP37 CP852 CP861 CP866 CP949 CP1251 CP1256 |
542 | CP424 CP855 CP862 CP869 CP950 CP1252 CP1257 |
543 | CP737 CP856 CP863 CP874 CP1006 CP1253 CP1258 |
544 | CP775 CP857 CP864 CP932 CP1047 CP1254 |
545 | CP850 CP860 CP865 CP936 CP1250 CP1255 |
546 | |
547 | WinLatin1 => CP1252 |
548 | WinLatin2 => CP1250 |
549 | WinCyrillic => CP1251 |
550 | WinGreek => CP1253 |
551 | WinTurkiskh => CP1254 |
552 | WinHebrew => CP1255 |
553 | WinArabic => CP1256 |
554 | WinBaltic => CP1257 |
555 | WinVietnamese => CP1258 |
556 | |
4a42e14c |
557 | (All the CPI<NNN...> are available also as IBMI<NNN...>.) |
21938dfa |
558 | |
559 | The Mac codepages: |
560 | |
0b3236bb |
561 | MacCentralEuropean MacJapanese |
562 | MacCroatian MacRoman |
1853dd5f |
563 | MacCyrillic MacRomanian |
0b3236bb |
564 | MacDingbats MacSami |
565 | MacGreek MacThai |
566 | MacIcelandic MacTurkish |
567 | MacUkraine |
21938dfa |
568 | |
569 | Miscellaneous: |
570 | |
571 | 7bit-greek IR-197 |
572 | 7bit-kana NeXTstep |
573 | 7bit-latin1 POSIX-BC |
574 | DingBats Roman8 |
575 | GSM 0338 Symbol |
576 | |
4411f3b6 |
577 | =head1 PERL ENCODING API |
578 | |
579 | =head2 Generic Encoding Interface |
580 | |
581 | =over 4 |
582 | |
583 | =item * |
584 | |
585 | $bytes = encode(ENCODING, $string[, CHECK]) |
586 | |
47bfe92f |
587 | Encodes string from Perl's internal form into I<ENCODING> and returns |
588 | a sequence of octets. For CHECK see L</"Handling Malformed Data">. |
4411f3b6 |
589 | |
681a7c68 |
590 | For example to convert (internally UTF-8 encoded) Unicode data |
591 | to octets: |
592 | |
593 | $octets = encode("utf8", $unicode); |
594 | |
4411f3b6 |
595 | =item * |
596 | |
597 | $string = decode(ENCODING, $bytes[, CHECK]) |
598 | |
47bfe92f |
599 | Decode sequence of octets assumed to be in I<ENCODING> into Perl's |
600 | internal form and returns the resulting string. For CHECK see |
601 | L</"Handling Malformed Data">. |
602 | |
681a7c68 |
603 | For example to convert ISO 8859-1 data to UTF-8: |
604 | |
605 | $utf8 = decode("latin1", $latin1); |
606 | |
47bfe92f |
607 | =item * |
608 | |
609 | from_to($string, FROM_ENCODING, TO_ENCODING[, CHECK]) |
610 | |
2b106fbe |
611 | Convert B<in-place> the data between two encodings. How did the data |
612 | in $string originally get to be in FROM_ENCODING? Either using |
e9692b5b |
613 | encode() or through PerlIO: See L</"Encoding and IO">. For CHECK |
2b106fbe |
614 | see L</"Handling Malformed Data">. |
615 | |
616 | For example to convert ISO 8859-1 data to UTF-8: |
617 | |
618 | from_to($data, "iso-8859-1", "utf-8"); |
619 | |
620 | and to convert it back: |
621 | |
622 | from_to($data, "utf-8", "iso-8859-1"); |
4411f3b6 |
623 | |
ab97ca19 |
624 | Note that because the conversion happens in place, the data to be |
625 | converted cannot be a string constant, it must be a scalar variable. |
626 | |
4411f3b6 |
627 | =back |
628 | |
629 | =head2 Handling Malformed Data |
630 | |
631 | If CHECK is not set, C<undef> is returned. If the data is supposed to |
47bfe92f |
632 | be UTF-8, an optional lexical warning (category utf8) is given. If |
633 | CHECK is true but not a code reference, dies. |
4411f3b6 |
634 | |
47bfe92f |
635 | It would desirable to have a way to indicate that transform should use |
636 | the encodings "replacement character" - no such mechanism is defined yet. |
4411f3b6 |
637 | |
638 | It is also planned to allow I<CHECK> to be a code reference. |
639 | |
47bfe92f |
640 | This is not yet implemented as there are design issues with what its |
641 | arguments should be and how it returns its results. |
4411f3b6 |
642 | |
643 | =over 4 |
644 | |
645 | =item Scheme 1 |
646 | |
647 | Passed remaining fragment of string being processed. |
648 | Modifies it in place to remove bytes/characters it can understand |
649 | and returns a string used to represent them. |
650 | e.g. |
651 | |
652 | sub fixup { |
653 | my $ch = substr($_[0],0,1,''); |
654 | return sprintf("\x{%02X}",ord($ch); |
655 | } |
656 | |
657 | This scheme is close to how underlying C code for Encode works, but gives |
658 | the fixup routine very little context. |
659 | |
660 | =item Scheme 2 |
661 | |
47bfe92f |
662 | Passed original string, and an index into it of the problem area, and |
663 | output string so far. Appends what it will to output string and |
664 | returns new index into original string. For example: |
4411f3b6 |
665 | |
666 | sub fixup { |
667 | # my ($s,$i,$d) = @_; |
668 | my $ch = substr($_[0],$_[1],1); |
669 | $_[2] .= sprintf("\x{%02X}",ord($ch); |
670 | return $_[1]+1; |
671 | } |
672 | |
47bfe92f |
673 | This scheme gives maximal control to the fixup routine but is more |
674 | complicated to code, and may need internals of Encode to be tweaked to |
675 | keep original string intact. |
4411f3b6 |
676 | |
677 | =item Other Schemes |
678 | |
679 | Hybrids of above. |
680 | |
681 | Multiple return values rather than in-place modifications. |
682 | |
683 | Index into the string could be pos($str) allowing s/\G...//. |
684 | |
685 | =back |
686 | |
687 | =head2 UTF-8 / utf8 |
688 | |
689 | The Unicode consortium defines the UTF-8 standard as a way of encoding |
47bfe92f |
690 | the entire Unicode repertiore as sequences of octets. This encoding is |
691 | expected to become very widespread. Perl can use this form internaly |
692 | to represent strings, so conversions to and from this form are |
693 | particularly efficient (as octets in memory do not have to change, |
694 | just the meta-data that tells Perl how to treat them). |
4411f3b6 |
695 | |
696 | =over 4 |
697 | |
698 | =item * |
699 | |
700 | $bytes = encode_utf8($string); |
701 | |
47bfe92f |
702 | The characters that comprise string are encoded in Perl's superset of UTF-8 |
4411f3b6 |
703 | and the resulting octets returned as a sequence of bytes. All possible |
704 | characters have a UTF-8 representation so this function cannot fail. |
705 | |
706 | =item * |
707 | |
708 | $string = decode_utf8($bytes [,CHECK]); |
709 | |
47bfe92f |
710 | The sequence of octets represented by $bytes is decoded from UTF-8 |
711 | into a sequence of logical characters. Not all sequences of octets |
712 | form valid UTF-8 encodings, so it is possible for this call to fail. |
713 | For CHECK see L</"Handling Malformed Data">. |
4411f3b6 |
714 | |
715 | =back |
716 | |
717 | =head2 Other Encodings of Unicode |
718 | |
47bfe92f |
719 | UTF-16 is similar to UCS-2, 16 bit or 2-byte chunks. UCS-2 can only |
7a4efbb2 |
720 | represent 0..0xFFFF, while UTF-16 has a I<surrogate pair> scheme which |
47bfe92f |
721 | allows it to cover the whole Unicode range. |
4411f3b6 |
722 | |
7a4efbb2 |
723 | Surrogates are code points set aside to encode the 0x01000..0x10FFFF |
724 | range of Unicode code points in pairs of 16-bit units. The I<high |
725 | surrogates> are the range 0xD800..0xDBFF, and the I<low surrogates> |
726 | are the range 0xDC00..0xDFFFF. The surrogate encoding is |
727 | |
728 | $hi = ($uni - 0x10000) / 0x400 + 0xD800; |
729 | $lo = ($uni - 0x10000) % 0x400 + 0xDC00; |
730 | |
731 | and the decoding is |
732 | |
733 | $uni = 0x10000 + ($hi - 0xD8000) * 0x400 + ($lo - 0xDC00); |
734 | |
8040349a |
735 | Encode implements big-endian UCS-2 aliased to "iso-10646-1" as that |
47bfe92f |
736 | happens to be the name used by that representation when used with X11 |
737 | fonts. |
4411f3b6 |
738 | |
739 | UTF-32 or UCS-4 is 32-bit or 4-byte chunks. Perl's logical characters |
740 | can be considered as being in this form without encoding. An encoding |
47bfe92f |
741 | to transfer strings in this form (e.g. to write them to a file) would |
742 | need to |
4411f3b6 |
743 | |
c079d275 |
744 | pack('L*', unpack('U*', $string)); # native |
4411f3b6 |
745 | or |
c079d275 |
746 | pack('V*', unpack('U*', $string)); # little-endian |
4411f3b6 |
747 | or |
c079d275 |
748 | pack('N*', unpack('U*', $string)); # big-endian |
4411f3b6 |
749 | |
c079d275 |
750 | depending on the endianness required. |
4411f3b6 |
751 | |
51ef4e11 |
752 | No UTF-32 encodings are implemented yet. |
4411f3b6 |
753 | |
47bfe92f |
754 | Both UCS-2 and UCS-4 style encodings can have "byte order marks" by |
755 | representing the code point 0xFFFE as the very first thing in a file. |
4411f3b6 |
756 | |
51ef4e11 |
757 | =head2 Listing available encodings |
758 | |
759 | use Encode qw(encodings); |
760 | @list = encodings(); |
761 | |
762 | Returns a list of the canonical names of the available encodings. |
763 | |
764 | =head2 Defining Aliases |
765 | |
766 | use Encode qw(define_alias); |
767 | define_alias( newName => ENCODING); |
768 | |
47bfe92f |
769 | Allows newName to be used as am alias for ENCODING. ENCODING may be |
770 | either the name of an encoding or and encoding object (as above). |
51ef4e11 |
771 | |
772 | Currently I<newName> can be specified in the following ways: |
773 | |
774 | =over 4 |
775 | |
776 | =item As a simple string. |
777 | |
778 | =item As a qr// compiled regular expression, e.g.: |
779 | |
780 | define_alias( qr/^iso8859-(\d+)$/i => '"iso-8859-$1"' ); |
781 | |
47bfe92f |
782 | In this case if I<ENCODING> is not a reference it is C<eval>-ed to |
783 | allow C<$1> etc. to be subsituted. The example is one way to names as |
784 | used in X11 font names to alias the MIME names for the iso-8859-* |
785 | family. |
51ef4e11 |
786 | |
787 | =item As a code reference, e.g.: |
788 | |
789 | define_alias( sub { return /^iso8859-(\d+)$/i ? "iso-8859-$1" : undef } , ''); |
790 | |
791 | In this case C<$_> will be set to the name that is being looked up and |
47bfe92f |
792 | I<ENCODING> is passed to the sub as its first argument. The example |
793 | is another way to names as used in X11 font names to alias the MIME |
794 | names for the iso-8859-* family. |
51ef4e11 |
795 | |
796 | =back |
797 | |
798 | =head2 Defining Encodings |
799 | |
e9692b5b |
800 | use Encode qw(define_alias); |
801 | define_encoding( $object, 'canonicalName' [,alias...]); |
51ef4e11 |
802 | |
47bfe92f |
803 | Causes I<canonicalName> to be associated with I<$object>. The object |
804 | should provide the interface described in L</"IMPLEMENTATION CLASSES"> |
805 | below. If more than two arguments are provided then additional |
806 | arguments are taken as aliases for I<$object> as for C<define_alias>. |
51ef4e11 |
807 | |
4411f3b6 |
808 | =head1 Encoding and IO |
809 | |
810 | It is very common to want to do encoding transformations when |
811 | reading or writing files, network connections, pipes etc. |
47bfe92f |
812 | If Perl is configured to use the new 'perlio' IO system then |
4411f3b6 |
813 | C<Encode> provides a "layer" (See L<perliol>) which can transform |
814 | data as it is read or written. |
815 | |
8e86646e |
816 | Here is how the blind poet would modernise the encoding: |
817 | |
42234700 |
818 | use Encode; |
8e86646e |
819 | open(my $iliad,'<:encoding(iso-8859-7)','iliad.greek'); |
820 | open(my $utf8,'>:utf8','iliad.utf8'); |
821 | my @epic = <$iliad>; |
822 | print $utf8 @epic; |
823 | close($utf8); |
824 | close($illiad); |
4411f3b6 |
825 | |
826 | In addition the new IO system can also be configured to read/write |
827 | UTF-8 encoded characters (as noted above this is efficient): |
828 | |
e9692b5b |
829 | open(my $fh,'>:utf8','anything'); |
830 | print $fh "Any \x{0021} string \N{SMILEY FACE}\n"; |
4411f3b6 |
831 | |
832 | Either of the above forms of "layer" specifications can be made the default |
833 | for a lexical scope with the C<use open ...> pragma. See L<open>. |
834 | |
835 | Once a handle is open is layers can be altered using C<binmode>. |
836 | |
47bfe92f |
837 | Without any such configuration, or if Perl itself is built using |
4411f3b6 |
838 | system's own IO, then write operations assume that file handle accepts |
839 | only I<bytes> and will C<die> if a character larger than 255 is |
840 | written to the handle. When reading, each octet from the handle |
841 | becomes a byte-in-a-character. Note that this default is the same |
47bfe92f |
842 | behaviour as bytes-only languages (including Perl before v5.6) would |
843 | have, and is sufficient to handle native 8-bit encodings |
844 | e.g. iso-8859-1, EBCDIC etc. and any legacy mechanisms for handling |
845 | other encodings and binary data. |
846 | |
847 | In other cases it is the programs responsibility to transform |
848 | characters into bytes using the API above before doing writes, and to |
849 | transform the bytes read from a handle into characters before doing |
850 | "character operations" (e.g. C<lc>, C</\W+/>, ...). |
851 | |
47bfe92f |
852 | You can also use PerlIO to convert larger amounts of data you don't |
853 | want to bring into memory. For example to convert between ISO 8859-1 |
854 | (Latin 1) and UTF-8 (or UTF-EBCDIC in EBCDIC machines): |
855 | |
e9692b5b |
856 | open(F, "<:encoding(iso-8859-1)", "data.txt") or die $!; |
857 | open(G, ">:utf8", "data.utf") or die $!; |
858 | while (<F>) { print G } |
859 | |
860 | # Could also do "print G <F>" but that would pull |
861 | # the whole file into memory just to write it out again. |
862 | |
863 | More examples: |
47bfe92f |
864 | |
e9692b5b |
865 | open(my $f, "<:encoding(cp1252)") |
866 | open(my $g, ">:encoding(iso-8859-2)") |
867 | open(my $h, ">:encoding(latin9)") # iso-8859-15 |
47bfe92f |
868 | |
869 | See L<PerlIO> for more information. |
4411f3b6 |
870 | |
1768d7eb |
871 | See also L<encoding> for how to change the default encoding of the |
d521382b |
872 | data in your script. |
1768d7eb |
873 | |
4411f3b6 |
874 | =head1 Encoding How to ... |
875 | |
876 | To do: |
877 | |
878 | =over 4 |
879 | |
880 | =item * IO with mixed content (faking iso-2020-*) |
881 | |
882 | =item * MIME's Content-Length: |
883 | |
884 | =item * UTF-8 strings in binary data. |
885 | |
47bfe92f |
886 | =item * Perl/Encode wrappers on non-Unicode XS modules. |
4411f3b6 |
887 | |
888 | =back |
889 | |
890 | =head1 Messing with Perl's Internals |
891 | |
47bfe92f |
892 | The following API uses parts of Perl's internals in the current |
893 | implementation. As such they are efficient, but may change. |
4411f3b6 |
894 | |
895 | =over 4 |
896 | |
4411f3b6 |
897 | =item * is_utf8(STRING [, CHECK]) |
898 | |
899 | [INTERNAL] Test whether the UTF-8 flag is turned on in the STRING. |
47bfe92f |
900 | If CHECK is true, also checks the data in STRING for being well-formed |
901 | UTF-8. Returns true if successful, false otherwise. |
4411f3b6 |
902 | |
903 | =item * valid_utf8(STRING) |
904 | |
47bfe92f |
905 | [INTERNAL] Test whether STRING is in a consistent state. Will return |
906 | true if string is held as bytes, or is well-formed UTF-8 and has the |
907 | UTF-8 flag on. Main reason for this routine is to allow Perl's |
908 | testsuite to check that operations have left strings in a consistent |
909 | state. |
4411f3b6 |
910 | |
911 | =item * |
912 | |
913 | _utf8_on(STRING) |
914 | |
915 | [INTERNAL] Turn on the UTF-8 flag in STRING. The data in STRING is |
916 | B<not> checked for being well-formed UTF-8. Do not use unless you |
917 | B<know> that the STRING is well-formed UTF-8. Returns the previous |
918 | state of the UTF-8 flag (so please don't test the return value as |
919 | I<not> success or failure), or C<undef> if STRING is not a string. |
920 | |
921 | =item * |
922 | |
923 | _utf8_off(STRING) |
924 | |
925 | [INTERNAL] Turn off the UTF-8 flag in STRING. Do not use frivolously. |
926 | Returns the previous state of the UTF-8 flag (so please don't test the |
927 | return value as I<not> success or failure), or C<undef> if STRING is |
928 | not a string. |
929 | |
930 | =back |
931 | |
4edaa979 |
932 | =head1 IMPLEMENTATION CLASSES |
933 | |
934 | As mentioned above encodings are (in the current implementation at least) |
935 | defined by objects. The mapping of encoding name to object is via the |
51ef4e11 |
936 | C<%encodings> hash. |
4edaa979 |
937 | |
938 | The values of the hash can currently be either strings or objects. |
939 | The string form may go away in the future. The string form occurs |
940 | when C<encodings()> has scanned C<@INC> for loadable encodings but has |
941 | not actually loaded the encoding in question. This is because the |
47bfe92f |
942 | current "loading" process is all Perl and a bit slow. |
4edaa979 |
943 | |
47bfe92f |
944 | Once an encoding is loaded then value of the hash is object which |
945 | implements the encoding. The object should provide the following |
946 | interface: |
4edaa979 |
947 | |
948 | =over 4 |
949 | |
950 | =item -E<gt>name |
951 | |
952 | Should return the string representing the canonical name of the encoding. |
953 | |
954 | =item -E<gt>new_sequence |
955 | |
47bfe92f |
956 | This is a placeholder for encodings with state. It should return an |
957 | object which implements this interface, all current implementations |
958 | return the original object. |
4edaa979 |
959 | |
960 | =item -E<gt>encode($string,$check) |
961 | |
47bfe92f |
962 | Should return the octet sequence representing I<$string>. If I<$check> |
963 | is true it should modify I<$string> in place to remove the converted |
964 | part (i.e. the whole string unless there is an error). If an error |
965 | occurs it should return the octet sequence for the fragment of string |
966 | that has been converted, and modify $string in-place to remove the |
967 | converted part leaving it starting with the problem fragment. |
4edaa979 |
968 | |
47bfe92f |
969 | If check is is false then C<encode> should make a "best effort" to |
970 | convert the string - for example by using a replacement character. |
4edaa979 |
971 | |
972 | =item -E<gt>decode($octets,$check) |
973 | |
47bfe92f |
974 | Should return the string that I<$octets> represents. If I<$check> is |
975 | true it should modify I<$octets> in place to remove the converted part |
976 | (i.e. the whole sequence unless there is an error). If an error |
977 | occurs it should return the fragment of string that has been |
978 | converted, and modify $octets in-place to remove the converted part |
4edaa979 |
979 | leaving it starting with the problem fragment. |
980 | |
47bfe92f |
981 | If check is is false then C<decode> should make a "best effort" to |
982 | convert the string - for example by using Unicode's "\x{FFFD}" as a |
983 | replacement character. |
4edaa979 |
984 | |
985 | =back |
986 | |
47bfe92f |
987 | It should be noted that the check behaviour is different from the |
988 | outer public API. The logic is that the "unchecked" case is useful |
989 | when encoding is part of a stream which may be reporting errors |
990 | (e.g. STDERR). In such cases it is desirable to get everything |
991 | through somehow without causing additional errors which obscure the |
992 | original one. Also the encoding is best placed to know what the |
993 | correct replacement character is, so if that is the desired behaviour |
994 | then letting low level code do it is the most efficient. |
995 | |
996 | In contrast if check is true, the scheme above allows the encoding to |
997 | do as much as it can and tell layer above how much that was. What is |
998 | lacking at present is a mechanism to report what went wrong. The most |
999 | likely interface will be an additional method call to the object, or |
1000 | perhaps (to avoid forcing per-stream objects on otherwise stateless |
1001 | encodings) and additional parameter. |
1002 | |
1003 | It is also highly desirable that encoding classes inherit from |
1004 | C<Encode::Encoding> as a base class. This allows that class to define |
1005 | additional behaviour for all encoding objects. For example built in |
1006 | Unicode, UCS-2 and UTF-8 classes use : |
51ef4e11 |
1007 | |
1008 | package Encode::MyEncoding; |
1009 | use base qw(Encode::Encoding); |
1010 | |
1011 | __PACKAGE__->Define(qw(myCanonical myAlias)); |
1012 | |
47bfe92f |
1013 | To create an object with bless {Name => ...},$class, and call |
1014 | define_encoding. They inherit their C<name> method from |
1015 | C<Encode::Encoding>. |
4edaa979 |
1016 | |
1017 | =head2 Compiled Encodings |
1018 | |
47bfe92f |
1019 | F<Encode.xs> provides a class C<Encode::XS> which provides the |
1020 | interface described above. It calls a generic octet-sequence to |
1021 | octet-sequence "engine" that is driven by tables (defined in |
1022 | F<encengine.c>). The same engine is used for both encode and |
1023 | decode. C<Encode:XS>'s C<encode> forces Perl's characters to their |
1024 | UTF-8 form and then treats them as just another multibyte |
1025 | encoding. C<Encode:XS>'s C<decode> transforms the sequence and then |
1026 | turns the UTF-8-ness flag as that is the form that the tables are |
1027 | defined to produce. For details of the engine see the comments in |
1028 | F<encengine.c>. |
1029 | |
1030 | The tables are produced by the Perl script F<compile> (the name needs |
1031 | to change so we can eventually install it somewhere). F<compile> can |
1032 | currently read two formats: |
4edaa979 |
1033 | |
1034 | =over 4 |
1035 | |
1036 | =item *.enc |
1037 | |
47bfe92f |
1038 | This is a coined format used by Tcl. It is documented in |
1039 | Encode/EncodeFormat.pod. |
4edaa979 |
1040 | |
1041 | =item *.ucm |
1042 | |
1043 | This is the semi-standard format used by IBM's ICU package. |
1044 | |
1045 | =back |
1046 | |
1047 | F<compile> can write the following forms: |
1048 | |
1049 | =over 4 |
1050 | |
1051 | =item *.ucm |
1052 | |
1053 | See above - the F<Encode/*.ucm> files provided with the distribution have |
1054 | been created from the original Tcl .enc files using this approach. |
1055 | |
1056 | =item *.c |
1057 | |
1058 | Produces tables as C data structures - this is used to build in encodings |
1059 | into F<Encode.so>/F<Encode.dll>. |
1060 | |
1061 | =item *.xs |
1062 | |
47bfe92f |
1063 | In theory this allows encodings to be stand-alone loadable Perl |
1064 | extensions. The process has not yet been tested. The plan is to use |
1065 | this approach for large East Asian encodings. |
4edaa979 |
1066 | |
1067 | =back |
1068 | |
47bfe92f |
1069 | The set of encodings built-in to F<Encode.so>/F<Encode.dll> is |
1070 | determined by F<Makefile.PL>. The current set is as follows: |
4edaa979 |
1071 | |
1072 | =over 4 |
1073 | |
1074 | =item ascii and iso-8859-* |
1075 | |
1076 | That is all the common 8-bit "western" encodings. |
1077 | |
1078 | =item IBM-1047 and two other variants of EBCDIC. |
1079 | |
47bfe92f |
1080 | These are the same variants that are supported by EBCDIC Perl as |
1081 | "native" encodings. They are included to prove "reversibility" of |
1082 | some constructs in EBCDIC Perl. |
4edaa979 |
1083 | |
1084 | =item symbol and dingbats as used by Tk on X11. |
1085 | |
47bfe92f |
1086 | (The reason Encode got started was to support Perl/Tk.) |
4edaa979 |
1087 | |
1088 | =back |
1089 | |
47bfe92f |
1090 | That set is rather ad hoc and has been driven by the needs of the |
1091 | tests rather than the needs of typical applications. It is likely |
1092 | to be rationalized. |
4edaa979 |
1093 | |
4411f3b6 |
1094 | =head1 SEE ALSO |
1095 | |
1768d7eb |
1096 | L<perlunicode>, L<perlebcdic>, L<perlfunc/open>, L<PerlIO>, L<encoding> |
4411f3b6 |
1097 | |
1098 | =cut |
1099 | |