Commit | Line | Data |
2c674647 |
1 | package Encode; |
2 | |
3 | $VERSION = 0.01; |
4 | |
5 | require DynaLoader; |
6 | require Exporter; |
7 | |
8 | @ISA = qw(Exporter DynaLoader); |
9 | |
4411f3b6 |
10 | # Public, encouraged API is exported by default |
11 | @EXPORT = qw ( |
12 | encode |
13 | decode |
14 | encode_utf8 |
15 | decode_utf8 |
16 | find_encoding |
17 | ); |
18 | |
2c674647 |
19 | @EXPORT_OK = |
20 | qw( |
4411f3b6 |
21 | encodings |
2c674647 |
22 | from_to |
23 | is_utf8 |
4411f3b6 |
24 | is_8bit |
25 | is_16bit |
a12c0f56 |
26 | utf8_upgrade |
27 | utf8_downgrade |
4411f3b6 |
28 | _utf8_on |
29 | _utf8_off |
2c674647 |
30 | ); |
31 | |
32 | bootstrap Encode (); |
33 | |
4411f3b6 |
34 | # Documentation moved after __END__ for speed - NI-S |
2c674647 |
35 | |
bf230f3d |
36 | use Carp; |
37 | |
2f2b4ff2 |
38 | # The global hash is declared in XS code |
4411f3b6 |
39 | $encoding{Unicode} = bless({},'Encode::Unicode'); |
40 | $encoding{utf8} = bless({},'Encode::utf8'); |
9b37254d |
41 | $encoding{'iso10646-1'} = bless({},'Encode::iso10646_1'); |
5345d506 |
42 | |
656753f8 |
43 | sub encodings |
44 | { |
45 | my ($class) = @_; |
5345d506 |
46 | foreach my $dir (@INC) |
656753f8 |
47 | { |
5345d506 |
48 | if (opendir(my $dh,"$dir/Encode")) |
656753f8 |
49 | { |
5345d506 |
50 | while (defined(my $name = readdir($dh))) |
51 | { |
52 | if ($name =~ /^(.*)\.enc$/) |
53 | { |
54 | next if exists $encoding{$1}; |
55 | $encoding{$1} = "$dir/$name"; |
56 | } |
57 | } |
58 | closedir($dh); |
656753f8 |
59 | } |
5345d506 |
60 | } |
61 | return keys %encoding; |
62 | } |
63 | |
64 | sub loadEncoding |
65 | { |
66 | my ($class,$name,$file) = @_; |
67 | if (open(my $fh,$file)) |
68 | { |
69 | my $type; |
70 | while (1) |
71 | { |
72 | my $line = <$fh>; |
73 | $type = substr($line,0,1); |
74 | last unless $type eq '#'; |
75 | } |
76 | $class .= ('::'.(($type eq 'E') ? 'Escape' : 'Table')); |
c8991b40 |
77 | #warn "Loading $file"; |
5345d506 |
78 | return $class->read($fh,$name,$type); |
656753f8 |
79 | } |
80 | else |
81 | { |
5345d506 |
82 | return undef; |
656753f8 |
83 | } |
656753f8 |
84 | } |
85 | |
656753f8 |
86 | sub getEncoding |
87 | { |
88 | my ($class,$name) = @_; |
5345d506 |
89 | my $enc; |
90 | unless (ref($enc = $encoding{$name})) |
656753f8 |
91 | { |
5345d506 |
92 | $enc = $class->loadEncoding($name,$enc) if defined $enc; |
93 | unless (ref($enc)) |
656753f8 |
94 | { |
5345d506 |
95 | foreach my $dir (@INC) |
656753f8 |
96 | { |
5345d506 |
97 | last if ($enc = $class->loadEncoding($name,"$dir/Encode/$name.enc")); |
656753f8 |
98 | } |
87714904 |
99 | } |
5345d506 |
100 | $encoding{$name} = $enc; |
656753f8 |
101 | } |
5345d506 |
102 | return $enc; |
656753f8 |
103 | } |
104 | |
4411f3b6 |
105 | sub find_encoding |
106 | { |
107 | my ($name) = @_; |
108 | return __PACKAGE__->getEncoding($name); |
109 | } |
110 | |
111 | sub encode |
112 | { |
113 | my ($name,$string,$check) = @_; |
114 | my $enc = find_encoding($name); |
115 | croak("Unknown encoding '$name'") unless defined $enc; |
50d26985 |
116 | my $octets = $enc->encode($string,$check); |
4411f3b6 |
117 | return undef if ($check && length($string)); |
118 | return $octets; |
119 | } |
120 | |
121 | sub decode |
122 | { |
123 | my ($name,$octets,$check) = @_; |
124 | my $enc = find_encoding($name); |
125 | croak("Unknown encoding '$name'") unless defined $enc; |
50d26985 |
126 | my $string = $enc->decode($octets,$check); |
4411f3b6 |
127 | return undef if ($check && length($octets)); |
128 | return $string; |
129 | } |
130 | |
131 | sub from_to |
132 | { |
133 | my ($string,$from,$to,$check) = @_; |
134 | my $f = find_encoding($from); |
135 | croak("Unknown encoding '$from'") unless defined $f; |
136 | my $t = find_encoding($to); |
137 | croak("Unknown encoding '$to'") unless defined $t; |
50d26985 |
138 | my $uni = $f->decode($string,$check); |
4411f3b6 |
139 | return undef if ($check && length($string)); |
50d26985 |
140 | $string = $t->encode($uni,$check); |
4411f3b6 |
141 | return undef if ($check && length($uni)); |
142 | return length($_[0] = $string); |
143 | } |
144 | |
145 | sub encode_utf8 |
146 | { |
147 | my ($str) = @_; |
148 | utf8_encode($str); |
149 | return $str; |
150 | } |
151 | |
152 | sub decode_utf8 |
153 | { |
154 | my ($str) = @_; |
155 | return undef unless utf8_decode($str); |
156 | return $str; |
157 | } |
158 | |
50d26985 |
159 | package Encode::Encoding; |
160 | # Base class for classes which implement encodings |
4edaa979 |
161 | |
50d26985 |
162 | # Temporary legacy methods |
4edaa979 |
163 | sub toUnicode { shift->decode(@_) } |
164 | sub fromUnicode { shift->encode(@_) } |
165 | |
166 | sub new_sequence { return $_[0] } |
50d26985 |
167 | |
168 | package Encode::XS; |
169 | use base 'Encode::Encoding'; |
170 | |
656753f8 |
171 | package Encode::Unicode; |
50d26985 |
172 | use base 'Encode::Encoding'; |
656753f8 |
173 | |
9b37254d |
174 | # Dummy package that provides the encode interface but leaves data |
a12c0f56 |
175 | # as UTF-8 encoded. It is here so that from_to() works. |
656753f8 |
176 | |
177 | sub name { 'Unicode' } |
178 | |
50d26985 |
179 | sub decode |
a12c0f56 |
180 | { |
181 | my ($obj,$str,$chk) = @_; |
182 | Encode::utf8_upgrade($str); |
183 | $_[1] = '' if $chk; |
184 | return $str; |
185 | } |
656753f8 |
186 | |
50d26985 |
187 | *encode = \&decode; |
656753f8 |
188 | |
4411f3b6 |
189 | package Encode::utf8; |
50d26985 |
190 | use base 'Encode::Encoding'; |
4411f3b6 |
191 | |
192 | # package to allow long-hand |
193 | # $octets = encode( utf8 => $string ); |
194 | # |
195 | |
196 | sub name { 'utf8' } |
197 | |
50d26985 |
198 | sub decode |
4411f3b6 |
199 | { |
200 | my ($obj,$octets,$chk) = @_; |
2a936312 |
201 | my $str = Encode::decode_utf8($octets); |
4411f3b6 |
202 | if (defined $str) |
203 | { |
204 | $_[1] = '' if $chk; |
205 | return $str; |
206 | } |
207 | return undef; |
208 | } |
209 | |
50d26985 |
210 | sub encode |
4411f3b6 |
211 | { |
212 | my ($obj,$string,$chk) = @_; |
2a936312 |
213 | my $octets = Encode::encode_utf8($string); |
4411f3b6 |
214 | $_[1] = '' if $chk; |
215 | return $octets; |
4411f3b6 |
216 | } |
217 | |
656753f8 |
218 | package Encode::Table; |
50d26985 |
219 | use base 'Encode::Encoding'; |
656753f8 |
220 | |
221 | sub read |
222 | { |
223 | my ($class,$fh,$name,$type) = @_; |
224 | my $rep = $class->can("rep_$type"); |
225 | my ($def,$sym,$pages) = split(/\s+/,scalar(<$fh>)); |
226 | my @touni; |
227 | my %fmuni; |
228 | my $count = 0; |
229 | $def = hex($def); |
656753f8 |
230 | while ($pages--) |
231 | { |
87714904 |
232 | my $line = <$fh>; |
233 | chomp($line); |
234 | my $page = hex($line); |
656753f8 |
235 | my @page; |
236 | my $ch = $page * 256; |
237 | for (my $i = 0; $i < 16; $i++) |
238 | { |
239 | my $line = <$fh>; |
240 | for (my $j = 0; $j < 16; $j++) |
241 | { |
242 | my $val = hex(substr($line,0,4,'')); |
243 | if ($val || !$ch) |
244 | { |
245 | my $uch = chr($val); |
246 | push(@page,$uch); |
87714904 |
247 | $fmuni{$uch} = $ch; |
656753f8 |
248 | $count++; |
249 | } |
250 | else |
251 | { |
252 | push(@page,undef); |
253 | } |
254 | $ch++; |
255 | } |
256 | } |
257 | $touni[$page] = \@page; |
258 | } |
259 | |
260 | return bless {Name => $name, |
261 | Rep => $rep, |
262 | ToUni => \@touni, |
263 | FmUni => \%fmuni, |
264 | Def => $def, |
265 | Num => $count, |
266 | },$class; |
267 | } |
268 | |
269 | sub name { shift->{'Name'} } |
270 | |
271 | sub rep_S { 'C' } |
272 | |
5dcbab34 |
273 | sub rep_D { 'n' } |
656753f8 |
274 | |
5dcbab34 |
275 | sub rep_M { ($_[0] > 255) ? 'n' : 'C' } |
656753f8 |
276 | |
277 | sub representation |
278 | { |
279 | my ($obj,$ch) = @_; |
280 | $ch = 0 unless @_ > 1; |
281 | $obj-{'Rep'}->($ch); |
282 | } |
283 | |
50d26985 |
284 | sub decode |
656753f8 |
285 | { |
bf230f3d |
286 | my ($obj,$str,$chk) = @_; |
656753f8 |
287 | my $rep = $obj->{'Rep'}; |
288 | my $touni = $obj->{'ToUni'}; |
289 | my $uni = ''; |
290 | while (length($str)) |
291 | { |
292 | my $ch = ord(substr($str,0,1,'')); |
bf230f3d |
293 | my $x; |
656753f8 |
294 | if (&$rep($ch) eq 'C') |
295 | { |
bf230f3d |
296 | $x = $touni->[0][$ch]; |
656753f8 |
297 | } |
298 | else |
299 | { |
bf230f3d |
300 | $x = $touni->[$ch][ord(substr($str,0,1,''))]; |
656753f8 |
301 | } |
bf230f3d |
302 | unless (defined $x) |
303 | { |
304 | last if $chk; |
305 | # What do we do here ? |
306 | $x = ''; |
307 | } |
308 | $uni .= $x; |
656753f8 |
309 | } |
bf230f3d |
310 | $_[1] = $str if $chk; |
656753f8 |
311 | return $uni; |
312 | } |
313 | |
50d26985 |
314 | sub encode |
656753f8 |
315 | { |
bf230f3d |
316 | my ($obj,$uni,$chk) = @_; |
656753f8 |
317 | my $fmuni = $obj->{'FmUni'}; |
318 | my $str = ''; |
319 | my $def = $obj->{'Def'}; |
87714904 |
320 | my $rep = $obj->{'Rep'}; |
656753f8 |
321 | while (length($uni)) |
322 | { |
323 | my $ch = substr($uni,0,1,''); |
63eec5db |
324 | my $x = $fmuni->{chr(ord($ch))}; |
bf230f3d |
325 | unless (defined $x) |
326 | { |
327 | last if ($chk); |
328 | $x = $def; |
329 | } |
87714904 |
330 | $str .= pack(&$rep($x),$x); |
331 | } |
332 | $_[1] = $uni if $chk; |
333 | return $str; |
334 | } |
335 | |
9b37254d |
336 | package Encode::iso10646_1; |
50d26985 |
337 | use base 'Encode::Encoding'; |
338 | |
9b37254d |
339 | # Encoding is 16-bit network order Unicode |
340 | # Used for X font encodings |
87714904 |
341 | |
342 | sub name { 'iso10646-1' } |
343 | |
50d26985 |
344 | sub decode |
87714904 |
345 | { |
346 | my ($obj,$str,$chk) = @_; |
347 | my $uni = ''; |
348 | while (length($str)) |
349 | { |
5dcbab34 |
350 | my $code = unpack('n',substr($str,0,2,'')) & 0xffff; |
87714904 |
351 | $uni .= chr($code); |
352 | } |
353 | $_[1] = $str if $chk; |
a12c0f56 |
354 | Encode::utf8_upgrade($uni); |
87714904 |
355 | return $uni; |
356 | } |
357 | |
50d26985 |
358 | sub encode |
87714904 |
359 | { |
360 | my ($obj,$uni,$chk) = @_; |
361 | my $str = ''; |
362 | while (length($uni)) |
363 | { |
364 | my $ch = substr($uni,0,1,''); |
365 | my $x = ord($ch); |
366 | unless ($x < 32768) |
367 | { |
368 | last if ($chk); |
369 | $x = 0; |
370 | } |
5dcbab34 |
371 | $str .= pack('n',$x); |
656753f8 |
372 | } |
bf230f3d |
373 | $_[1] = $uni if $chk; |
656753f8 |
374 | return $str; |
375 | } |
376 | |
2f2b4ff2 |
377 | |
656753f8 |
378 | package Encode::Escape; |
50d26985 |
379 | use base 'Encode::Encoding'; |
380 | |
656753f8 |
381 | use Carp; |
382 | |
383 | sub read |
384 | { |
385 | my ($class,$fh,$name) = @_; |
386 | my %self = (Name => $name, Num => 0); |
387 | while (<$fh>) |
388 | { |
389 | my ($key,$val) = /^(\S+)\s+(.*)$/; |
390 | $val =~ s/^\{(.*?)\}/$1/g; |
391 | $val =~ s/\\x([0-9a-f]{2})/chr(hex($1))/ge; |
392 | $self{$key} = $val; |
393 | } |
394 | return bless \%self,$class; |
395 | } |
396 | |
397 | sub name { shift->{'Name'} } |
398 | |
50d26985 |
399 | sub decode |
656753f8 |
400 | { |
401 | croak("Not implemented yet"); |
402 | } |
403 | |
50d26985 |
404 | sub encode |
656753f8 |
405 | { |
406 | croak("Not implemented yet"); |
407 | } |
408 | |
4411f3b6 |
409 | # switch back to Encode package in case we ever add AutoLoader |
410 | package Encode; |
411 | |
656753f8 |
412 | 1; |
413 | |
2a936312 |
414 | __END__ |
415 | |
4411f3b6 |
416 | =head1 NAME |
417 | |
418 | Encode - character encodings |
419 | |
420 | =head1 SYNOPSIS |
421 | |
422 | use Encode; |
423 | |
424 | =head1 DESCRIPTION |
425 | |
426 | The C<Encode> module provides the interfaces between perl's strings |
427 | and the rest of the system. Perl strings are sequences of B<characters>. |
428 | |
429 | The repertoire of characters that Perl can represent is at least that |
430 | defined by the Unicode Consortium. On most platforms the ordinal values |
431 | of the characters (as returned by C<ord(ch)>) is the "Unicode codepoint" for |
432 | the character (the exceptions are those platforms where the legacy |
433 | encoding is some variant of EBCDIC rather than a super-set of ASCII |
434 | - see L<perlebcdic>). |
435 | |
436 | Traditionaly computer data has been moved around in 8-bit chunks |
437 | often called "bytes". These chunks are also known as "octets" in |
438 | networking standards. Perl is widely used to manipulate data of |
439 | many types - not only strings of characters representing human or |
440 | computer languages but also "binary" data being the machines representation |
441 | of numbers, pixels in an image - or just about anything. |
442 | |
443 | When perl is processing "binary data" the programmer wants perl to process |
444 | "sequences of bytes". This is not a problem for perl - as a byte has 256 |
445 | possible values it easily fits in perl's much larger "logical character". |
446 | |
447 | =head2 TERMINOLOGY |
448 | |
4ac9195f |
449 | =over 4 |
4411f3b6 |
450 | |
451 | =item * |
452 | |
453 | I<character>: a character in the range 0..(2**32-1) (or more). |
454 | (What perl's strings are made of.) |
455 | |
456 | =item * |
457 | |
458 | I<byte>: a character in the range 0..255 |
459 | (A special case of a perl character.) |
460 | |
461 | =item * |
462 | |
463 | I<octet>: 8 bits of data, with ordinal values 0..255 |
464 | (Term for bytes passed to or from a non-perl context, e.g. disk file.) |
465 | |
466 | =back |
467 | |
468 | The marker [INTERNAL] marks Internal Implementation Details, in |
469 | general meant only for those who think they know what they are doing, |
470 | and such details may change in future releases. |
471 | |
472 | =head1 ENCODINGS |
473 | |
474 | =head2 Characteristics of an Encoding |
475 | |
476 | An encoding has a "repertoire" of characters that it can represent, |
477 | and for each representable character there is at least one sequence of |
478 | octets that represents it. |
479 | |
480 | =head2 Types of Encodings |
481 | |
482 | Encodings can be divided into the following types: |
483 | |
484 | =over 4 |
485 | |
486 | =item * Fixed length 8-bit (or less) encodings. |
487 | |
488 | Each character is a single octet so may have a repertoire of up to |
489 | 256 characters. ASCII and iso-8859-* are typical examples. |
490 | |
491 | =item * Fixed length 16-bit encodings |
492 | |
493 | Each character is two octets so may have a repertoire of up to |
494 | 65,536 characters. Unicode's UCS-2 is an example. Also used for |
495 | encodings for East Asian languages. |
496 | |
497 | =item * Fixed length 32-bit encodings. |
498 | |
499 | Not really very "encoded" encodings. The Unicode code points |
500 | are just represented as 4-octet integers. None the less because |
501 | different architectures use different representations of integers |
502 | (so called "endian") there at least two disctinct encodings. |
503 | |
504 | =item * Multi-byte encodings |
505 | |
506 | The number of octets needed to represent a character varies. |
507 | UTF-8 is a particularly complex but regular case of a multi-byte |
508 | encoding. Several East Asian countries use a multi-byte encoding |
509 | where 1-octet is used to cover western roman characters and Asian |
510 | characters get 2-octets. |
511 | (UTF-16 is strictly a multi-byte encoding taking either 2 or 4 octets |
512 | to represent a Unicode code point.) |
513 | |
514 | =item * "Escape" encodings. |
515 | |
516 | These encodings embed "escape sequences" into the octet sequence |
517 | which describe how the following octets are to be interpreted. |
518 | The iso-2022-* family is typical. Following the escape sequence |
519 | octets are encoded by an "embedded" encoding (which will be one |
520 | of the above types) until another escape sequence switches to |
521 | a different "embedded" encoding. |
522 | |
523 | These schemes are very flexible and can handle mixed languages but are |
524 | very complex to process (and have state). |
525 | No escape encodings are implemented for perl yet. |
526 | |
527 | =back |
528 | |
529 | =head2 Specifying Encodings |
530 | |
531 | Encodings can be specified to the API described below in two ways: |
532 | |
533 | =over 4 |
534 | |
535 | =item 1. By name |
536 | |
537 | Encoding names are strings with characters taken from a restricted repertoire. |
538 | See L</"Encoding Names">. |
539 | |
540 | =item 2. As an object |
541 | |
542 | Encoding objects are returned by C<find_encoding($name)>. |
543 | |
544 | =back |
545 | |
546 | =head2 Encoding Names |
547 | |
548 | Encoding names are case insensitive. White space in names is ignored. |
549 | In addition an encoding may have aliases. Each encoding has one "canonical" name. |
550 | The "canonical" name is chosen from the names of the encoding by picking |
551 | the first in the following sequence: |
552 | |
553 | =over 4 |
554 | |
555 | =item * The MIME name as defined in IETF RFC-XXXX. |
556 | |
557 | =item * The name in the IANA registry. |
558 | |
559 | =item * The name used by the the organization that defined it. |
560 | |
561 | =back |
562 | |
563 | Because of all the alias issues, and because in the general case |
564 | encodings have state C<Encode> uses the encoding object internally |
565 | once an operation is in progress. |
566 | |
567 | I<Aliasing is not yet implemented.> |
568 | |
569 | =head1 PERL ENCODING API |
570 | |
571 | =head2 Generic Encoding Interface |
572 | |
573 | =over 4 |
574 | |
575 | =item * |
576 | |
577 | $bytes = encode(ENCODING, $string[, CHECK]) |
578 | |
579 | Encodes string from perl's internal form into I<ENCODING> and returns a |
580 | sequence of octets. |
581 | See L</"Handling Malformed Data">. |
582 | |
583 | =item * |
584 | |
585 | $string = decode(ENCODING, $bytes[, CHECK]) |
586 | |
587 | Decode sequence of octets assumed to be in I<ENCODING> into perls internal |
588 | form and returns the resuting string. |
589 | See L</"Handling Malformed Data">. |
590 | |
591 | =back |
592 | |
593 | =head2 Handling Malformed Data |
594 | |
595 | If CHECK is not set, C<undef> is returned. If the data is supposed to |
596 | be UTF-8, an optional lexical warning (category utf8) is given. |
597 | If CHECK is true but not a code reference, dies. |
598 | |
599 | It would desirable to have a way to indicate that transform should use the |
600 | encodings "replacement character" - no such mechanism is defined yet. |
601 | |
602 | It is also planned to allow I<CHECK> to be a code reference. |
603 | |
604 | This is not yet implemented as there are design issues with what its arguments |
605 | should be and how it returns its results. |
606 | |
607 | =over 4 |
608 | |
609 | =item Scheme 1 |
610 | |
611 | Passed remaining fragment of string being processed. |
612 | Modifies it in place to remove bytes/characters it can understand |
613 | and returns a string used to represent them. |
614 | e.g. |
615 | |
616 | sub fixup { |
617 | my $ch = substr($_[0],0,1,''); |
618 | return sprintf("\x{%02X}",ord($ch); |
619 | } |
620 | |
621 | This scheme is close to how underlying C code for Encode works, but gives |
622 | the fixup routine very little context. |
623 | |
624 | =item Scheme 2 |
625 | |
626 | Passed original string, and an index into it of the problem area, |
627 | and output string so far. |
628 | Appends what it will to output string and returns new index into |
629 | original string. |
630 | e.g. |
631 | |
632 | sub fixup { |
633 | # my ($s,$i,$d) = @_; |
634 | my $ch = substr($_[0],$_[1],1); |
635 | $_[2] .= sprintf("\x{%02X}",ord($ch); |
636 | return $_[1]+1; |
637 | } |
638 | |
639 | This scheme gives maximal control to the fixup routine but is more complicated |
640 | to code, and may need internals of Encode to be tweaked to keep original |
641 | string intact. |
642 | |
643 | =item Other Schemes |
644 | |
645 | Hybrids of above. |
646 | |
647 | Multiple return values rather than in-place modifications. |
648 | |
649 | Index into the string could be pos($str) allowing s/\G...//. |
650 | |
651 | =back |
652 | |
653 | =head2 UTF-8 / utf8 |
654 | |
655 | The Unicode consortium defines the UTF-8 standard as a way of encoding |
656 | the entire Unicode repertiore as sequences of octets. This encoding |
657 | is expected to become very widespread. Perl can use this form internaly |
658 | to represent strings, so conversions to and from this form are particularly |
659 | efficient (as octets in memory do not have to change, just the meta-data |
660 | that tells perl how to treat them). |
661 | |
662 | =over 4 |
663 | |
664 | =item * |
665 | |
666 | $bytes = encode_utf8($string); |
667 | |
668 | The characters that comprise string are encoded in perl's superset of UTF-8 |
669 | and the resulting octets returned as a sequence of bytes. All possible |
670 | characters have a UTF-8 representation so this function cannot fail. |
671 | |
672 | =item * |
673 | |
674 | $string = decode_utf8($bytes [,CHECK]); |
675 | |
676 | The sequence of octets represented by $bytes is decoded from UTF-8 into |
677 | a sequence of logical characters. Not all sequences of octets form valid |
678 | UTF-8 encodings, so it is possible for this call to fail. |
679 | See L</"Handling Malformed Data">. |
680 | |
681 | =back |
682 | |
683 | =head2 Other Encodings of Unicode |
684 | |
685 | UTF-16 is similar to UCS-2, 16 bit or 2-byte chunks. |
686 | UCS-2 can only represent 0..0xFFFF, while UTF-16 has a "surogate pair" |
687 | scheme which allows it to cover the whole Unicode range. |
688 | |
689 | Encode implements big-endian UCS-2 as the encoding "iso10646-1" as that |
690 | happens to be the name used by that representation when used with X11 fonts. |
691 | |
692 | UTF-32 or UCS-4 is 32-bit or 4-byte chunks. Perl's logical characters |
693 | can be considered as being in this form without encoding. An encoding |
694 | to transfer strings in this form (e.g. to write them to a file) would need to |
695 | |
696 | pack('L',map(chr($_),split(//,$string))); # native |
697 | or |
698 | pack('V',map(chr($_),split(//,$string))); # little-endian |
699 | or |
700 | pack('N',map(chr($_),split(//,$string))); # big-endian |
701 | |
702 | depending on the endian required. |
703 | |
704 | No UTF-32 encodings are not yet implemented. |
705 | |
706 | Both UCS-2 and UCS-4 style encodings can have "byte order marks" by representing |
707 | the code point 0xFFFE as the very first thing in a file. |
708 | |
709 | =head1 Encoding and IO |
710 | |
711 | It is very common to want to do encoding transformations when |
712 | reading or writing files, network connections, pipes etc. |
713 | If perl is configured to use the new 'perlio' IO system then |
714 | C<Encode> provides a "layer" (See L<perliol>) which can transform |
715 | data as it is read or written. |
716 | |
717 | open(my $ilyad,'>:encoding(iso8859-7)','ilyad.greek'); |
718 | print $ilyad @epic; |
719 | |
720 | In addition the new IO system can also be configured to read/write |
721 | UTF-8 encoded characters (as noted above this is efficient): |
722 | |
723 | open(my $fh,'>:utf8','anything'); |
724 | print $fh "Any \x{0021} string \N{SMILEY FACE}\n"; |
725 | |
726 | Either of the above forms of "layer" specifications can be made the default |
727 | for a lexical scope with the C<use open ...> pragma. See L<open>. |
728 | |
729 | Once a handle is open is layers can be altered using C<binmode>. |
730 | |
731 | Without any such configuration, or if perl itself is built using |
732 | system's own IO, then write operations assume that file handle accepts |
733 | only I<bytes> and will C<die> if a character larger than 255 is |
734 | written to the handle. When reading, each octet from the handle |
735 | becomes a byte-in-a-character. Note that this default is the same |
736 | behaviour as bytes-only languages (including perl before v5.6) would have, |
737 | and is sufficient to handle native 8-bit encodings e.g. iso-8859-1, |
738 | EBCDIC etc. and any legacy mechanisms for handling other encodings |
739 | and binary data. |
740 | |
741 | In other cases it is the programs responsibility |
742 | to transform characters into bytes using the API above before |
743 | doing writes, and to transform the bytes read from a handle into characters |
744 | before doing "character operations" (e.g. C<lc>, C</\W+/>, ...). |
745 | |
746 | =head1 Encoding How to ... |
747 | |
748 | To do: |
749 | |
750 | =over 4 |
751 | |
752 | =item * IO with mixed content (faking iso-2020-*) |
753 | |
754 | =item * MIME's Content-Length: |
755 | |
756 | =item * UTF-8 strings in binary data. |
757 | |
758 | =item * perl/Encode wrappers on non-Unicode XS modules. |
759 | |
760 | =back |
761 | |
762 | =head1 Messing with Perl's Internals |
763 | |
764 | The following API uses parts of perl's internals in the current implementation. |
765 | As such they are efficient, but may change. |
766 | |
767 | =over 4 |
768 | |
769 | =item * |
770 | |
771 | $num_octets = utf8_upgrade($string); |
772 | |
773 | Converts internal representation of string to the UTF-8 form. |
774 | Returns the number of octets necessary to represent the string as UTF-8. |
775 | |
776 | =item * utf8_downgrade($string[, CHECK]) |
777 | |
778 | Converts internal representation of string to be un-encoded bytes. |
779 | |
780 | =item * is_utf8(STRING [, CHECK]) |
781 | |
782 | [INTERNAL] Test whether the UTF-8 flag is turned on in the STRING. |
783 | If CHECK is true, also checks the data in STRING for being |
784 | well-formed UTF-8. Returns true if successful, false otherwise. |
785 | |
786 | =item * valid_utf8(STRING) |
787 | |
788 | [INTERNAL] Test whether STRING is in a consistent state. |
789 | Will return true if string is held as bytes, or is well-formed UTF-8 |
790 | and has the UTF-8 flag on. |
791 | Main reason for this routine is to allow perl's testsuite to check |
792 | that operations have left strings in a consistent state. |
793 | |
794 | =item * |
795 | |
796 | _utf8_on(STRING) |
797 | |
798 | [INTERNAL] Turn on the UTF-8 flag in STRING. The data in STRING is |
799 | B<not> checked for being well-formed UTF-8. Do not use unless you |
800 | B<know> that the STRING is well-formed UTF-8. Returns the previous |
801 | state of the UTF-8 flag (so please don't test the return value as |
802 | I<not> success or failure), or C<undef> if STRING is not a string. |
803 | |
804 | =item * |
805 | |
806 | _utf8_off(STRING) |
807 | |
808 | [INTERNAL] Turn off the UTF-8 flag in STRING. Do not use frivolously. |
809 | Returns the previous state of the UTF-8 flag (so please don't test the |
810 | return value as I<not> success or failure), or C<undef> if STRING is |
811 | not a string. |
812 | |
813 | =back |
814 | |
4edaa979 |
815 | =head1 IMPLEMENTATION CLASSES |
816 | |
817 | As mentioned above encodings are (in the current implementation at least) |
818 | defined by objects. The mapping of encoding name to object is via the |
819 | C<%Encode::encodings> hash. (It is a package hash to allow XS code to get |
820 | at it.) |
821 | |
822 | The values of the hash can currently be either strings or objects. |
823 | The string form may go away in the future. The string form occurs |
824 | when C<encodings()> has scanned C<@INC> for loadable encodings but has |
825 | not actually loaded the encoding in question. This is because the |
826 | current "loading" process is all perl and a bit slow. |
827 | |
828 | Once an encoding is loaded then value of the hash is object which implements |
829 | the encoding. The object should provide the following interface: |
830 | |
831 | =over 4 |
832 | |
833 | =item -E<gt>name |
834 | |
835 | Should return the string representing the canonical name of the encoding. |
836 | |
837 | =item -E<gt>new_sequence |
838 | |
839 | This is a placeholder for encodings with state. It should return an object |
840 | which implements this interface, all current implementations return the |
841 | original object. |
842 | |
843 | =item -E<gt>encode($string,$check) |
844 | |
845 | Should return the octet sequence representing I<$string>. If I<$check> is true |
846 | it should modify I<$string> in place to remove the converted part (i.e. |
847 | the whole string unless there is an error). |
848 | If an error occurs it should return the octet sequence for the |
849 | fragment of string that has been converted, and modify $string in-place |
850 | to remove the converted part leaving it starting with the problem fragment. |
851 | |
852 | If check is is false then C<encode> should make a "best effort" to convert |
853 | the string - for example by using a replacement character. |
854 | |
855 | =item -E<gt>decode($octets,$check) |
856 | |
857 | Should return the string that I<$octets> represents. If I<$check> is true |
858 | it should modify I<$octets> in place to remove the converted part (i.e. |
859 | the whole sequence unless there is an error). |
860 | If an error occurs it should return the fragment of string |
861 | that has been converted, and modify $octets in-place to remove the converted part |
862 | leaving it starting with the problem fragment. |
863 | |
864 | If check is is false then C<decode> should make a "best effort" to convert |
865 | the string - for example by using Unicode's "\x{FFFD}" as a replacement character. |
866 | |
867 | =back |
868 | |
869 | It should be noted that the check behaviour is different from the outer |
870 | public API. The logic is that the "unchecked" case is useful when |
871 | encoding is part of a stream which may be reporting errors (e.g. STDERR). |
872 | In such cases it is desirable to get everything through somehow without |
873 | causing additional errors which obscure the original one. Also the encoding |
874 | is best placed to know what the correct replacement character is, so if that |
875 | is the desired behaviour then letting low level code do it is the most efficient. |
876 | |
877 | In contrast if check is true, the scheme above allows the encoding to do as |
878 | much as it can and tell layer above how much that was. What is lacking |
879 | at present is a mechanism to report what went wrong. The most likely interface |
880 | will be an additional method call to the object, or perhaps |
881 | (to avoid forcing per-stream objects on otherwise stateless encodings) |
882 | and additional parameter. |
883 | |
884 | It is also highly desirable that encoding classes inherit from C<Encode::Encoding> |
885 | as a base class. This allows that class to define additional behaviour for |
886 | all encoding objects. |
887 | |
888 | =head2 Compiled Encodings |
889 | |
890 | F<Encode.xs> provides a class C<Encode::XS> which provides the interface described |
891 | above. It calls a generic octet-sequence to octet-sequence "engine" that is |
892 | driven by tables (defined in F<encengine.c>). The same engine is used for both |
893 | encode and decode. C<Encode:XS>'s C<encode> forces perl's characters to their UTF-8 form |
894 | and then treats them as just another multibyte encoding. C<Encode:XS>'s C<decode> transforms |
895 | the sequence and then turns the UTF-8-ness flag as that is the form that the tables |
896 | are defined to produce. For details of the engine see the comments in F<encengine.c>. |
897 | |
898 | The tables are produced by the perl script F<compile> (the name needs to change so |
899 | we can eventually install it somewhere). F<compile> can currently read two formats: |
900 | |
901 | =over 4 |
902 | |
903 | =item *.enc |
904 | |
905 | This is a coined format used by Tcl. It is documented in Encode/EncodeFormat.pod. |
906 | |
907 | =item *.ucm |
908 | |
909 | This is the semi-standard format used by IBM's ICU package. |
910 | |
911 | =back |
912 | |
913 | F<compile> can write the following forms: |
914 | |
915 | =over 4 |
916 | |
917 | =item *.ucm |
918 | |
919 | See above - the F<Encode/*.ucm> files provided with the distribution have |
920 | been created from the original Tcl .enc files using this approach. |
921 | |
922 | =item *.c |
923 | |
924 | Produces tables as C data structures - this is used to build in encodings |
925 | into F<Encode.so>/F<Encode.dll>. |
926 | |
927 | =item *.xs |
928 | |
929 | In theory this allows encodings to be stand-alone loadable perl extensions. |
930 | The process has not yet been tested. The plan is to use this approach |
931 | for large East Asian encodings. |
932 | |
933 | =back |
934 | |
935 | The set of encodings built-in to F<Encode.so>/F<Encode.dll> is determined by |
936 | F<Makefile.PL>. The current set is as follows: |
937 | |
938 | =over 4 |
939 | |
940 | =item ascii and iso-8859-* |
941 | |
942 | That is all the common 8-bit "western" encodings. |
943 | |
944 | =item IBM-1047 and two other variants of EBCDIC. |
945 | |
946 | These are the same variants that are supported by EBCDIC perl as "native" encodings. |
947 | They are included to prove "reversibility" of some constructs in EBCDIC perl. |
948 | |
949 | =item symbol and dingbats as used by Tk on X11. |
950 | |
951 | (The reason Encode got started was to support perl/Tk.) |
952 | |
953 | =back |
954 | |
955 | That set is rather ad. hoc. and has been driven by the needs of the tests rather |
956 | than the needs of typical applications. It is likely to be rationalized. |
957 | |
4411f3b6 |
958 | =head1 SEE ALSO |
959 | |
960 | L<perlunicode>, L<perlebcdic>, L<perlfunc/open> |
961 | |
962 | =cut |
963 | |
964 | |
2a936312 |
965 | |