[p5sagit/p5-mst-13.2.git] / ext / Encode / lib / Encode / Unicode.pm

package Encode::Unicode;

use strict;
use warnings;

our $VERSION = do { my @r = (q$Revision: 1.29 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r };

#
# Aux. subs & constants
#

sub FBCHAR(){ 0xFFFd }
sub BOM_BE(){ 0xFeFF }
sub BOM16LE(){ 0xFFFe }
sub BOM32LE(){ 0xFFFe0000 }

sub valid_ucs2($){
    return 
	(0 <= $_[0] && $_[0] < 0xD800) 
	    || 	( 0xDFFF < $_[0] && $_[0] <= 0xFFFF);
}

sub issurrogate($){   0xD800 <= $_[0]  && $_[0] <= 0xDFFF }
sub isHiSurrogate($){ 0xD800 <= $_[0]  && $_[0] <  0xDC00 }
sub isLoSurrogate($){ 0xDC00 <= $_[0]  && $_[0] <= 0xDFFF }

sub ensurrogate($){
    use integer; # we have divisions
    my $uni = shift;
    my  $hi = ($uni - 0x10000) / 0x400 + 0xD800;
    my  $lo = ($uni - 0x10000) % 0x400 + 0xDC00;
    return ($hi, $lo);
}

sub desurrogate($$){
    my ($hi, $lo) = @_;
    return 0x10000 + ($hi - 0xD800)*0x400 + ($lo - 0xDC00);
}

sub Mask { {2 => 0xffff,  4 => 0xffffffff} }

#
# Object Generator 8 transcoders all at once!
#

require Encode;
for my $name (qw(UTF-16 UTF-16BE UTF-16LE
                 UTF-32 UTF-32BE UTF-32LE
                        UCS-2BE  UCS-2LE))
{
    my ($size, $endian, $ucs2, $mask);
    $name =~ /^(\w+)-(\d+)(\w*)$/o;
    if ($ucs2 = ($1 eq 'UCS')){
	$size = 2;
    }else{
	$size = $2/8;
    }
    $endian = ($3 eq 'BE') ? 'n' : ($3 eq 'LE') ? 'v' : '' ;
    $size == 4 and $endian = uc($endian);

    $Encode::Encoding{$name} = 	
	bless {
	       Name   =>   $name,
	       size   =>   $size,
	       endian => $endian,
	       ucs2   =>   $ucs2,
	      } => __PACKAGE__;

}

sub name { shift->{'Name'} }
sub new_sequence { $_[0] };

#
# two implementation of (en|de)code exist.  *_modern use
# an array and *_classic stick with substr.  *_classic is much
# slower but more memory conservative.  *_modern is default.

sub set_transcoder{
    no warnings qw(redefine);
    my $type = shift;
    if     ($type eq "modern"){
	*decode = \&decode_modern;
	*encode = \&encode_modern;
    }elsif($type eq "classic"){
	*decode = \&decode_classic;
	*encode = \&encode_classic;
    }else{
	require Carp; 
	Carp::croak __PACKAGE__, "::set_transcoder(modern|classic)";
    }
}

set_transcoder("modern");

#
# *_modern are much faster but guzzle more memory
#

sub decode_modern
{
    my ($obj, $str, $chk ) = @_;
    my ($size, $endian, $ucs2) = @$obj{qw(size endian ucs2)};

    # warn "$size, $endian, $ucs2";
    $endian ||= BOMB($size, substr($str, 0, $size, ''))
	or poisoned2death($obj, "Where's the BOM?");
    my  $mask = Mask->{$size};
    my $utf8   = '';
    my @ord = unpack("$endian*", $str);
    undef $str; # to conserve memory
    while (@ord){
	my $ord = shift @ord;
	unless ($size == 4 or valid_ucs2($ord &= $mask)){
	    if ($ucs2){
		$chk and 
		    poisoned2death($obj, "no surrogates allowed", $ord);
		shift @ord; # skip the next one as well
		$ord = FBCHAR;
	    }else{
		unless (isHiSurrogate($ord)){
		    poisoned2death($obj, "Malformed HI surrogate", $ord);
		}
		my $lo = shift @ord;
		unless (isLoSurrogate($lo &= $mask)){
		    poisoned2death($obj, "Malformed LO surrogate", $ord, $lo);
		}
		$ord = desurrogate($ord, $lo);
	    }
	}
	$utf8 .= chr($ord);
    }
    utf8::upgrade($utf8);
    return $utf8;
}

sub encode_modern
{
    my ($obj, $utf8, $chk) = @_;
    my ($size, $endian, $ucs2) = @$obj{qw(size endian ucs2)};
    my @str = ();
    unless ($endian){
	$endian = ($size == 4) ? 'N' : 'n';
	push @str, BOM_BE;
    }
    my @ord = unpack("U*", $utf8);
    undef $utf8; # to conserve memory
    for my $ord (@ord){
	unless ($size == 4 or valid_ucs2($ord)) {
	    unless(issurrogate($ord)){
		if ($ucs2){
		    $chk and 
			poisoned2death($obj, "code point too high", $ord);

		    push @str, FBCHAR;
		}else{
		 
		    push @str, ensurrogate($ord);
		}
	    }else{  # not supposed to happen
		push @str, FBCHAR;
	    }
	}else{
	    push @str, $ord;
	}
    }
    return pack("$endian*", @str);
}

#
# *_classic are slower but more memory conservative
#

sub decode_classic
{
    my ($obj, $str, $chk ) = @_;
    my ($size, $endian, $ucs2) = @$obj{qw(size endian ucs2)};

    # warn "$size, $endian, $ucs2";
    $endian ||= BOMB($size, substr($str, 0, $size, ''))
	or poisoned2death($obj, "Where's the BOM?");
    my  $mask = Mask->{$size};
    my $utf8   = '';
    my @ord = unpack("$endian*", $str);
    while (length($str)){
	 my $ord = unpack($endian, substr($str, 0, $size, ''));
	unless ($size == 4 or valid_ucs2($ord &= $mask)){
	    if ($ucs2){
		$chk and 
		    poisoned2death($obj, "no surrogates allowed", $ord);
		substr($str,0,$size,''); # skip the next one as well
		$ord = FBCHAR;
	    }else{
		unless (isHiSurrogate($ord)){
		    poisoned2death($obj, "Malformed HI surrogate", $ord);
		}
		my $lo = unpack($endian ,substr($str,0,$size,''));
		unless (isLoSurrogate($lo &= $mask)){
		    poisoned2death($obj, "Malformed LO surrogate", $ord, $lo);
		}
		$ord = desurrogate($ord, $lo);
	    }
	}
	$utf8 .= chr($ord);
    }
    utf8::upgrade($utf8);
    return $utf8;
}

sub encode_classic
{
    my ($obj, $utf8, $chk) = @_;
    my ($size, $endian, $ucs2) = @$obj{qw(size endian ucs2)};
    # warn join ", ", $size, $ucs2, $endian, $mask;
    my $str   = '';
    unless ($endian){
	$endian = ($size == 4) ? 'N' : 'n';
	$str .= pack($endian, BOM_BE);
    }
    while (length($utf8)){
	my $ord  = ord(substr($utf8,0,1,''));
	unless ($size == 4 or valid_ucs2($ord)) {
	    unless(issurrogate($ord)){
		if ($ucs2){
		    $chk and 
			poisoned2death($obj, "code point too high", $ord);
		    $str .= pack($endian, FBCHAR);
		}else{
		    $str .= pack($endian.2, ensurrogate($ord));
		}
	    }else{  # not supposed to happen
		$str .= pack($endian, FBCHAR);
	    }
	}else{
	    $str .= pack($endian, $ord);
	}
    }
    return $str;
}

sub BOMB {
    my ($size, $bom) = @_;
    my $N = $size == 2 ? 'n' : 'N';
    my $ord = unpack($N, $bom);
    return ($ord eq BOM_BE) ? $N : 
	($ord eq BOM16LE) ? 'v' : ($ord eq BOM32LE) ? 'V' : undef;
}

sub poisoned2death{
    my $obj = shift;
    my $msg = shift;
    my $pair = join(", ", map {sprintf "\\x%x", $_} @_);
    require Carp;
    Carp::croak $obj->name, ":", $msg, "<$pair>.", caller;
}

1;
__END__

=head1 NAME

Encode::Unicode -- Various Unicode Transform Format

=cut

=head1 SYNOPSIS

    use Encode qw/encode decode/; 
    $ucs2 = encode("UCS-2BE", $utf8);
    $utf8 = decode("UCS-2BE", $ucs2);

=head1 ABSTRACT

This module implements all Character Encoding Schemes of Unicode that
are officially documented by Unicode Consortium (except, of course,
for UTF-8, which is a native format in perl).

=over 4

=item L<http://www.unicode.org/glossary/> says:

I<Character Encoding Scheme> A character encoding form plus byte
serialization. There are seven character encoding schemes in Unicode:
UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE and UTF-32LE.

=item Quick Reference

                Decodes from ord(N)           Encodes chr(N) to...
       octet/char BOM S.P d800-dfff  ord > 0xffff     \x{1abcd} ==
  ---------------+-----------------+------------------------------
  UCS-2BE	2   N   N  is bogus                  Not Available
  UCS-2LE       2   N   N     bogus                  Not Available
  UTF-16      2/4   Y   Y  is   S.P           S.P            BE/LE
  UTF-16BE    2/4   N   Y       S.P           S.P    0xd82a,0xdfcd
  UTF-16LE	2   N   Y       S.P           S.P    0x2ad8,0xcddf
  UTF-32	4   Y   -  is bogus         As is            BE/LE
  UTF-32BE	4   N   -     bogus         As is       0x0010abcd
  UTF-32LE	4   N   -     bogus         As is       0xcdab1000
  UTF-8       1-4   -   -     bogus   >= 4 octets   \xf0\x9a\af\8d
  ---------------+-----------------+------------------------------

=back

=head1 Size, Endianness, and BOM

You can categorize these CES by 3 criteria;  Size of each character,
Endianness, and Byte Order Mark.

=head2 by Size

UCS-2 is a fixed-length encoding with each character taking 16 bits.
It B<does not> support I<Surrogate Pairs>.  When a surrogate pair is
encountered during decode(), its place is filled with \xFFFD without
I<CHECK> or croaks if I<CHECK>.  When a character whose ord value is
larger than 0xFFFF is encountered, it uses 0xFFFD without I<CHECK> or
croaks if <CHECK>.

UTF-16 is almost the same as UCS-2 but it supports I<Surrogate Pairs>.
When it encounters a high surrogate (0xD800-0xDBFF), it fetches the
following low surrogate (0xDC00-0xDFFF), C<desurrogate>s them to form a
character.  Bogus surrogates result in death.  When \x{10000} or above
is encountered during encode(), it C<ensurrogate>s them and pushes the
surrogate pair to the output stream.

UTF-32 is a fixed-length encoding with each character taking 32 bits.
Since it is 32-bit there is no need for I<Surrogate Pairs>.

=head2 by Endianness

First (and now failed) goal of Unicode was to map all character
repertories into a fixed-length integer so programmers are happy.
Since each character is either I<short> or I<long> in C, you have to
put endianness of each platform when you pass data to one another.

Anything marked as BE is Big Endian (or network byte order) and LE is
Little Endian (aka VAX byte order).  For anything without, a character
called Byte Order Mark (BOM) is prepended to the head of string.

=over 4

=item BOM as integer when fetched in network byte order

              16         32 bits/char
  -------------------------
  BE      0xFeFF 0x0000FeFF
  LE      0xFFeF 0xFFFe0000
  -------------------------

=back
 
This modules handles BOM as follows.

=over 4

=item *

When BE or LE is explicitly stated as the name of encoding, BOM is
simply treated as one of characters (ZERO WIDTH NO-BREAK SPACE).

=item *

When BE or LE is omitted during decode(), it checks if BOM is in the
beginning of the string and if found endianness is set to what BOM
says.  If not found, dies. 

=item *

When BE or LE is omitted during encode(), it returns a BE-encoded
string with BOM prepended.  So when you want to encode a whole text
file, make sure you encode() by whole text, not line by line or each
line, not file, is prepended with BOMs.

=item *

C<UCS-2> is an exception.  Unlike others this is an alias of UCS-2BE.
UCS-2 is already registered by IANA and others that way.

=back

=head1 Surrogate Pairs

To say the least, surrogate pairs were the biggest mistake of the
Unicode Consortium.  But according to the late Douglas Adams in I<The
Hitchhiker's Guide to the Galaxy> Trilogy, C<In the beginning the
Universe was created. This has made a lot of people very angry and
been widely regarded as a bad move>.  Their mistake was not of this
magnitude so let's forgive them.

(I don't dare make any comparison with Unicode Consortium and the
Vogons here ;)  Or, comparing Encode to Babel Fish is completely
appropriate -- if you can only stick this into your ear :)

Surrogate pairs were born when Unicode Consortium finally
admitted that 16 bits were not big enough to hold all the world's
character repertoire.  But they have already made UCS-2 16-bit.  What
do we do?

Back then 0xD800-0xDFFF was not allocated.  Let's split them half and
use the first half to represent C<upper half of a character> and the
latter C<lower half of a character>.  That way you can represent 1024
* 1024 = 1048576 more characters.  Now we can store character ranges
up to \x{10ffff} even with 16-bit encodings.  This pair of
half-character is now called a I<Surrogate Pair> and UTF-16 is the
name of the encoding that embraces them.

Here is a fomula to ensurrogate a Unicode character \x{10000} and
above;

  $hi = ($uni - 0x10000) / 0x400 + 0xD800;
  $lo = ($uni - 0x10000) % 0x400 + 0xDC00;

And to desurrogate;

 $uni = 0x10000 + ($hi - 0xD800) * 0x400 + ($lo - 0xDC00);

Note this move has made \x{D800}-\x{DFFF} into a forbidden zone but
perl does not prohibit the use of characters within this range.  To perl, 
every one of \x{0000_0000} up to \x{ffff_ffff} (*) is I<a character>.

  (*) or \x{ffff_ffff_ffff_ffff} if your perl is compiled with 64-bit
  integer support! (**)

  (**) Is anything beyond \x{11_0000} still Unicode :?

=head1 SEE ALSO

L<Encode>, L<http://www.unicode.org/glossary/>,

RFC 2781 L<http://rfc.net/rfc2781.html>,

L<http://www.unicode.org/unicode/faq/utf_bom.html>

Ch. 15, pp. 403 of C<Programming Perl (3rd Edition)>
by Larry Wall, Tom Christiansen, Jon Orwant; 
O'Reilly & Associates; ISBN 0-596-00027-8

=cut
Commit	Line	Data
f2a2953c	1	package Encode::Unicode;
f2a2953c	2
df1df145	3	use strict;
f2a2953c	4	use warnings;
f2a2953c	5
fcb875d4	6	our $VERSION = do { my @r = (q$Revision: 1.29 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r };
f2a2953c	7
	8	#
	9	# Aux. subs & constants
	10	#
	11
	12	sub FBCHAR(){ 0xFFFd }
	13	sub BOM_BE(){ 0xFeFF }
	14	sub BOM16LE(){ 0xFFFe }
fdd579e2	15	sub BOM32LE(){ 0xFFFe0000 }
f2a2953c	16
f2a2953c	17	sub valid_ucs2($){
fcb875d4	18	return
	19	(0 <= $_[0] && $_[0] < 0xD800)
	20	\|\| ( 0xDFFF < $_[0] && $_[0] <= 0xFFFF);
f2a2953c	21	}
	22
	23	sub issurrogate($){ 0xD800 <= $_[0] && $_[0] <= 0xDFFF }
	24	sub isHiSurrogate($){ 0xD800 <= $_[0] && $_[0] < 0xDC00 }
	25	sub isLoSurrogate($){ 0xDC00 <= $_[0] && $_[0] <= 0xDFFF }
	26
	27	sub ensurrogate($){
	28	use integer; # we have divisions
	29	my $uni = shift;
	30	my $hi = ($uni - 0x10000) / 0x400 + 0xD800;
	31	my $lo = ($uni - 0x10000) % 0x400 + 0xDC00;
	32	return ($hi, $lo);
	33	}
	34
	35	sub desurrogate($$){
	36	my ($hi, $lo) = @_;
	37	return 0x10000 + ($hi - 0xD800)*0x400 + ($lo - 0xDC00);
	38	}
ee981de6	39
f2a2953c	40	sub Mask { {2 => 0xffff, 4 => 0xffffffff} }
df1df145	41
f2a2953c	42	#
	43	# Object Generator 8 transcoders all at once!
	44	#
df1df145	45
f2a2953c	46	require Encode;
	47	for my $name (qw(UTF-16 UTF-16BE UTF-16LE
	48	UTF-32 UTF-32BE UTF-32LE
	49	UCS-2BE UCS-2LE))
df1df145	50	{
f2a2953c	51	my ($size, $endian, $ucs2, $mask);
	52	$name =~ /^(\w+)-(\d+)(\w*)$/o;
	53	if ($ucs2 = ($1 eq 'UCS')){
	54	$size = 2;
	55	}else{
	56	$size = $2/8;
df1df145	57	}
f2a2953c	58	$endian = ($3 eq 'BE') ? 'n' : ($3 eq 'LE') ? 'v' : '' ;
	59	$size == 4 and $endian = uc($endian);
	60
	61	$Encode::Encoding{$name} =
	62	bless {
	63	Name => $name,
	64	size => $size,
	65	endian => $endian,
	66	ucs2 => $ucs2,
c731e18e	67	} => __PACKAGE__;
f2a2953c	68
df1df145	69	}
df1df145	70
f2a2953c	71	sub name { shift->{'Name'} }
	72	sub new_sequence { $_[0] };
	73
	74	#
fcb875d4	75	# two implementation of (en\|de)code exist. *_modern use
	76	# an array and _classic stick with substr. _classic is much
	77	# slower but more memory conservative. *_modern is default.
f2a2953c	78
	79	sub set_transcoder{
	80	no warnings qw(redefine);
	81	my $type = shift;
	82	if ($type eq "modern"){
	83	*decode = \&decode_modern;
	84	*encode = \&encode_modern;
	85	}elsif($type eq "classic"){
	86	*decode = \&decode_classic;
	87	*encode = \&encode_classic;
	88	}else{
fcb875d4	89	require Carp;
f2a2953c	90	Carp::croak __PACKAGE__, "::set_transcoder(modern\|classic)";
	91	}
	92	}
	93
	94	set_transcoder("modern");
	95
	96	#
	97	# *_modern are much faster but guzzle more memory
	98	#
	99
	100	sub decode_modern
df1df145	101	{
f2a2953c	102	my ($obj, $str, $chk ) = @_;
	103	my ($size, $endian, $ucs2) = @$obj{qw(size endian ucs2)};
	104
	105	# warn "$size, $endian, $ucs2";
	106	$endian \|\|= BOMB($size, substr($str, 0, $size, ''))
	107	or poisoned2death($obj, "Where's the BOM?");
	108	my $mask = Mask->{$size};
	109	my $utf8 = '';
	110	my @ord = unpack("$endian*", $str);
	111	undef $str; # to conserve memory
	112	while (@ord){
	113	my $ord = shift @ord;
	114	unless ($size == 4 or valid_ucs2($ord &= $mask)){
	115	if ($ucs2){
fcb875d4	116	$chk and
f2a2953c	117	poisoned2death($obj, "no surrogates allowed", $ord);
	118	shift @ord; # skip the next one as well
	119	$ord = FBCHAR;
	120	}else{
	121	unless (isHiSurrogate($ord)){
	122	poisoned2death($obj, "Malformed HI surrogate", $ord);
	123	}
	124	my $lo = shift @ord;
	125	unless (isLoSurrogate($lo &= $mask)){
	126	poisoned2death($obj, "Malformed LO surrogate", $ord, $lo);
	127	}
	128	$ord = desurrogate($ord, $lo);
	129	}
	130	}
	131	$utf8 .= chr($ord);
df1df145	132	}
f2a2953c	133	utf8::upgrade($utf8);
	134	return $utf8;
	135	}
	136
	137	sub encode_modern
	138	{
	139	my ($obj, $utf8, $chk) = @_;
	140	my ($size, $endian, $ucs2) = @$obj{qw(size endian ucs2)};
	141	my @str = ();
	142	unless ($endian){
	143	$endian = ($size == 4) ? 'N' : 'n';
	144	push @str, BOM_BE;
	145	}
	146	my @ord = unpack("U*", $utf8);
	147	undef $utf8; # to conserve memory
	148	for my $ord (@ord){
	149	unless ($size == 4 or valid_ucs2($ord)) {
	150	unless(issurrogate($ord)){
	151	if ($ucs2){
fcb875d4	152	$chk and
f2a2953c	153	poisoned2death($obj, "code point too high", $ord);
	154
	155	push @str, FBCHAR;
	156	}else{
fcb875d4	157
f2a2953c	158	push @str, ensurrogate($ord);
	159	}
	160	}else{ # not supposed to happen
	161	push @str, FBCHAR;
	162	}
	163	}else{
	164	push @str, $ord;
	165	}
	166	}
	167	return pack("$endian*", @str);
	168	}
	169
	170	#
	171	# *_classic are slower but more memory conservative
	172	#
	173
	174	sub decode_classic
	175	{
	176	my ($obj, $str, $chk ) = @_;
	177	my ($size, $endian, $ucs2) = @$obj{qw(size endian ucs2)};
	178
	179	# warn "$size, $endian, $ucs2";
	180	$endian \|\|= BOMB($size, substr($str, 0, $size, ''))
	181	or poisoned2death($obj, "Where's the BOM?");
	182	my $mask = Mask->{$size};
	183	my $utf8 = '';
	184	my @ord = unpack("$endian*", $str);
	185	while (length($str)){
	186	my $ord = unpack($endian, substr($str, 0, $size, ''));
	187	unless ($size == 4 or valid_ucs2($ord &= $mask)){
	188	if ($ucs2){
fcb875d4	189	$chk and
f2a2953c	190	poisoned2death($obj, "no surrogates allowed", $ord);
	191	substr($str,0,$size,''); # skip the next one as well
	192	$ord = FBCHAR;
	193	}else{
	194	unless (isHiSurrogate($ord)){
	195	poisoned2death($obj, "Malformed HI surrogate", $ord);
	196	}
	197	my $lo = unpack($endian ,substr($str,0,$size,''));
	198	unless (isLoSurrogate($lo &= $mask)){
	199	poisoned2death($obj, "Malformed LO surrogate", $ord, $lo);
	200	}
	201	$ord = desurrogate($ord, $lo);
	202	}
	203	}
	204	$utf8 .= chr($ord);
	205	}
	206	utf8::upgrade($utf8);
	207	return $utf8;
	208	}
	209
	210	sub encode_classic
	211	{
	212	my ($obj, $utf8, $chk) = @_;
	213	my ($size, $endian, $ucs2) = @$obj{qw(size endian ucs2)};
	214	# warn join ", ", $size, $ucs2, $endian, $mask;
	215	my $str = '';
	216	unless ($endian){
	217	$endian = ($size == 4) ? 'N' : 'n';
	218	$str .= pack($endian, BOM_BE);
	219	}
	220	while (length($utf8)){
	221	my $ord = ord(substr($utf8,0,1,''));
	222	unless ($size == 4 or valid_ucs2($ord)) {
	223	unless(issurrogate($ord)){
	224	if ($ucs2){
fcb875d4	225	$chk and
f2a2953c	226	poisoned2death($obj, "code point too high", $ord);
	227	$str .= pack($endian, FBCHAR);
	228	}else{
	229	$str .= pack($endian.2, ensurrogate($ord));
	230	}
	231	}else{ # not supposed to happen
	232	$str .= pack($endian, FBCHAR);
	233	}
	234	}else{
	235	$str .= pack($endian, $ord);
	236	}
	237	}
	238	return $str;
	239	}
	240
	241	sub BOMB {
	242	my ($size, $bom) = @_;
	243	my $N = $size == 2 ? 'n' : 'N';
	244	my $ord = unpack($N, $bom);
fcb875d4	245	return ($ord eq BOM_BE) ? $N :
f2a2953c	246	($ord eq BOM16LE) ? 'v' : ($ord eq BOM32LE) ? 'V' : undef;
	247	}
	248
	249	sub poisoned2death{
	250	my $obj = shift;
	251	my $msg = shift;
	252	my $pair = join(", ", map {sprintf "\\x%x", $_} @_);
	253	require Carp;
	254	Carp::croak $obj->name, ":", $msg, "<$pair>.", caller;
df1df145	255	}
	256
	257	1;
	258	__END__
67d7b5ef	259
	260	=head1 NAME
	261
f2a2953c	262	Encode::Unicode -- Various Unicode Transform Format
67d7b5ef	263
67d7b5ef	264	=cut
f2a2953c	265
	266	=head1 SYNOPSIS
	267
fcb875d4	268	use Encode qw/encode decode/;
f2a2953c	269	$ucs2 = encode("UCS-2BE", $utf8);
	270	$utf8 = decode("UCS-2BE", $ucs2);
	271
	272	=head1 ABSTRACT
	273
	274	This module implements all Character Encoding Schemes of Unicode that
	275	are officially documented by Unicode Consortium (except, of course,
	276	for UTF-8, which is a native format in perl).
	277
	278	=over 4
	279
	280	=item L<http://www.unicode.org/glossary/> says:
	281
	282	I<Character Encoding Scheme> A character encoding form plus byte
	283	serialization. There are seven character encoding schemes in Unicode:
	284	UTF-8, UTF-16, UTF-16BE, UTF-16LE, UTF-32, UTF-32BE and UTF-32LE.
	285
	286	=item Quick Reference
	287
	288	Decodes from ord(N) Encodes chr(N) to...
	289	octet/char BOM S.P d800-dfff ord > 0xffff \x{1abcd} ==
	290	---------------+-----------------+------------------------------
	291	UCS-2BE 2 N N is bogus Not Available
	292	UCS-2LE 2 N N bogus Not Available
	293	UTF-16 2/4 Y Y is S.P S.P BE/LE
	294	UTF-16BE 2/4 N Y S.P S.P 0xd82a,0xdfcd
	295	UTF-16LE 2 N Y S.P S.P 0x2ad8,0xcddf
	296	UTF-32 4 Y - is bogus As is BE/LE
	297	UTF-32BE 4 N - bogus As is 0x0010abcd
	298	UTF-32LE 4 N - bogus As is 0xcdab1000
	299	UTF-8 1-4 - - bogus >= 4 octets \xf0\x9a\af\8d
	300	---------------+-----------------+------------------------------
	301
	302	=back
	303
	304	=head1 Size, Endianness, and BOM
	305
	306	You can categorize these CES by 3 criteria; Size of each character,
	307	Endianness, and Byte Order Mark.
	308
	309	=head2 by Size
	310
	311	UCS-2 is a fixed-length encoding with each character taking 16 bits.
fcb875d4	312	It B<does not> support I<Surrogate Pairs>. When a surrogate pair is
	313	encountered during decode(), its place is filled with \xFFFD without
	314	I<CHECK> or croaks if I<CHECK>. When a character whose ord value is
	315	larger than 0xFFFF is encountered, it uses 0xFFFD without I<CHECK> or
	316	croaks if <CHECK>.
f2a2953c	317
fcb875d4	318	UTF-16 is almost the same as UCS-2 but it supports I<Surrogate Pairs>.
f2a2953c	319	When it encounters a high surrogate (0xD800-0xDBFF), it fetches the
fcb875d4	320	following low surrogate (0xDC00-0xDFFF), C<desurrogate>s them to form a
f2a2953c	321	character. Bogus surrogates result in death. When \x{10000} or above
fcb875d4	322	is encountered during encode(), it C<ensurrogate>s them and pushes the
f2a2953c	323	surrogate pair to the output stream.
	324
	325	UTF-32 is a fixed-length encoding with each character taking 32 bits.
fcb875d4	326	Since it is 32-bit there is no need for I<Surrogate Pairs>.
f2a2953c	327
	328	=head2 by Endianness
	329
	330	First (and now failed) goal of Unicode was to map all character
fcb875d4	331	repertories into a fixed-length integer so programmers are happy.
f2a2953c	332	Since each character is either I<short> or I<long> in C, you have to
	333	put endianness of each platform when you pass data to one another.
	334
	335	Anything marked as BE is Big Endian (or network byte order) and LE is
	336	Little Endian (aka VAX byte order). For anything without, a character
	337	called Byte Order Mark (BOM) is prepended to the head of string.
	338
	339	=over 4
	340
fcb875d4	341	=item BOM as integer when fetched in network byte order
f2a2953c	342
fcb875d4	343	16 32 bits/char
	344	-------------------------
	345	BE 0xFeFF 0x0000FeFF
	346	LE 0xFFeF 0xFFFe0000
	347	-------------------------
f2a2953c	348
f2a2953c	349	=back
fcb875d4	350
f2a2953c	351	This modules handles BOM as follows.
	352
	353	=over 4
	354
	355	=item *
	356
	357	When BE or LE is explicitly stated as the name of encoding, BOM is
	358	simply treated as one of characters (ZERO WIDTH NO-BREAK SPACE).
	359
	360	=item *
	361
	362	When BE or LE is omitted during decode(), it checks if BOM is in the
	363	beginning of the string and if found endianness is set to what BOM
fcb875d4	364	says. If not found, dies.
f2a2953c	365
	366	=item *
	367
	368	When BE or LE is omitted during encode(), it returns a BE-encoded
	369	string with BOM prepended. So when you want to encode a whole text
	370	file, make sure you encode() by whole text, not line by line or each
	371	line, not file, is prepended with BOMs.
	372
	373	=item *
	374
	375	C<UCS-2> is an exception. Unlike others this is an alias of UCS-2BE.
	376	UCS-2 is already registered by IANA and others that way.
	377
fdd579e2	378	=back
f2a2953c	379
fcb875d4	380	=head1 Surrogate Pairs
f2a2953c	381
fcb875d4	382	To say the least, surrogate pairs were the biggest mistake of the
	383	Unicode Consortium. But according to the late Douglas Adams in I<The
	384	Hitchhiker's Guide to the Galaxy> Trilogy, C<In the beginning the
	385	Universe was created. This has made a lot of people very angry and
	386	been widely regarded as a bad move>. Their mistake was not of this
	387	magnitude so let's forgive them.
f2a2953c	388
f2a2953c	389	(I don't dare make any comparison with Unicode Consortium and the
c731e18e	390	Vogons here ;) Or, comparing Encode to Babel Fish is completely
c731e18e	391	appropriate -- if you can only stick this into your ear :)
f2a2953c	392
fcb875d4	393	Surrogate pairs were born when Unicode Consortium finally
	394	admitted that 16 bits were not big enough to hold all the world's
	395	character repertoire. But they have already made UCS-2 16-bit. What
f2a2953c	396	do we do?
	397
	398	Back then 0xD800-0xDFFF was not allocated. Let's split them half and
	399	use the first half to represent C<upper half of a character> and the
	400	latter C<lower half of a character>. That way you can represent 1024
	401	* 1024 = 1048576 more characters. Now we can store character ranges
	402	up to \x{10ffff} even with 16-bit encodings. This pair of
	403	half-character is now called a I<Surrogate Pair> and UTF-16 is the
fcb875d4	404	name of the encoding that embraces them.
f2a2953c	405
	406	Here is a fomula to ensurrogate a Unicode character \x{10000} and
	407	above;
	408
	409	$hi = ($uni - 0x10000) / 0x400 + 0xD800;
	410	$lo = ($uni - 0x10000) % 0x400 + 0xDC00;
	411
	412	And to desurrogate;
	413
	414	$uni = 0x10000 + ($hi - 0xD800) * 0x400 + ($lo - 0xDC00);
	415
fcb875d4	416	Note this move has made \x{D800}-\x{DFFF} into a forbidden zone but
	417	perl does not prohibit the use of characters within this range. To perl,
	418	every one of \x{0000_0000} up to \x{ffff_ffff} (*) is I<a character>.
	419
	420	(*) or \x{ffff_ffff_ffff_ffff} if your perl is compiled with 64-bit
	421	integer support! (**)
	422
	423	(**) Is anything beyond \x{11_0000} still Unicode :?
f2a2953c	424
	425	=head1 SEE ALSO
	426
fdd579e2	427	L<Encode>, L<http://www.unicode.org/glossary/>,
f2a2953c	428
fdd579e2	429	RFC 2781 L<http://rfc.net/rfc2781.html>,
	430
	431	L<http://www.unicode.org/unicode/faq/utf_bom.html>
	432
fcb875d4	433	Ch. 15, pp. 403 of C<Programming Perl (3rd Edition)>
	434	by Larry Wall, Tom Christiansen, Jon Orwant;
	435	O'Reilly & Associates; ISBN 0-596-00027-8
	436
fdd579e2	437	=cut