From: Steve Peters Date: Wed, 12 Mar 2008 14:20:49 +0000 (+0000) Subject: Upgrade to Encode-2.24 X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=0dbed2e582e5e5a1aa8cf32cd879b06d966ccbc2;p=p5sagit%2Fp5-mst-13.2.git Upgrade to Encode-2.24 p4raw-id: //depot/perl@33493 --- diff --git a/ext/Encode/Changes b/ext/Encode/Changes index 704b96c..4ceb2ba 100644 --- a/ext/Encode/Changes +++ b/ext/Encode/Changes @@ -1,8 +1,28 @@ # Revision history for Perl extension Encode. # -# $Id: Changes,v 2.23 2007/05/29 18:15:32 dankogai Exp dankogai $ +# $Id: Changes,v 2.24 2008/03/12 09:51:11 dankogai Exp $ # -$Revision: 2.23 $ $Date: 2007/05/29 18:15:32 $ +$Revision: 2.24 $ $Date: 2008/03/12 09:51:11 $ +! lib/Encode/Config.pm + adds and fixes also adds cp858 support. +! Encode.pm encoding.pm lib/Encode/Alias.pm ucm/cp858.ucm + Merged perl@33486. + > Change 33486 by rgs@scipion on 2008/03/12 08:50:11 + An unfortunate side-effect of Encode and Encode::Alias use'ing each + other, and Encode::Alias exporting functions into Encode for it to use + as methods, broke the loading of the find_alias() Encode method in some + cases since 5.10. Breaking the recursive inheritance fixes it. + Message-Id: +! Encode.pm + POD fix by tels + Message-Id: <200711281835.36125@bloodgate.com> +! bin/ucmlint + Fix by MIYAGAWA via CodeRepos + http://coderepos.org/share/changeset/1791 +! encoding.pm t/mime_header_iso2022jp.t + ported back from Perl 5.10-RC1 + +2.23 2007/05/29 18:15:32 ! Encode.xs got rid of global fallback_cb; encode_method() now takes one more argument which is a coderef to fallback. This should make diff --git a/ext/Encode/Encode.pm b/ext/Encode/Encode.pm index 97a5a6f..c76fcdb 100644 --- a/ext/Encode/Encode.pm +++ b/ext/Encode/Encode.pm @@ -1,10 +1,10 @@ # -# $Id: Encode.pm,v 2.23 2007/05/29 18:15:32 dankogai Exp dankogai $ +# $Id: Encode.pm,v 2.24 2008/03/12 09:58:12 dankogai Exp dankogai $ # package Encode; use strict; use warnings; -our $VERSION = sprintf "%d.%02d", q$Revision: 2.23 $ =~ /(\d+)/g; +our $VERSION = sprintf "%d.%02d", q$Revision: 2.24 $ =~ /(\d+)/g; sub DEBUG () { 0 } use XSLoader (); XSLoader::load( __PACKAGE__, $VERSION ); @@ -734,7 +734,7 @@ you're not interested in this, then bitwise-or the bitmask with it. =back -=Head2 coderef for CHECK +=head2 coderef for CHECK As of Encode 2.12 CHECK can also be a code reference which takes the ord value of unmapped caharacter as an argument and returns a string diff --git a/ext/Encode/Encode.xs b/ext/Encode/Encode.xs index 5acdc75..1424071 100644 --- a/ext/Encode/Encode.xs +++ b/ext/Encode/Encode.xs @@ -1,5 +1,5 @@ /* - $Id: Encode.xs,v 2.14 2007/05/29 18:15:32 dankogai Exp dankogai $ + $Id: Encode.xs,v 2.14 2007/05/29 18:15:32 dankogai Exp $ */ #define PERL_NO_GET_CONTEXT diff --git a/ext/Encode/Makefile.PL b/ext/Encode/Makefile.PL index d02bc41..6b7d582 100644 --- a/ext/Encode/Makefile.PL +++ b/ext/Encode/Makefile.PL @@ -1,5 +1,5 @@ # -# $Id: Makefile.PL,v 2.5 2007/05/29 18:15:32 dankogai Exp dankogai $ +# $Id: Makefile.PL,v 2.5 2007/05/29 18:15:32 dankogai Exp $ # use 5.007003; use strict; diff --git a/ext/Encode/bin/ucmlint b/ext/Encode/bin/ucmlint index c5d755b..622376d 100644 --- a/ext/Encode/bin/ucmlint +++ b/ext/Encode/bin/ucmlint @@ -1,10 +1,10 @@ #!/usr/local/bin/perl # -# $Id: ucmlint,v 2.1 2006/05/03 18:24:10 dankogai Exp $ +# $Id: ucmlint,v 2.2 2008/03/12 09:51:11 dankogai Exp $ # use strict; -our $VERSION = do { my @r = (q$Revision: 2.1 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; +our $VERSION = do { my @r = (q$Revision: 2.2 $ =~ /\d+/g); sprintf "%d."."%02d" x $#r, @r }; use Getopt::Std; our %Opt; @@ -30,7 +30,7 @@ $0 -[Dehfv] [ucm files ...] } $| = 1; -my (%Hdr, %U2E, %E2U); +my (%Hdr, %U2E, %E2U, %Fallback); my $in_charmap = 0; my $nerror = 0; my $nwarning = 0; @@ -39,92 +39,94 @@ sub nit($;$){ my ($msg, $level) = @_; my $lstr; if ($level == 2){ - $lstr = 'notice'; + $lstr = 'notice'; }elsif ($level == 1){ - $lstr = 'warning'; $nwarning++; + $lstr = 'warning'; $nwarning++; }else{ - $lstr = 'error'; $nerror++; + $lstr = 'error'; $nerror++; } print "$ARGV:$lstr in line $.: $msg\n"; } for $ARGV (@ARGV){ open UCM, $ARGV or die "$ARGV:$!"; - %Hdr = %U2E = %E2U = (); + %Hdr = %U2E = %E2U = %Fallback = (); $in_charmap = $nerror = $nwarning = 0; $. = 0; while(){ - chomp; - s/\s*#.*$//o; /^$/ and next; - if ($_ eq "CHARMAP"){ - $in_charmap = 1; - for my $must (qw/code_set_name mb_cur_min mb_cur_max/){ - exists $Hdr{$must} or nit "<$must> nonexistent"; - } - $Hdr{mb_cur_min} > $Hdr{mb_cur_max} - and nit sprintf("mb_cur_min(%d) > mb_cur_max(%d)", - $Hdr{mb_cur_min},$Hdr{mb_cur_max}); - $in_charmap = 1; - next; - } - unless ($in_charmap){ - my($hkey, $hvalue) = /^<(\S+)>\s+[\"\']?([^\"\']+)/o or next; - $Opt{D} and warn "$hkey => $hvalue"; - if ($hkey eq "code_set_name"){ # name check - exists $Hdr{code_set_name} - and nit "Duplicate : $hkey"; - } - if ($hkey eq "code_set_alias"){ # alias check - $hvalue eq $Hdr{code_set_name} - and nit qq(alias "$hvalue" is already in ); - } - $Hdr{$hkey} = $hvalue; - }else{ - my $name = $Hdr{code_set_name}; - my($unistr, $encstr, $fb) = /^(\S+)\s+(\S+)\s(\S+)/o or next; - $Opt{v} and nit $_, 2; - my $uni = uniparse($unistr); - my $enc = encparse($encstr); - $fb =~ /^\|([0123])$/ or nit "malformed fallback: $fb"; - $fb = $1; - $Opt{f} and $fb = 0; - unless ($fb == 1){ # check uni -> enc - if (exists $U2E{$uni}){ - nit "dupe encode map: U$uni => $U2E{$uni} and $enc", 1; - }else{ - $U2E{$uni} = $enc; - if ($Opt{e} and $fb != 3) { - my $e = hex2enc($enc); - my $u = hex2uni($uni); - my $eu = Encode::encode($name, $u); - $e eq $eu - or nit qq(encode('$name', $uni) != $enc); + chomp; + s/\s*#.*$//o; /^$/ and next; + if ($_ eq "CHARMAP"){ + $in_charmap = 1; + for my $must (qw/code_set_name mb_cur_min mb_cur_max/){ + exists $Hdr{$must} or nit "<$must> nonexistent"; } + $Hdr{mb_cur_min} > $Hdr{mb_cur_max} + and nit sprintf("mb_cur_min(%d) > mb_cur_max(%d)", + $Hdr{mb_cur_min},$Hdr{mb_cur_max}); + $in_charmap = 1; + next; } - } - unless ($fb == 3){ # check enc -> uni - if (exists $E2U{$enc}){ - nit "dupe decode map: $enc => U$E2U{$enc} and U$uni", 1; + unless ($in_charmap){ + my($hkey, $hvalue) = /^<(\S+)>\s+[\"\']?([^\"\']+)/o or next; + $Opt{D} and warn "$hkey => $hvalue"; + if ($hkey eq "code_set_name"){ # name check + exists $Hdr{code_set_name} + and nit "Duplicate : $hkey"; + } + if ($hkey eq "code_set_alias"){ # alias check + $hvalue eq $Hdr{code_set_name} + and nit qq(alias "$hvalue" is already in ); + } + $Hdr{$hkey} = $hvalue; }else{ - $E2U{$enc} = $uni; - if ($Opt{e} and $fb != 1) { - my $e = hex2enc($enc); - my $u = hex2uni($uni); - $Opt{D} and warn "$uni, $enc"; - my $de = Encode::decode($name, $e); - $de eq $u - or nit qq(decode('$name', $enc) != $uni); + my $name = $Hdr{code_set_name}; + my($unistr, $encstr, $fb) = /^(\S+)\s+(\S+)\s(\S+)/o or next; + $Opt{v} and nit $_, 2; + my $uni = uniparse($unistr); + my $enc = encparse($encstr); + $fb =~ /^\|([0123])$/ or nit "malformed fallback: $fb"; + $fb = $1; + $Opt{f} and $fb = 0; + unless ($fb == 3){ # check uni -> enc + if (exists $U2E{$uni}){ + nit "dupe encode map: U$uni => $U2E{$uni} and $enc", 1; + }else{ + $U2E{$uni} = $enc; + $Fallback{$uni}{$enc} = 1 if $fb == 1; + if ($Opt{e}) { + my $e = hex2enc($enc); + my $u = hex2uni($uni); + my $eu = Encode::encode($name, $u); + $e eq $eu + or nit qq(encode('$name', $uni) != $enc); + } + } } + unless ($fb == 1){ # check enc -> uni + if (exists $E2U{$enc}){ + nit "dupe decode map: $enc => U$E2U{$enc} and U$uni", 1; + }else{ + $E2U{$enc} = $uni; + $Fallback{$enc}{$uni} = 1 if $fb == 3; + if ($Opt{e}) { + my $e = hex2enc($enc); + my $u = hex2uni($uni); + $Opt{D} and warn "$uni, $enc"; + my $de = Encode::decode($name, $e); + $de eq $u + or nit qq(decode('$name', $enc) != $uni); + } + } + } + # warn "$uni, $enc, $fb"; } - } - # warn "$uni, $enc, $fb"; - } } $in_charmap or nit "Where is CHARMAP?"; checkRT(); printf ("$ARGV: %s error%s found\n", - ($nerror == 0 ? 'no' : $nerror), - ($nerror > 1 ? 's' : '')); + ($nerror == 0 ? 'no' : $nerror), + ($nerror > 1 ? 's' : '')); } exit; @@ -138,14 +140,14 @@ sub hex2uni{ sub checkRT{ for my $uni (keys %E2U){ - my $enc = $U2E{$uni} or next; # okay - $E2U{$U2E{$uni}} eq $uni or - nit "RT failure: U$uni => $enc =>U$E2U{$U2E{$uni}}"; + my $enc = $U2E{$uni} or next; # okay + $E2U{$U2E{$uni}} eq $uni or $Fallback{$uni}{$enc} or + nit "RT failure: U$uni => $enc =>U$E2U{$U2E{$uni}}"; } for my $enc (keys %E2U){ - my $uni = $E2U{$enc} or next; # okay - $U2E{$E2U{$enc}} eq $enc or - nit "RT failure: $enc => U$uni => $U2E{$E2U{$enc}}"; + my $uni = $E2U{$enc} or next; # okay + $U2E{$E2U{$enc}} eq $enc or $Fallback{$enc}{$uni} or + nit "RT failure: $enc => U$uni => $U2E{$E2U{$enc}}"; } } @@ -155,8 +157,8 @@ sub uniparse{ my @u; push @u, $1 while($str =~ /\G/ig); for my $u (@u){ - $u =~ /^([0-9A-Za-z]+)$/o - or nit "malformed Unicode character: $u"; + $u =~ /^([0-9A-Za-z]+)$/o + or nit "malformed Unicode character: $u"; } return join(',', @u); } @@ -165,10 +167,10 @@ sub encparse{ my $str = shift; my @e; for my $e (split /\\x/io, $str){ - $e or next; # first \x - $e =~ /^([0-9A-Za-z]{1,2})$/io - or nit "Hex $e in $str is bogus"; - push @e, $1; + $e or next; # first \x + $e =~ /^([0-9A-Za-z]{1,2})$/io + or nit "Hex $e in $str is bogus"; + push @e, $1; } return join(',', @e); } diff --git a/ext/Encode/lib/Encode/Alias.pm b/ext/Encode/lib/Encode/Alias.pm index 24e7d82..403293b 100644 --- a/ext/Encode/lib/Encode/Alias.pm +++ b/ext/Encode/lib/Encode/Alias.pm @@ -2,7 +2,7 @@ package Encode::Alias; use strict; use warnings; no warnings 'redefine'; -our $VERSION = '2.07_01'; #do { my @r = ( q$Revision: 2.7 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 2.8 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; sub DEBUG () { 0 } use base qw(Exporter); diff --git a/ext/Encode/lib/Encode/Config.pm b/ext/Encode/lib/Encode/Config.pm index c9f431b..1286a47 100644 --- a/ext/Encode/lib/Encode/Config.pm +++ b/ext/Encode/lib/Encode/Config.pm @@ -2,7 +2,7 @@ # Demand-load module list # package Encode::Config; -our $VERSION = do { my @r = ( q$Revision: 2.4 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; +our $VERSION = do { my @r = ( q$Revision: 2.5 $ =~ /\d+/g ); sprintf "%d." . "%02d" x $#r, @r }; use strict; use warnings; @@ -38,6 +38,7 @@ our %ExtModule = ( 'cp855' => 'Encode::Byte', 'cp856' => 'Encode::Byte', 'cp857' => 'Encode::Byte', + 'cp858' => 'Encode::Byte', 'cp860' => 'Encode::Byte', 'cp861' => 'Encode::Byte', 'cp862' => 'Encode::Byte', diff --git a/ext/Encode/ucm/macJapanese.ucm b/ext/Encode/ucm/macJapanese.ucm index 892d3f7..2ba9410 100644 --- a/ext/Encode/ucm/macJapanese.ucm +++ b/ext/Encode/ucm/macJapanese.ucm @@ -1,5 +1,5 @@ # -# $Id: macJapanese.ucm,v 2.0 2004/05/16 20:55:27 dankogai Exp $ +# $Id: macJapanese.ucm,v 2.1 2008/03/12 09:51:11 dankogai Exp $ # # Original table can be obtained at # http://www.unicode.org/Public/MAPPINGS/VENDORS/APPLE/JAPANESE.TXT @@ -135,7 +135,7 @@ CHARMAP \x7B |0 # LEFT CURLY BRACKET \x7C |0 # VERTICAL LINE \x7D |0 # RIGHT CURLY BRACKET - \x7E |0 # TILDE # Apple change from standard Shift-JIS + \x7E |0 # TILDE # Apple change from standard Shift_JIS \xA0 |0 # NO-BREAK SPACE # Apple addition \x81\x91 |0 # CENT SIGN \x81\x92 |0 # POUND SIGN