lib/Unicode/Collate/t/contract.t Unicode::Collate
lib/Unicode/Collate/t/hangtype.t Unicode::Collate
lib/Unicode/Collate/t/hangul.t Unicode::Collate
+lib/Unicode/Collate/t/illegal.t Unicode::Collate
+lib/Unicode/Collate/t/illegalp.t Unicode::Collate
lib/Unicode/Collate/t/index.t Unicode::Collate
lib/Unicode/Collate/t/normal.t Unicode::Collate
lib/Unicode/Collate/t/test.t Unicode::Collate
use Carp;
use File::Spec;
+no warnings 'utf8';
+
require Exporter;
-our $VERSION = '0.30';
+our $VERSION = '0.31';
our $PACKAGE = __PACKAGE__;
our @ISA = qw(Exporter);
or croak "Illegal UCA version (passed $self->{UCA_Version}).";
$self->{variable} ||= $self->{alternate} || $self->{variableTable} ||
- $self->{alternateTable} || $self->{alternate} || 'shifted';
+ $self->{alternateTable} || 'shifted';
$self->{variable} = $self->{alternate} = lc($self->{variable});
exists $VariableOK{ $self->{variable} }
or croak "$PACKAGE unknown variable tag name: $self->{variable}";
}
for (my $i = 0; $i < @src; $i++) {
- next if _isNonCharacter($src[$i]);
+ next if _isIllegal($src[$i]);
my $i_orig = $i;
my $jcps = $src[$i];
);
}
-sub _isNonCharacter {
+sub _isIllegal {
my $code = shift;
return ! defined $code # removed
|| ($code < 0 || 0x10FFFF < $code) # out of range
You can use another collation element table if desired.
The table file must be put into a directory
-where F<Unicode/Collate.pm> is installed.
-E.g. in F<perl/lib/Unicode/Collate> directory
-when you have F<perl/lib/Unicode/Collate.pm>.
+where F<Unicode/Collate.pm> is installed; e.g. into
+F<perl/lib/Unicode/Collate/> if you have F<perl/lib/Unicode/Collate.pm>.
-By default, the filename F<"allkeys.txt"> is used.
+By default, the filename F<allkeys.txt> is used.
If C<undef> is passed explicitly as the value for this key,
no file is read (but you can define collation elements via C<entry>).
=head2 Conformance Test
-The Conformance Test for the UCA is provided
-in L<http://www.unicode.org/reports/tr10/CollationTest.html>
-and L<http://www.unicode.org/reports/tr10/CollationTest.zip>
+The Conformance Test for the UCA is available
+under L<http://www.unicode.org/Public/UCA/>.
For F<CollationTest_SHIFTED.txt>,
a collator via C<Unicode::Collate-E<gt>new( )> should be used;
=head1 AUTHOR
-SADAHIRO Tomoyuki, <SADAHIRO@cpan.org>
+SADAHIRO Tomoyuki <SADAHIRO@cpan.org>
http://homepage1.nifty.com/nomenclator/perl/
=item The Default Unicode Collation Element Table (DUCET)
-L<http://www.unicode.org/reports/tr10/allkeys.txt>
+L<http://www.unicode.org/Public/UCA/latest/allkeys.txt>
=item The conformance test for the UCA
-L<http://www.unicode.org/reports/tr10/CollationTest.html>
+L<http://www.unicode.org/Public/UCA/latest/CollationTest.html>
-L<http://www.unicode.org/reports/tr10/CollationTest.zip>
+L<http://www.unicode.org/Public/UCA/latest/CollationTest.zip>
=item Hangul Syllable Type
-http://www.unicode.org/Public/UNIDATA/HangulSyllableType.txt
+L<http://www.unicode.org/Public/UNIDATA/HangulSyllableType.txt>
=item Unicode Normalization Forms - UAX #15
Revision history for Perl module Unicode::Collate.
+0.31 Sun Nov 16 15:40:15 2003
+ - Illegal code points (surrogate and noncharacter; they are definitely
+ ignorable) will be distinguished from NULL ("\0");
+ but porting is not successful in the case of ((Pure Perl) and
+ (Perl 5.7.3 or before)). If perl 5.6.X is used, XSUB may help it
+ in place of broken CORE::unpack('U*') in older perl.
+ - added illegal.t and illegalp.t.
+ - added XSUB edition (EXPERIMENTAL) where some functions are implemented
+ in XSUB (Pure Perl edition is also supported.)
+
0.30 Mon Oct 13 21:26:37 2003
- fix: Completely ignorable in table should be able to be overrided
by non-ignorable in entry.
- fix: Maximum length for contraction must not be shortened
- by a shorter contraction following.
+ by a shorter contraction following in table and/or entry.
- added normal.t.
- some doc fixes
0.29 Mon Oct 13 12:18:23 2003
+ - now UCA Version 11.
- supported hangul_terminator.
- fix: Base_Unicode_Version falsely returns Perl's Unicode version.
C4 in UTS #10 requires UTS's Unicode version.
-Unicode/Collate version 0.30
+Unicode/Collate version 0.31
===============================
NAME
INSTALLATION
Perl 5.6.1 or later
+(recommended: Perl 5.8.0 or later)
+
+To use this module, it is recommended to install a table file
+in the UCA format, by copying it into the directory
+where F<Unicode/Collate.pm> is installed;
+e.g. into F<perl/lib/Unicode/Collate/> directory
+if you have F<perl/lib/Unicode/Collate.pm>.
+
+The most preferable one is "The Default Unicode Collation Element Table",
+available from the Unicode consortium's website:
+
+ http://www.unicode.org/Public/UCA/latest/allkeys.txt (latest version)
+
+Though this distribution contains a subset of allkeys.txt, named "keys.txt",
+this one is intended only for doing a test of this module
+and practically useless for any other purpose.
To install this module type the following:
make test
make install
-To use this module, it is better to install a table file in the UCA format,
-by copying it into the lib/Unicode/Collate directory.
+If you have a C compiler and want to use XSUB edition,
+type the following (!! "enableXS" must run before "Makefile.PL" !!):
-The most preferable one is "The Default Unicode Collation Element Table",
-available from the Unicode consortium's website:
+ perl enableXS
+ perl Makefile.PL
+ make
+ make test
+ make install
- http://www.unicode.org/reports/tr10/allkeys.txt
+If you decide to install pure Perl (i.e. non-XS) edition after trying
+to build XSUB, type the following:
-Though this distribution contains a subset of allkeys.txt, named "keys.txt",
-this one is intended only for doing a test of this module
-and practically useless for any other purpose.
+ make clean
+ perl disableXS
+ perl Makefile.PL
+ make
+ make test
+ make install
DEPENDENCIES
--- /dev/null
+
+BEGIN {
+ unless ("A" eq pack('U', 0x41)) {
+ print "1..0 # Unicode::Collate " .
+ "cannot stringify a Unicode code point\n";
+ exit 0;
+ }
+}
+
+BEGIN {
+ if ($ENV{PERL_CORE}) {
+ chdir('t') if -d 't';
+ @INC = $^O eq 'MacOS' ? qw(::lib) : qw(../lib);
+ }
+}
+
+use Test;
+use strict;
+use warnings;
+
+BEGIN {
+ use Unicode::Collate;
+
+ unless (exists &Unicode::Collate::bootstrap or 5.008 <= $]) {
+ print "1..0 # skipped: XSUB, or Perl 5.8.0 or later".
+ " needed for this test\n";
+ print $@;
+ exit;
+ }
+}
+
+BEGIN { plan tests => 22 };
+
+ok(1);
+
+#########################
+
+no warnings 'utf8';
+
+# NULL is tailorable but illegal code points are not.
+# illegal code points should be always ingored
+# (cf. UCA, 7.1.1 Illegal code points).
+
+my $illeg = Unicode::Collate->new(
+ entry => <<'ENTRIES',
+0000 ; [.0020.0000.0000.0000] # [0000] NULL
+0001 ; [.0021.0000.0000.0001] # [0001] START OF HEADING
+FFFE ; [.0022.0000.0000.FFFE] # <noncharacter-FFFE>
+FFFF ; [.0023.0000.0000.FFFF] # <noncharacter-FFFF>
+D800 ; [.0024.0000.0000.D800] # <surrogate-D800>
+DFFF ; [.0025.0000.0000.DFFF] # <surrogate-DFFF>
+FDD0 ; [.0026.0000.0000.FDD0] # <noncharacter-FDD0>
+FDEF ; [.0027.0000.0000.FDEF] # <noncharacter-FDEF>
+0002 ; [.0030.0000.0000.0002] # [0002] START OF TEXT
+10FFFF; [.0040.0000.0000.10FFFF] # <noncharacter-10FFFF>
+110000; [.0041.0000.0000.110000] # <out-of-range 110000>
+ENTRIES
+ level => 1,
+ table => undef,
+ normalization => undef,
+);
+
+ok($illeg->lt("", "\x00"));
+ok($illeg->lt("", "\x01"));
+ok($illeg->eq("", "\x{FFFE}"));
+ok($illeg->eq("", "\x{FFFF}"));
+ok($illeg->eq("", "\x{D800}"));
+ok($illeg->eq("", "\x{DFFF}"));
+ok($illeg->eq("", "\x{FDD0}"));
+ok($illeg->eq("", "\x{FDEF}"));
+ok($illeg->lt("", "\x02"));
+ok($illeg->eq("", "\x{10FFFF}"));
+ok($illeg->eq("", "\x{110000}"));
+
+ok($illeg->lt("\x00", "\x01"));
+ok($illeg->lt("\x01", "\x02"));
+ok($illeg->ne("\0", "\x{D800}"));
+ok($illeg->ne("\0", "\x{DFFF}"));
+ok($illeg->ne("\0", "\x{FDD0}"));
+ok($illeg->ne("\0", "\x{FDEF}"));
+ok($illeg->ne("\0", "\x{FFFE}"));
+ok($illeg->ne("\0", "\x{FFFF}"));
+ok($illeg->ne("\0", "\x{10FFFF}"));
+ok($illeg->ne("\0", "\x{110000}"));
+
--- /dev/null
+
+BEGIN {
+ unless ("A" eq pack('U', 0x41)) {
+ print "1..0 # Unicode::Collate " .
+ "cannot stringify a Unicode code point\n";
+ exit 0;
+ }
+}
+
+BEGIN {
+ if ($ENV{PERL_CORE}) {
+ chdir('t') if -d 't';
+ @INC = $^O eq 'MacOS' ? qw(::lib) : qw(../lib);
+ }
+}
+
+use Test;
+BEGIN { plan tests => 17 };
+
+use strict;
+use warnings;
+
+ok(1);
+
+#
+# No test for Unicode::Collate is included in this .t file.
+#
+# UCA conformance test requires completely ignorable characters
+# (including noncharacters) must be able to be ordered in code point order;
+# If not so, Unicode::Collate must not be compliant with UCA.
+#
+# ~~~ CollationTest_SHIFTED.txt in CollationTest-4.0.0
+#
+# 206F 0021; # ! NOMINAL DIGIT SHAPES [| | | 0251]
+# D800 0021; # ! <surrogate-D800> [| | | 0251]
+# DFFF 0021; # ! <surrogate-DFFF> [| | | 0251]
+# FDD0 0021; # ! <noncharacter-FDD0> [| | | 0251]
+# FFFB 0021; # ! INTERLINEAR ANNOTATION TERMINATOR [| | | 0251]
+# FFFE 0021; # ! <noncharacter-FFFE> [| | | 0251]
+# FFFF 0021; # ! <noncharacter-FFFF> [| | | 0251]
+# 1D165 0021; # ! MS. Cm. STEM [| | | 0251]
+#
+# ~~~ CollationTest_NON_IGNORABLE.txt in CollationTest-4.0.0
+#
+# 206F 0021; # ! NOMINAL DIGIT SHAPES [0251 | 0020 | 0002 |]
+# D800 0021; # ! <surrogate-D800> [0251 | 0020 | 0002 |]
+# DFFF 0021; # ! <surrogate-DFFF> [0251 | 0020 | 0002 |]
+# FDD0 0021; # ! <noncharacter-FDD0> [0251 | 0020 | 0002 |]
+# FFFB 0021; # ! INTERLINEAR ANNOTATION TERMINATOR [0251 | 0020 | 0002 |]
+# FFFE 0021; # ! <noncharacter-FFFE> [0251 | 0020 | 0002 |]
+# FFFF 0021; # ! <noncharacter-FFFF> [0251 | 0020 | 0002 |]
+# 1D165 0021; # ! MS. Cm. STEM [0251 | 0020 | 0002 |]
+#
+
+no warnings 'utf8';
+
+ok("\x{206F}!" lt "\x{D800}!");
+ok(pack('U*', 0x206F, 0x21) lt pack('U*', 0xD800, 0x21));
+
+ok("\x{D800}!" lt "\x{DFFF}!");
+ok(pack('U*', 0xD800, 0x21) lt pack('U*', 0xDFFF, 0x21));
+
+ok("\x{DFFF}!" lt "\x{FDD0}!");
+ok(pack('U*', 0xDFFF, 0x21) lt pack('U*', 0xFDD0, 0x21) );
+
+ok("\x{FDD0}!" lt "\x{FFFB}!");
+ok(pack('U*', 0xFDD0, 0x21) lt pack('U*', 0xFFFB, 0x21));
+
+ok("\x{FFFB}!" lt "\x{FFFE}!");
+ok(pack('U*', 0xFFFB, 0x21) lt pack('U*', 0xFFFE, 0x21));
+
+ok("\x{FFFE}!" lt "\x{FFFF}!");
+ok(pack('U*', 0xFFFE, 0x21) lt pack('U*', 0xFFFF, 0x21));
+
+ok("\x{FFFF}!" lt "\x{1D165}!");
+ok(pack('U*', 0xFFFF, 0x21) lt pack('U*', 0x1D165, 0x21));
+
+ok("\000!" lt "\x{FFFF}!");
+ok(pack('U*', 0, 0x21) lt pack('U*', 0xFFFF, 0x21));
+
projects / p5sagit/p5-mst-13.2.git / commitdiff