4 print "1..0 # Unicode::Collate not ported to EBCDIC\n";
10 if ($ENV{PERL_CORE}) {
17 BEGIN { plan tests => 184 };
20 #########################
22 ok(1); # If we made it this far, we're ok.
24 my $UCA_Version = "9";
26 ok(Unicode::Collate::UCA_Version, $UCA_Version);
27 ok(Unicode::Collate->UCA_Version, $UCA_Version);
29 my $Collator = Unicode::Collate->new(
31 normalization => undef,
34 ok(ref $Collator, "Unicode::Collate");
36 ok($Collator->UCA_Version, $UCA_Version);
37 ok($Collator->UCA_Version(), $UCA_Version);
40 join(':', $Collator->sort(
41 qw/ lib strict Carp ExtUtils CGI Time warnings Math overload Pod CPAN /
44 qw/ Carp CGI CPAN ExtUtils lib Math overload Pod strict Time warnings /
48 ok($Collator->cmp("", ""), 0);
49 ok($Collator->eq("", ""));
50 ok($Collator->cmp("", "perl"), -1);
54 my $A_acute = pack('U', 0x00C1);
55 my $a_acute = pack('U', 0x00E1);
56 my $acute = pack('U', 0x0301);
58 ok($Collator->cmp("A$acute", $A_acute), 0); # @version 3.1.1 (prev: -1)
59 ok($Collator->cmp($a_acute, $A_acute), -1);
60 ok($Collator->eq("A\cA$acute", $A_acute)); # UCA v9
62 my %old_level = $Collator->change(level => 1);
63 ok($Collator->eq("A$acute", $A_acute));
64 ok($Collator->eq("A", $A_acute));
66 ok($Collator->change(level => 2)->eq($a_acute, $A_acute));
67 ok($Collator->lt("A", $A_acute));
69 ok($Collator->change(%old_level)->lt("A", $A_acute));
70 ok($Collator->lt("A", $A_acute));
71 ok($Collator->lt("A", $a_acute));
72 ok($Collator->lt($a_acute, $A_acute));
76 eval { require Unicode::Normalize };
79 my $NFD = Unicode::Collate->new(
82 0430 ; [.0CB5.0020.0002.0430] # CYRILLIC SMALL LETTER A
83 0410 ; [.0CB5.0020.0008.0410] # CYRILLIC CAPITAL LETTER A
84 04D3 ; [.0CBD.0020.0002.04D3] # CYRILLIC SMALL LETTER A WITH DIAERESIS
85 0430 0308 ; [.0CBD.0020.0002.04D3] # CYRILLIC SMALL LETTER A WITH DIAERESIS
86 04D2 ; [.0CBD.0020.0008.04D2] # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
87 0410 0308 ; [.0CBD.0020.0008.04D2] # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
88 0430 3099 ; [.0CBE.0020.0002.04D3] # A WITH KATAKANA VOICED
89 0430 3099 0308 ; [.0CBF.0020.0002.04D3] # A WITH KATAKANA VOICED, DIAERESIS
92 ok($NFD->eq("\x{4D3}\x{325}", "\x{430}\x{308}\x{325}"));
93 ok($NFD->lt("\x{430}\x{308}A", "\x{430}\x{308}B"));
94 ok($NFD->lt("\x{430}\x{3099}B", "\x{430}\x{308}\x{3099}A"));
95 ok($NFD->eq("\x{0430}\x{3099}\x{309A}\x{0308}",
96 "\x{0430}\x{309A}\x{3099}\x{0308}") );
107 my $trad = Unicode::Collate->new(
109 normalization => undef,
110 ignoreName => qr/HANGUL|HIRAGANA|KATAKANA|BOPOMOFO/,
112 entry => << 'ENTRIES',
113 0063 0068 ; [.0A3F.0020.0002.0063] % "ch" in traditional Spanish
114 0043 0068 ; [.0A3F.0020.0008.0043] # "Ch" in traditional Spanish
115 00DF ; [.0BA7.0020.0004.00DF][.0000.0153.0004.00DF][.0BA7.0020.001F.00DF] # sz
118 # 0063 ; [.0A3D.0020.0002.0063] # LATIN SMALL LETTER C
119 # 0064 ; [.0A49.0020.0002.0064] # LATIN SMALL LETTER D
120 # 0073 ; [.0BA7.0020.0002.0073] # LATIN SMALL LETTER S
123 join(':', $trad->sort( qw/ acha aca ada acia acka / ) ),
124 join(':', qw/ aca acia acka acha ada / ),
128 join(':', $Collator->sort( qw/ acha aca ada acia acka / ) ),
129 join(':', qw/ aca acha acia acka ada / ),
131 ok($trad->eq("ocho", "oc\cAho")); # UCA v9
132 ok($trad->eq("ocho", "oc\000\cA\000\x7Fho")); # UCA v9
134 my $hiragana = "\x{3042}\x{3044}";
135 my $katakana = "\x{30A2}\x{30A4}";
137 # HIRAGANA and KATAKANA are ignorable via ignoreName
138 ok($trad->eq($hiragana, ""));
139 ok($trad->eq("", $katakana));
140 ok($trad->eq($hiragana, $katakana));
141 ok($trad->eq($katakana, $hiragana));
145 $Collator->change(level => 2);
147 ok($Collator->{level}, 2);
149 ok( $Collator->cmp("ABC","abc"), 0);
150 ok( $Collator->eq("ABC","abc") );
151 ok( $Collator->le("ABC","abc") );
152 ok( $Collator->cmp($hiragana, $katakana), 0);
153 ok( $Collator->eq($hiragana, $katakana) );
154 ok( $Collator->ge($hiragana, $katakana) );
157 ok( $Collator->eq("a\x{AC00}b", "a\x{1100}\x{1161}b") );
158 ok( $Collator->eq("a\x{AE00}b", "a\x{1100}\x{1173}\x{11AF}b") );
159 ok( $Collator->gt("a\x{AE00}b", "a\x{1100}\x{1173}b\x{11AF}") );
160 ok( $Collator->lt("a\x{AC00}b", "a\x{AE00}b") );
161 ok( $Collator->gt("a\x{D7A3}b", "a\x{C544}b") );
162 ok( $Collator->lt("a\x{C544}b", "a\x{30A2}b") ); # hangul < hiragana
164 $Collator->change(%old_level, katakana_before_hiragana => 1);
166 ok($Collator->{level}, 4);
168 ok( $Collator->cmp("abc", "ABC"), -1);
169 ok( $Collator->ne("abc", "ABC") );
170 ok( $Collator->lt("abc", "ABC") );
171 ok( $Collator->le("abc", "ABC") );
172 ok( $Collator->cmp($hiragana, $katakana), 1);
173 ok( $Collator->ne($hiragana, $katakana) );
174 ok( $Collator->gt($hiragana, $katakana) );
175 ok( $Collator->ge($hiragana, $katakana) );
177 $Collator->change(upper_before_lower => 1);
179 ok( $Collator->cmp("abc", "ABC"), 1);
180 ok( $Collator->ge("abc", "ABC"), 1);
181 ok( $Collator->gt("abc", "ABC"), 1);
182 ok( $Collator->cmp($hiragana, $katakana), 1);
183 ok( $Collator->ge($hiragana, $katakana), 1);
184 ok( $Collator->gt($hiragana, $katakana), 1);
186 $Collator->change(katakana_before_hiragana => 0);
188 ok( $Collator->cmp("abc", "ABC"), 1);
189 ok( $Collator->cmp($hiragana, $katakana), -1);
191 $Collator->change(upper_before_lower => 0);
193 ok( $Collator->cmp("abc", "ABC"), -1);
194 ok( $Collator->le("abc", "ABC") );
195 ok( $Collator->cmp($hiragana, $katakana), -1);
196 ok( $Collator->lt($hiragana, $katakana) );
200 my $ignoreAE = Unicode::Collate->new(
202 normalization => undef,
203 ignoreChar => qr/^[aAeE]$/,
206 ok($ignoreAE->eq("element","lament"));
207 ok($ignoreAE->eq("Perl","ePrl"));
211 my $onlyABC = Unicode::Collate->new(
213 normalization => undef,
214 entry => << 'ENTRIES',
215 0061 ; [.0101.0020.0002.0061] # LATIN SMALL LETTER A
216 0041 ; [.0101.0020.0008.0041] # LATIN CAPITAL LETTER A
217 0062 ; [.0102.0020.0002.0062] # LATIN SMALL LETTER B
218 0042 ; [.0102.0020.0008.0042] # LATIN CAPITAL LETTER B
219 0063 ; [.0103.0020.0002.0063] # LATIN SMALL LETTER C
220 0043 ; [.0103.0020.0008.0043] # LATIN CAPITAL LETTER C
225 join(':', $onlyABC->sort( qw/ ABA BAC cc A Ab cAc aB / ) ),
226 join(':', qw/ A aB Ab ABA BAC cAc cc / ),
231 my $undefAE = Unicode::Collate->new(
233 normalization => undef,
234 undefChar => qr/^[aAeE]$/,
237 ok($undefAE ->gt("edge","fog"));
238 ok($Collator->lt("edge","fog"));
239 ok($undefAE ->gt("lake","like"));
240 ok($Collator->lt("lake","like"));
244 $Collator->change(level => 2);
248 my $orig = "This is a Perl book.";
251 my $ret = "This is a camel book.";
254 if (my($pos,$len) = $Collator->index($str, $sub)) {
255 substr($str, $pos, $len, $rep);
260 $Collator->change(%old_level);
263 if (my($pos,$len) = $Collator->index($str, $sub)) {
264 substr($str, $pos, $len, $rep);
273 $Collator->change(level => 1);
275 $str = "Pe\x{300}rl";
278 if (my($pos, $len) = $Collator->index($str, $sub)) {
279 $match = substr($str, $pos, $len);
281 ok($match, "Pe\x{300}");
283 $str = "P\x{300}e\x{300}\x{301}\x{303}rl";
286 if (my($pos, $len) = $Collator->index($str, $sub)) {
287 $match = substr($str, $pos, $len);
289 ok($match, "P\x{300}e\x{300}\x{301}\x{303}");
291 $Collator->change(%old_level);
295 %old_level = $trad->change(level => 1);
297 $str = "Ich mu\x{00DF} studieren.";
298 $sub = "m\x{00FC}ss";
300 if (my($pos, $len) = $trad->index($str, $sub)) {
301 $match = substr($str, $pos, $len);
303 ok($match, "mu\x{00DF}");
305 $trad->change(%old_level);
307 $str = "Ich mu\x{00DF} studieren.";
308 $sub = "m\x{00FC}ss";
311 if (my($pos, $len) = $trad->index($str, $sub)) {
312 $match = substr($str, $pos, $len);
317 if (my($pos,$len) = $Collator->index("", "")) {
318 $match = substr("", $pos, $len);
323 if (my($pos,$len) = $Collator->index("", "abc")) {
324 $match = substr("", $pos, $len);
330 # Table is undefined, then no entry is defined.
332 my $undef_table = Unicode::Collate->new(
334 normalization => undef,
338 # in the Unicode code point order
339 ok($undef_table->lt('', 'A'));
340 ok($undef_table->lt('ABC', 'B'));
342 # Hangul should be decomposed (even w/o Unicode::Normalize).
344 ok($undef_table->lt("Perl", "\x{AC00}"));
345 ok($undef_table->eq("\x{AC00}", "\x{1100}\x{1161}"));
346 ok($undef_table->eq("\x{AE00}", "\x{1100}\x{1173}\x{11AF}"));
347 ok($undef_table->lt("\x{AE00}", "\x{3042}"));
349 # U+AE00: Hangul GEUL
352 # Weight for CJK Ideographs is defined, though.
354 ok($undef_table->lt("", "\x{4E00}"));
355 ok($undef_table->lt("\x{4E8C}","ABC"));
356 ok($undef_table->lt("\x{4E00}","\x{3042}"));
357 ok($undef_table->lt("\x{4E00}","\x{4E8C}"));
358 # U+4E00: Ideograph "ONE"
359 # U+4E8C: Ideograph "TWO"
364 my $few_entries = Unicode::Collate->new(
365 entry => <<'ENTRIES',
366 0050 ; [.0101.0020.0002.0050] # P
367 0045 ; [.0102.0020.0002.0045] # E
368 0052 ; [.0103.0020.0002.0052] # R
369 004C ; [.0104.0020.0002.004C] # L
370 1100 ; [.0105.0020.0002.1100] # Hangul Jamo initial G
371 1175 ; [.0106.0020.0002.1175] # Hangul Jamo middle I
372 5B57 ; [.0107.0020.0002.5B57] # CJK Ideograph "Letter"
375 normalization => undef,
378 # defined before undefined
380 my $sortABC = join '',
381 $few_entries->sort(split //, "ABCDEFGHIJKLMNOPQRSTUVWXYZ ");
383 ok($sortABC eq "PERL ABCDFGHIJKMNOQSTUVWXYZ");
385 ok($few_entries->lt('E', 'D'));
386 ok($few_entries->lt("\x{5B57}", "\x{4E00}"));
387 ok($few_entries->lt("\x{AE30}", "\x{AC00}"));
389 # Hangul must be decomposed.
391 ok($few_entries->eq("\x{AC00}", "\x{1100}\x{1161}"));
395 my $all_undef_8 = Unicode::Collate->new(
397 normalization => undef,
398 overrideCJK => undef,
399 overrideHangul => undef,
403 # All in the Unicode code point order.
404 # No hangul decomposition.
406 ok($all_undef_8->lt("\x{3402}", "\x{4E00}"));
407 ok($all_undef_8->lt("\x{4DFF}", "\x{4E00}"));
408 ok($all_undef_8->lt("\x{4E00}", "\x{AC00}"));
409 ok($all_undef_8->gt("\x{AC00}", "\x{1100}\x{1161}"));
410 ok($all_undef_8->gt("\x{AC00}", "\x{ABFF}"));
414 my $all_undef_9 = Unicode::Collate->new(
416 normalization => undef,
417 overrideCJK => undef,
418 overrideHangul => undef,
422 # CJK Ideo. < CJK ext A/B < Others.
423 # No hangul decomposition.
425 ok($all_undef_9->lt("\x{4E00}", "\x{3402}"));
426 ok($all_undef_9->lt("\x{3402}", "\x{20000}"));
427 ok($all_undef_9->lt("\x{20000}", "\x{AC00}"));
428 ok($all_undef_9->gt("\x{AC00}", "\x{1100}\x{1161}"));
429 ok($all_undef_9->gt("\x{AC00}", "\x{ABFF}")); # U+ABFF: not assigned
433 my $ignoreCJK = Unicode::Collate->new(
435 normalization => undef,
436 overrideCJK => sub {()},
437 entry => <<'ENTRIES',
438 5B57 ; [.0107.0020.0002.5B57] # CJK Ideograph "Letter"
442 # All CJK Unified Ideographs except U+5B57 are ignored.
444 ok($ignoreCJK->eq("\x{4E00}", ""));
445 ok($ignoreCJK->lt("\x{4E00}", "\0"));
446 ok($ignoreCJK->eq("Pe\x{4E00}rl", "Perl")); # U+4E00 is a CJK.
447 ok($ignoreCJK->gt("\x{4DFF}", "\x{4E00}")); # U+4DFF is not CJK.
448 ok($ignoreCJK->lt("Pe\x{5B57}rl", "Perl")); # 'r' is unassigned.
452 my $ignoreHangul = Unicode::Collate->new(
454 normalization => undef,
455 overrideHangul => sub {()},
456 entry => <<'ENTRIES',
457 AE00 ; [.0100.0020.0002.AE00] # Hangul GEUL
461 # All Hangul Syllables except U+AE00 are ignored.
463 ok($ignoreHangul->eq("\x{AC00}", ""));
464 ok($ignoreHangul->lt("\x{AC00}", "\0"));
465 ok($ignoreHangul->lt("\x{AC00}", "\x{AE00}"));
466 ok($ignoreHangul->lt("\x{AC00}", "\x{1100}\x{1161}")); # Jamo are not ignored.
467 ok($ignoreHangul->lt("Pe\x{AE00}rl", "Perl")); # 'r' is unassigned.
471 my %origAlter = $Collator->change(alternate => 'Blanked');
473 ok($Collator->lt("death", "de luge"));
474 ok($Collator->lt("de luge", "de-luge"));
475 ok($Collator->lt("de-luge", "deluge"));
476 ok($Collator->lt("deluge", "de\x{2010}luge"));
477 ok($Collator->lt("deluge", "de Luge"));
479 $Collator->change(alternate => 'Non-ignorable');
481 ok($Collator->lt("de luge", "de Luge"));
482 ok($Collator->lt("de Luge", "de-luge"));
483 ok($Collator->lt("de-Luge", "de\x{2010}luge"));
484 ok($Collator->lt("de-luge", "death"));
485 ok($Collator->lt("death", "deluge"));
487 $Collator->change(alternate => 'Shifted');
489 ok($Collator->lt("death", "de luge"));
490 ok($Collator->lt("de luge", "de-luge"));
491 ok($Collator->lt("de-luge", "deluge"));
492 ok($Collator->lt("deluge", "de Luge"));
493 ok($Collator->lt("de Luge", "deLuge"));
495 $Collator->change(alternate => 'Shift-Trimmed');
497 ok($Collator->lt("death", "deluge"));
498 ok($Collator->lt("deluge", "de luge"));
499 ok($Collator->lt("de luge", "de-luge"));
500 ok($Collator->lt("de-luge", "deLuge"));
501 ok($Collator->lt("deLuge", "de Luge"));
503 $Collator->change(%origAlter);
505 ok($Collator->{alternate}, 'shifted');
509 my $overCJK = Unicode::Collate->new(
511 normalization => undef,
512 entry => <<'ENTRIES',
513 0061 ; [.0101.0020.0002.0061] # latin a
514 0041 ; [.0101.0020.0008.0041] # LATIN A
515 4E00 ; [.B1FC.0030.0004.4E00] # Ideograph; B1FC = FFFF - 4E03.
518 my $u = 0xFFFF - $_[0]; # reversed
523 ok($overCJK->lt("a", "A")); # diff. at level 3.
524 ok($overCJK->lt( "\x{4E03}", "\x{4E00}")); # diff. at level 2.
525 ok($overCJK->lt("A\x{4E03}", "A\x{4E00}"));
526 ok($overCJK->lt("A\x{4E03}", "a\x{4E00}"));
527 ok($overCJK->lt("a\x{4E03}", "A\x{4E00}"));
531 # rearrange : 0x0E40..0x0E44, 0x0EC0..0x0EC4 (default)
533 my %old_rearrange = $Collator->change(rearrange => undef);
535 ok($Collator->gt("\x{0E41}A", "\x{0E40}B"));
536 ok($Collator->gt("A\x{0E41}A", "A\x{0E40}B"));
538 $Collator->change(rearrange => [ 0x61 ]); # 'a'
540 ok($Collator->gt("ab", "AB")); # as 'ba' > 'AB'
542 $Collator->change(%old_rearrange);
544 ok($Collator->lt("ab", "AB"));
545 ok($Collator->lt("\x{0E40}", "\x{0E41}"));
546 ok($Collator->lt("\x{0E40}A", "\x{0E41}B"));
547 ok($Collator->lt("\x{0E41}A", "\x{0E40}B"));
548 ok($Collator->lt("A\x{0E41}A", "A\x{0E40}B"));
550 ok($all_undef_8->lt("\x{0E40}", "\x{0E41}"));
551 ok($all_undef_8->lt("\x{0E40}A", "\x{0E41}B"));
552 ok($all_undef_8->lt("\x{0E41}A", "\x{0E40}B"));
553 ok($all_undef_8->lt("A\x{0E41}A", "A\x{0E40}B"));
557 my $no_rearrange = Unicode::Collate->new(
559 normalization => undef,
563 ok($no_rearrange->lt("A", "B"));
564 ok($no_rearrange->lt("\x{0E40}", "\x{0E41}"));
565 ok($no_rearrange->lt("\x{0E40}A", "\x{0E41}B"));
566 ok($no_rearrange->gt("\x{0E41}A", "\x{0E40}B"));
567 ok($no_rearrange->gt("A\x{0E41}A", "A\x{0E40}B"));
571 my $undef_rearrange = Unicode::Collate->new(
573 normalization => undef,
577 ok($undef_rearrange->lt("A", "B"));
578 ok($undef_rearrange->lt("\x{0E40}", "\x{0E41}"));
579 ok($undef_rearrange->lt("\x{0E40}A", "\x{0E41}B"));
580 ok($undef_rearrange->gt("\x{0E41}A", "\x{0E40}B"));
581 ok($undef_rearrange->gt("A\x{0E41}A", "A\x{0E40}B"));
585 my $dropArticles = Unicode::Collate->new(
587 normalization => undef,
590 $string =~ s/\b(?:an?|the)\s+//ig;
595 ok($dropArticles->eq("camel", "a camel"));
596 ok($dropArticles->eq("Perl", "The Perl"));
597 ok($dropArticles->lt("the pen", "a pencil"));
598 ok($Collator->lt("Perl", "The Perl"));
599 ok($Collator->gt("the pen", "a pencil"));
603 my $backLevel1 = Unicode::Collate->new(
605 normalization => undef,
609 # all strings are reversed at level 1.
611 ok($backLevel1->gt("AB", "BA"));
612 ok($backLevel1->gt("\x{3042}\x{3044}", "\x{3044}\x{3042}"));
616 my $backLevel2 = Unicode::Collate->new(
618 normalization => undef,
619 undefName => qr/HANGUL|HIRAGANA|KATAKANA|BOPOMOFO/,
623 ok($backLevel2->gt("Ca\x{300}ca\x{302}", "ca\x{302}ca\x{300}"));
624 ok($backLevel2->gt("ca\x{300}ca\x{302}", "Ca\x{302}ca\x{300}"));
625 ok($Collator ->lt("Ca\x{300}ca\x{302}", "ca\x{302}ca\x{300}"));
626 ok($Collator ->lt("ca\x{300}ca\x{302}", "Ca\x{302}ca\x{300}"));
629 # HIRAGANA and KATAKANA are made undefined via undefName.
630 # So they are after CJK Unified Ideographs.
632 ok($backLevel2->lt("\x{4E00}", $hiragana));
633 ok($backLevel2->lt("\x{4E03}", $katakana));
634 ok($Collator ->gt("\x{4E00}", $hiragana));
635 ok($Collator ->gt("\x{4E03}", $katakana));
639 # Shifted; ignorable after variable
641 ok($Collator->eq("?\x{300}!\x{301}\x{315}", "?!"));
642 ok($Collator->eq("?\x{300}A\x{300}", "?A\x{300}"));
643 ok($Collator->eq("?\x{300}", "?"));
645 $Collator->change(alternate => 'Non-ignorable');
647 ok($Collator->gt("?\x{300}", "?"));
649 $Collator->change(alternate => 'Shifted');