3 unless ("A" eq pack('U', 0x41) || "A" eq pack('U', ord("A"))) {
4 print "1..0 # Unicode::Collate " .
5 "cannot stringify a Unicode code point\n";
11 if ($ENV{PERL_CORE}) {
13 @INC = $^O eq 'MacOS' ? qw(::lib) : qw(../lib);
18 BEGIN { plan tests => 194 };
21 our $IsEBCDIC = ord("A") != 0x41;
23 #########################
25 ok(1); # If we made it this far, we're ok.
27 my $UCA_Version = "9";
29 ok(Unicode::Collate::UCA_Version, $UCA_Version);
30 ok(Unicode::Collate->UCA_Version, $UCA_Version);
32 my $Collator = Unicode::Collate->new(
34 normalization => undef,
37 ok(ref $Collator, "Unicode::Collate");
39 ok($Collator->UCA_Version, $UCA_Version);
40 ok($Collator->UCA_Version(), $UCA_Version);
43 join(':', $Collator->sort(
44 qw/ lib strict Carp ExtUtils CGI Time warnings Math overload Pod CPAN /
47 qw/ Carp CGI CPAN ExtUtils lib Math overload Pod strict Time warnings /
51 ok($Collator->cmp("", ""), 0);
52 ok($Collator->eq("", ""));
53 ok($Collator->cmp("", "perl"), -1);
57 sub _pack_U { Unicode::Collate::pack_U(@_) }
58 sub _unpack_U { Unicode::Collate::unpack_U(@_) }
60 my $A_acute = _pack_U(0xC1);
61 my $a_acute = _pack_U(0xE1);
62 my $acute = _pack_U(0x0301);
64 ok($Collator->cmp("A$acute", $A_acute), 0); # @version 3.1.1 (prev: -1)
65 ok($Collator->cmp($a_acute, $A_acute), -1);
66 ok($Collator->eq("A\cA$acute", $A_acute)); # UCA v9. \cA is invariant.
68 my %old_level = $Collator->change(level => 1);
69 ok($Collator->eq("A$acute", $A_acute));
70 ok($Collator->eq("A", $A_acute));
72 ok($Collator->change(level => 2)->eq($a_acute, $A_acute));
73 ok($Collator->lt("A", $A_acute));
75 ok($Collator->change(%old_level)->lt("A", $A_acute));
76 ok($Collator->lt("A", $A_acute));
77 ok($Collator->lt("A", $a_acute));
78 ok($Collator->lt($a_acute, $A_acute));
82 eval { require Unicode::Normalize };
84 if (!$@ && !$IsEBCDIC) {
85 my $NFD = Unicode::Collate->new(
88 0430 ; [.0CB5.0020.0002.0430] # CYRILLIC SMALL LETTER A
89 0410 ; [.0CB5.0020.0008.0410] # CYRILLIC CAPITAL LETTER A
90 04D3 ; [.0CBD.0020.0002.04D3] # CYRILLIC SMALL LETTER A WITH DIAERESIS
91 0430 0308 ; [.0CBD.0020.0002.04D3] # CYRILLIC SMALL LETTER A WITH DIAERESIS
92 04D2 ; [.0CBD.0020.0008.04D2] # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
93 0410 0308 ; [.0CBD.0020.0008.04D2] # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
94 0430 3099 ; [.0CBE.0020.0002.04D3] # A WITH KATAKANA VOICED
95 0430 3099 0308 ; [.0CBF.0020.0002.04D3] # A WITH KATAKANA VOICED, DIAERESIS
98 ok($NFD->eq("\x{4D3}\x{325}", "\x{430}\x{308}\x{325}"));
99 ok($NFD->lt("\x{430}\x{308}A", "\x{430}\x{308}B"));
100 ok($NFD->lt("\x{430}\x{3099}B", "\x{430}\x{308}\x{3099}A"));
101 ok($NFD->eq("\x{0430}\x{3099}\x{309A}\x{0308}",
102 "\x{0430}\x{309A}\x{3099}\x{0308}") );
113 my $trad = Unicode::Collate->new(
115 normalization => undef,
116 ignoreName => qr/HANGUL|HIRAGANA|KATAKANA|BOPOMOFO/,
118 entry => << 'ENTRIES',
119 0063 0068 ; [.0A3F.0020.0002.0063] % "ch" in traditional Spanish
120 0043 0068 ; [.0A3F.0020.0008.0043] # "Ch" in traditional Spanish
123 # 0063 ; [.0A3D.0020.0002.0063] # LATIN SMALL LETTER C
124 # 0064 ; [.0A49.0020.0002.0064] # LATIN SMALL LETTER D
125 # Deutsch sz is included in 'keys.txt';
128 join(':', $trad->sort( qw/ acha aca ada acia acka / ) ),
129 join(':', qw/ aca acia acka acha ada / ),
133 join(':', $Collator->sort( qw/ acha aca ada acia acka / ) ),
134 join(':', qw/ aca acha acia acka ada / ),
136 ok($trad->eq("ocho", "oc\cAho")); # UCA v9
137 ok($trad->eq("ocho", "oc\0\cA\0\cBho")); # UCA v9
139 my $hiragana = "\x{3042}\x{3044}";
140 my $katakana = "\x{30A2}\x{30A4}";
142 # HIRAGANA and KATAKANA are ignorable via ignoreName
143 ok($trad->eq($hiragana, ""));
144 ok($trad->eq("", $katakana));
145 ok($trad->eq($hiragana, $katakana));
146 ok($trad->eq($katakana, $hiragana));
150 $Collator->change(level => 2);
152 ok($Collator->{level}, 2);
154 ok( $Collator->cmp("ABC","abc"), 0);
155 ok( $Collator->eq("ABC","abc") );
156 ok( $Collator->le("ABC","abc") );
157 ok( $Collator->cmp($hiragana, $katakana), 0);
158 ok( $Collator->eq($hiragana, $katakana) );
159 ok( $Collator->ge($hiragana, $katakana) );
162 ok( $Collator->eq("a\x{AC00}b", "a\x{1100}\x{1161}b") );
163 ok( $Collator->eq("a\x{AE00}b", "a\x{1100}\x{1173}\x{11AF}b") );
164 ok( $Collator->gt("a\x{AE00}b", "a\x{1100}\x{1173}b\x{11AF}") );
165 ok( $Collator->lt("a\x{AC00}b", "a\x{AE00}b") );
166 ok( $Collator->gt("a\x{D7A3}b", "a\x{C544}b") );
167 ok( $Collator->lt("a\x{C544}b", "a\x{30A2}b") ); # hangul < hiragana
169 $Collator->change(%old_level, katakana_before_hiragana => 1);
171 ok($Collator->{level}, 4);
173 ok( $Collator->cmp("abc", "ABC"), -1);
174 ok( $Collator->ne("abc", "ABC") );
175 ok( $Collator->lt("abc", "ABC") );
176 ok( $Collator->le("abc", "ABC") );
177 ok( $Collator->cmp($hiragana, $katakana), 1);
178 ok( $Collator->ne($hiragana, $katakana) );
179 ok( $Collator->gt($hiragana, $katakana) );
180 ok( $Collator->ge($hiragana, $katakana) );
182 $Collator->change(upper_before_lower => 1);
184 ok( $Collator->cmp("abc", "ABC"), 1);
185 ok( $Collator->ge("abc", "ABC"), 1);
186 ok( $Collator->gt("abc", "ABC"), 1);
187 ok( $Collator->cmp($hiragana, $katakana), 1);
188 ok( $Collator->ge($hiragana, $katakana), 1);
189 ok( $Collator->gt($hiragana, $katakana), 1);
191 $Collator->change(katakana_before_hiragana => 0);
193 ok( $Collator->cmp("abc", "ABC"), 1);
194 ok( $Collator->cmp($hiragana, $katakana), -1);
196 $Collator->change(upper_before_lower => 0);
198 ok( $Collator->cmp("abc", "ABC"), -1);
199 ok( $Collator->le("abc", "ABC") );
200 ok( $Collator->cmp($hiragana, $katakana), -1);
201 ok( $Collator->lt($hiragana, $katakana) );
205 my $ignoreAE = Unicode::Collate->new(
207 normalization => undef,
208 ignoreChar => qr/^[aAeE]$/,
211 ok($ignoreAE->eq("element","lament"));
212 ok($ignoreAE->eq("Perl","ePrl"));
216 my $onlyABC = Unicode::Collate->new(
218 normalization => undef,
219 entry => << 'ENTRIES',
220 0061 ; [.0101.0020.0002.0061] # LATIN SMALL LETTER A
221 0041 ; [.0101.0020.0008.0041] # LATIN CAPITAL LETTER A
222 0062 ; [.0102.0020.0002.0062] # LATIN SMALL LETTER B
223 0042 ; [.0102.0020.0008.0042] # LATIN CAPITAL LETTER B
224 0063 ; [.0103.0020.0002.0063] # LATIN SMALL LETTER C
225 0043 ; [.0103.0020.0008.0043] # LATIN CAPITAL LETTER C
230 join(':', $onlyABC->sort( qw/ ABA BAC cc A Ab cAc aB / ) ),
231 join(':', qw/ A aB Ab ABA BAC cAc cc / ),
236 my $undefAE = Unicode::Collate->new(
238 normalization => undef,
239 undefChar => qr/^[aAeE]$/,
242 ok($undefAE ->gt("edge","fog"));
243 ok($Collator->lt("edge","fog"));
244 ok($undefAE ->gt("lake","like"));
245 ok($Collator->lt("lake","like"));
249 # Table is undefined, then no entry is defined.
251 my $undef_table = Unicode::Collate->new(
253 normalization => undef,
257 # in the Unicode code point order
258 ok($undef_table->lt('', 'A'));
259 ok($undef_table->lt('ABC', 'B'));
261 # Hangul should be decomposed (even w/o Unicode::Normalize).
263 ok($undef_table->lt("Perl", "\x{AC00}"));
264 ok($undef_table->eq("\x{AC00}", "\x{1100}\x{1161}"));
265 ok($undef_table->eq("\x{AE00}", "\x{1100}\x{1173}\x{11AF}"));
266 ok($undef_table->lt("\x{AE00}", "\x{3042}"));
268 # U+AE00: Hangul GEUL
271 # Weight for CJK Ideographs is defined, though.
273 ok($undef_table->lt("", "\x{4E00}"));
274 ok($undef_table->lt("\x{4E8C}","ABC"));
275 ok($undef_table->lt("\x{4E00}","\x{3042}"));
276 ok($undef_table->lt("\x{4E00}","\x{4E8C}"));
277 # U+4E00: Ideograph "ONE"
278 # U+4E8C: Ideograph "TWO"
283 my $few_entries = Unicode::Collate->new(
284 entry => <<'ENTRIES',
285 0050 ; [.0101.0020.0002.0050] # P
286 0045 ; [.0102.0020.0002.0045] # E
287 0052 ; [.0103.0020.0002.0052] # R
288 004C ; [.0104.0020.0002.004C] # L
289 1100 ; [.0105.0020.0002.1100] # Hangul Jamo initial G
290 1175 ; [.0106.0020.0002.1175] # Hangul Jamo middle I
291 5B57 ; [.0107.0020.0002.5B57] # CJK Ideograph "Letter"
294 normalization => undef,
297 # defined before undefined
299 my $sortABC = join '',
300 $few_entries->sort(split //, "ABCDEFGHIJKLMNOPQRSTUVWXYZ ");
302 ok($sortABC eq "PERL ABCDFGHIJKMNOQSTUVWXYZ");
304 ok($few_entries->lt('E', 'D'));
305 ok($few_entries->lt("\x{5B57}", "\x{4E00}"));
306 ok($few_entries->lt("\x{AE30}", "\x{AC00}"));
308 # Hangul must be decomposed.
310 ok($few_entries->eq("\x{AC00}", "\x{1100}\x{1161}"));
314 my $all_undef_8 = Unicode::Collate->new(
316 normalization => undef,
317 overrideCJK => undef,
318 overrideHangul => undef,
322 # All in the Unicode code point order.
323 # No hangul decomposition.
325 ok($all_undef_8->lt("\x{3402}", "\x{4E00}"));
326 ok($all_undef_8->lt("\x{4DFF}", "\x{4E00}"));
327 ok($all_undef_8->lt("\x{4E00}", "\x{AC00}"));
328 ok($all_undef_8->gt("\x{AC00}", "\x{1100}\x{1161}"));
329 ok($all_undef_8->gt("\x{AC00}", "\x{ABFF}"));
333 my $all_undef_9 = Unicode::Collate->new(
335 normalization => undef,
336 overrideCJK => undef,
337 overrideHangul => undef,
341 # CJK Ideo. < CJK ext A/B < Others.
342 # No hangul decomposition.
344 ok($all_undef_9->lt("\x{4E00}", "\x{3402}"));
345 ok($all_undef_9->lt("\x{3402}", "\x{20000}"));
346 ok($all_undef_9->lt("\x{20000}", "\x{AC00}"));
347 ok($all_undef_9->gt("\x{AC00}", "\x{1100}\x{1161}"));
348 ok($all_undef_9->gt("\x{AC00}", "\x{ABFF}")); # U+ABFF: not assigned
352 my $ignoreCJK = Unicode::Collate->new(
354 normalization => undef,
355 overrideCJK => sub {()},
356 entry => <<'ENTRIES',
357 5B57 ; [.0107.0020.0002.5B57] # CJK Ideograph "Letter"
361 # All CJK Unified Ideographs except U+5B57 are ignored.
363 ok($ignoreCJK->eq("\x{4E00}", ""));
364 ok($ignoreCJK->lt("\x{4E00}", "\0"));
365 ok($ignoreCJK->eq("Pe\x{4E00}rl", "Perl")); # U+4E00 is a CJK.
366 ok($ignoreCJK->gt("\x{4DFF}", "\x{4E00}")); # U+4DFF is not CJK.
367 ok($ignoreCJK->lt("Pe\x{5B57}rl", "Perl")); # 'r' is unassigned.
371 my $ignoreHangul = Unicode::Collate->new(
373 normalization => undef,
374 overrideHangul => sub {()},
375 entry => <<'ENTRIES',
376 AE00 ; [.0100.0020.0002.AE00] # Hangul GEUL
380 # All Hangul Syllables except U+AE00 are ignored.
382 ok($ignoreHangul->eq("\x{AC00}", ""));
383 ok($ignoreHangul->lt("\x{AC00}", "\0"));
384 ok($ignoreHangul->lt("\x{AC00}", "\x{AE00}"));
385 ok($ignoreHangul->lt("\x{AC00}", "\x{1100}\x{1161}")); # Jamo are not ignored.
386 ok($ignoreHangul->lt("Pe\x{AE00}rl", "Perl")); # 'r' is unassigned.
390 my %origAlter = $Collator->change(alternate => 'Blanked');
392 ok($Collator->lt("death", "de luge"));
393 ok($Collator->lt("de luge", "de-luge"));
394 ok($Collator->lt("de-luge", "deluge"));
395 ok($Collator->lt("deluge", "de\x{2010}luge"));
396 ok($Collator->lt("deluge", "de Luge"));
398 $Collator->change(alternate => 'Non-ignorable');
400 ok($Collator->lt("de luge", "de Luge"));
401 ok($Collator->lt("de Luge", "de-luge"));
402 ok($Collator->lt("de-Luge", "de\x{2010}luge"));
403 ok($Collator->lt("de-luge", "death"));
404 ok($Collator->lt("death", "deluge"));
406 $Collator->change(alternate => 'Shifted');
408 ok($Collator->lt("death", "de luge"));
409 ok($Collator->lt("de luge", "de-luge"));
410 ok($Collator->lt("de-luge", "deluge"));
411 ok($Collator->lt("deluge", "de Luge"));
412 ok($Collator->lt("de Luge", "deLuge"));
414 $Collator->change(alternate => 'Shift-Trimmed');
416 ok($Collator->lt("death", "deluge"));
417 ok($Collator->lt("deluge", "de luge"));
418 ok($Collator->lt("de luge", "de-luge"));
419 ok($Collator->lt("de-luge", "deLuge"));
420 ok($Collator->lt("deLuge", "de Luge"));
422 $Collator->change(%origAlter);
424 ok($Collator->{alternate}, 'shifted');
428 my $overCJK = Unicode::Collate->new(
430 normalization => undef,
431 entry => <<'ENTRIES',
432 0061 ; [.0101.0020.0002.0061] # latin a
433 0041 ; [.0101.0020.0008.0041] # LATIN A
434 4E00 ; [.B1FC.0030.0004.4E00] # Ideograph; B1FC = FFFF - 4E03.
437 my $u = 0xFFFF - $_[0]; # reversed
442 ok($overCJK->lt("a", "A")); # diff. at level 3.
443 ok($overCJK->lt( "\x{4E03}", "\x{4E00}")); # diff. at level 2.
444 ok($overCJK->lt("A\x{4E03}", "A\x{4E00}"));
445 ok($overCJK->lt("A\x{4E03}", "a\x{4E00}"));
446 ok($overCJK->lt("a\x{4E03}", "A\x{4E00}"));
450 # rearrange : 0x0E40..0x0E44, 0x0EC0..0x0EC4 (default)
452 my %old_rearrange = $Collator->change(rearrange => undef);
454 ok($Collator->gt("\x{0E41}A", "\x{0E40}B"));
455 ok($Collator->gt("A\x{0E41}A", "A\x{0E40}B"));
457 $Collator->change(rearrange => [ 0x61 ]);
458 # U+0061, 'a': This is a Unicode value, never a native value.
460 ok($Collator->gt("ab", "AB")); # as 'ba' > 'AB'
462 $Collator->change(%old_rearrange);
464 ok($Collator->lt("ab", "AB"));
465 ok($Collator->lt("\x{0E40}", "\x{0E41}"));
466 ok($Collator->lt("\x{0E40}A", "\x{0E41}B"));
467 ok($Collator->lt("\x{0E41}A", "\x{0E40}B"));
468 ok($Collator->lt("A\x{0E41}A", "A\x{0E40}B"));
470 ok($all_undef_8->lt("\x{0E40}", "\x{0E41}"));
471 ok($all_undef_8->lt("\x{0E40}A", "\x{0E41}B"));
472 ok($all_undef_8->lt("\x{0E41}A", "\x{0E40}B"));
473 ok($all_undef_8->lt("A\x{0E41}A", "A\x{0E40}B"));
477 my $no_rearrange = Unicode::Collate->new(
479 normalization => undef,
483 ok($no_rearrange->lt("A", "B"));
484 ok($no_rearrange->lt("\x{0E40}", "\x{0E41}"));
485 ok($no_rearrange->lt("\x{0E40}A", "\x{0E41}B"));
486 ok($no_rearrange->gt("\x{0E41}A", "\x{0E40}B"));
487 ok($no_rearrange->gt("A\x{0E41}A", "A\x{0E40}B"));
491 my $undef_rearrange = Unicode::Collate->new(
493 normalization => undef,
497 ok($undef_rearrange->lt("A", "B"));
498 ok($undef_rearrange->lt("\x{0E40}", "\x{0E41}"));
499 ok($undef_rearrange->lt("\x{0E40}A", "\x{0E41}B"));
500 ok($undef_rearrange->gt("\x{0E41}A", "\x{0E40}B"));
501 ok($undef_rearrange->gt("A\x{0E41}A", "A\x{0E40}B"));
505 my $dropArticles = Unicode::Collate->new(
507 normalization => undef,
510 $string =~ s/\b(?:an?|the)\s+//ig;
515 ok($dropArticles->eq("camel", "a camel"));
516 ok($dropArticles->eq("Perl", "The Perl"));
517 ok($dropArticles->lt("the pen", "a pencil"));
518 ok($Collator->lt("Perl", "The Perl"));
519 ok($Collator->gt("the pen", "a pencil"));
523 my $backLevel1 = Unicode::Collate->new(
525 normalization => undef,
529 # all strings are reversed at level 1.
531 ok($backLevel1->gt("AB", "BA"));
532 ok($backLevel1->gt("\x{3042}\x{3044}", "\x{3044}\x{3042}"));
536 my $backLevel2 = Unicode::Collate->new(
538 normalization => undef,
539 undefName => qr/HANGUL|HIRAGANA|KATAKANA|BOPOMOFO/,
543 ok($backLevel2->gt("Ca\x{300}ca\x{302}", "ca\x{302}ca\x{300}"));
544 ok($backLevel2->gt("ca\x{300}ca\x{302}", "Ca\x{302}ca\x{300}"));
545 ok($Collator ->lt("Ca\x{300}ca\x{302}", "ca\x{302}ca\x{300}"));
546 ok($Collator ->lt("ca\x{300}ca\x{302}", "Ca\x{302}ca\x{300}"));
548 # HIRAGANA and KATAKANA are made undefined via undefName.
549 # So they are after CJK Unified Ideographs.
551 ok($backLevel2->lt("\x{4E00}", $hiragana));
552 ok($backLevel2->lt("\x{4E03}", $katakana));
553 ok($Collator ->gt("\x{4E00}", $hiragana));
554 ok($Collator ->gt("\x{4E03}", $katakana));
558 # ignorable after variable
561 ok($Collator->eq("?\x{300}!\x{301}\x{315}", "?!"));
562 ok($Collator->eq("?\x{300}A\x{301}", "?$A_acute"));
563 ok($Collator->eq("?\x{300}", "?"));
564 ok($Collator->eq("?\x{344}", "?")); # U+0344 has two CEs.
566 $Collator->change(level => 3);
567 ok($Collator->eq("\cA", "?"));
569 $Collator->change(alternate => 'blanked', level => 4);
570 ok($Collator->eq("?\x{300}!\x{301}\x{315}", "?!"));
571 ok($Collator->eq("?\x{300}A\x{301}", "?$A_acute"));
572 ok($Collator->eq("?\x{300}", "?"));
573 ok($Collator->eq("?\x{344}", "?")); # U+0344 has two CEs.
575 $Collator->change(level => 3);
576 ok($Collator->eq("\cA", "?"));
578 $Collator->change(alternate => 'Non-ignorable', level => 4);
580 ok($Collator->lt("?\x{300}", "?!"));
581 ok($Collator->gt("?\x{300}A$acute", "?$A_acute"));
582 ok($Collator->gt("?\x{300}", "?"));
583 ok($Collator->gt("?\x{344}", "?"));
585 $Collator->change(level => 3);
586 ok($Collator->lt("\cA", "?"));
588 $Collator->change(alternate => 'Shifted', level => 4);
592 # According to Conformance Test,
593 # a L3-ignorable is treated as a completely ignorable.
595 my $L3ignorable = Unicode::Collate->new(
596 alternate => 'Non-ignorable',
598 normalization => undef,
599 entry => <<'ENTRIES',
600 0000 ; [.0000.0000.0000.0000] # [0000] NULL (in 6429)
601 0001 ; [.0000.0000.0000.0000] # [0001] START OF HEADING (in 6429)
602 0591 ; [.0000.0000.0000.0591] # HEBREW ACCENT ETNAHTA
603 1D165 ; [.0000.0000.0000.1D165] # MUSICAL SYMBOL COMBINING STEM
604 0021 ; [*024B.0020.0002.0021] # EXCLAMATION MARK
605 09BE ; [.114E.0020.0002.09BE] # BENGALI VOWEL SIGN AA
606 09C7 ; [.1157.0020.0002.09C7] # BENGALI VOWEL SIGN E
607 09CB ; [.1159.0020.0002.09CB] # BENGALI VOWEL SIGN O
608 09C7 09BE ; [.1159.0020.0002.09CB] # BENGALI VOWEL SIGN O
612 ok($L3ignorable->lt("\cA", "!"));
613 ok($L3ignorable->lt("\x{591}", "!"));
614 ok($L3ignorable->eq("\cA", "\x{591}"));
615 ok($L3ignorable->eq("\x{09C7}\x{09BE}A", "\x{09C7}\cA\x{09BE}A"));
616 ok($L3ignorable->eq("\x{09C7}\x{09BE}A", "\x{09C7}\x{0591}\x{09BE}A"));
617 ok($L3ignorable->eq("\x{09C7}\x{09BE}A", "\x{09C7}\x{1D165}\x{09BE}A"));
618 ok($L3ignorable->eq("\x{09C7}\x{09BE}A", "\x{09CB}A"));