[p5sagit/p5-mst-13.2.git] / ext / Unicode / Normalize / t / func.t


BEGIN {
    unless ("A" eq pack('U', 0x41)) {
	print "1..0 # Unicode::Normalize " .
	    "cannot stringify a Unicode code point\n";
	exit 0;
    }
}

BEGIN {
    if ($ENV{PERL_CORE}) {
        chdir('t') if -d 't';
        @INC = $^O eq 'MacOS' ? qw(::lib) : qw(../lib);
    }
}

#########################

use Test;
use strict;
use warnings;
BEGIN { plan tests => 202 };
use Unicode::Normalize qw(:all);
ok(1); # If we made it this far, we're ok.

sub _pack_U { Unicode::Normalize::pack_U(@_) }
sub hexU { _pack_U map hex, split ' ', shift }

#########################

ok(getCombinClass(   0),   0);
ok(getCombinClass(  41),   0);
ok(getCombinClass(  65),   0);
ok(getCombinClass( 768), 230);
ok(getCombinClass(1809),  36);

ok(getCanon(   0), undef);
ok(getCanon(0x29), undef);
ok(getCanon(0x41), undef);
ok(getCanon(0x00C0), _pack_U(0x0041, 0x0300));
ok(getCanon(0x00EF), _pack_U(0x0069, 0x0308));
ok(getCanon(0x304C), _pack_U(0x304B, 0x3099));
ok(getCanon(0x1EA4), _pack_U(0x0041, 0x0302, 0x0301));
ok(getCanon(0x1F82), _pack_U(0x03B1, 0x0313, 0x0300, 0x0345));
ok(getCanon(0x1FAF), _pack_U(0x03A9, 0x0314, 0x0342, 0x0345));
ok(getCanon(0xAC00), _pack_U(0x1100, 0x1161));
ok(getCanon(0xAE00), _pack_U(0x1100, 0x1173, 0x11AF));
ok(getCanon(0x212C), undef);
ok(getCanon(0x3243), undef);
ok(getCanon(0xFA2D), _pack_U(0x9DB4));

ok(getCompat(   0), undef);
ok(getCompat(0x29), undef);
ok(getCompat(0x41), undef);
ok(getCompat(0x00C0), _pack_U(0x0041, 0x0300));
ok(getCompat(0x00EF), _pack_U(0x0069, 0x0308));
ok(getCompat(0x304C), _pack_U(0x304B, 0x3099));
ok(getCompat(0x1EA4), _pack_U(0x0041, 0x0302, 0x0301));
ok(getCompat(0x1F82), _pack_U(0x03B1, 0x0313, 0x0300, 0x0345));
ok(getCompat(0x1FAF), _pack_U(0x03A9, 0x0314, 0x0342, 0x0345));
ok(getCompat(0x212C), _pack_U(0x0042));
ok(getCompat(0x3243), _pack_U(0x0028, 0x81F3, 0x0029));
ok(getCompat(0xAC00), _pack_U(0x1100, 0x1161));
ok(getCompat(0xAE00), _pack_U(0x1100, 0x1173, 0x11AF));
ok(getCompat(0xFA2D), _pack_U(0x9DB4));

ok(getComposite(   0,    0), undef);
ok(getComposite(   0, 0x29), undef);
ok(getComposite(0x29,    0), undef);
ok(getComposite(0x29, 0x29), undef);
ok(getComposite(   0, 0x41), undef);
ok(getComposite(0x41,    0), undef);
ok(getComposite(0x41, 0x41), undef);
ok(getComposite(12, 0x0300), undef);
ok(getComposite(0x0055, 0xFF00), undef);
ok(getComposite(0x0041, 0x0300), 0x00C0);
ok(getComposite(0x0055, 0x0300), 0x00D9);
ok(getComposite(0x0112, 0x0300), 0x1E14);
ok(getComposite(0x1100, 0x1161), 0xAC00);
ok(getComposite(0x1100, 0x1173), 0xADF8);
ok(getComposite(0x1100, 0x11AF), undef);
ok(getComposite(0x1173, 0x11AF), undef);
ok(getComposite(0xAC00, 0x11A7), undef);
ok(getComposite(0xAC00, 0x11A8), 0xAC01);
ok(getComposite(0xADF8, 0x11AF), 0xAE00);

sub uprops {
  my $uv = shift;
  my $r = "";
     $r .= isExclusion($uv)   ? 'X' : 'x';
     $r .= isSingleton($uv)   ? 'S' : 's';
     $r .= isNonStDecomp($uv) ? 'N' : 'n'; # Non-Starter Decomposition
     $r .= isComp_Ex($uv)     ? 'F' : 'f'; # Full exclusion (X + S + N)
     $r .= isComp2nd($uv)     ? 'B' : 'b'; # B = M = Y
     $r .= isNFD_NO($uv)      ? 'D' : 'd';
     $r .= isNFC_MAYBE($uv)   ? 'M' : 'm'; # Maybe
     $r .= isNFC_NO($uv)      ? 'C' : 'c';
     $r .= isNFKD_NO($uv)     ? 'K' : 'k';
     $r .= isNFKC_MAYBE($uv)  ? 'Y' : 'y'; # maYbe
     $r .= isNFKC_NO($uv)     ? 'G' : 'g';
  return $r;
}

ok(uprops(0x0000), 'xsnfbdmckyg');
ok(uprops(0x0029), 'xsnfbdmckyg');
ok(uprops(0x0041), 'xsnfbdmckyg');
ok(uprops(0x00A0), 'xsnfbdmcKyG'); # NO-BREAK SPACE
ok(uprops(0x00C0), 'xsnfbDmcKyg'); # LATIN CAPITAL LETTER A WITH GRAVE
ok(uprops(0x0300), 'xsnfBdMckYg'); # COMBINING GRAVE ACCENT
ok(uprops(0x0344), 'xsNFbDmCKyG'); # COMBINING GREEK DIALYTIKA TONOS
ok(uprops(0x0387), 'xSnFbDmCKyG'); # GREEK ANO TELEIA
ok(uprops(0x0958), 'XsnFbDmCKyG'); # DEVANAGARI LETTER QA
ok(uprops(0x0F43), 'XsnFbDmCKyG'); # TIBETAN LETTER GHA
ok(uprops(0x1100), 'xsnfbdmckyg'); # HANGUL CHOSEONG KIYEOK
ok(uprops(0x1161), 'xsnfBdMckYg'); # HANGUL JUNGSEONG A
ok(uprops(0x11AF), 'xsnfBdMckYg'); # HANGUL JONGSEONG RIEU
ok(uprops(0x212B), 'xSnFbDmCKyG'); # ANGSTROM SIGN
ok(uprops(0xAC00), 'xsnfbDmcKyg'); # HANGUL SYLLABLE GA
ok(uprops(0xF900), 'xSnFbDmCKyG'); # CJK COMPATIBILITY IDEOGRAPH-F900
ok(uprops(0xFB4E), 'XsnFbDmCKyG'); # HEBREW LETTER PE WITH RAFE
ok(uprops(0xFF71), 'xsnfbdmcKyG'); # HALFWIDTH KATAKANA LETTER A

ok(decompose(""), "");
ok(decompose("A"), "A");
ok(decompose("", 1), "");
ok(decompose("A", 1), "A");

ok(decompose(hexU("1E14 AC01")), hexU("0045 0304 0300 1100 1161 11A8"));
ok(decompose(hexU("AC00 AE00")), hexU("1100 1161 1100 1173 11AF"));
ok(decompose(hexU("304C FF76")), hexU("304B 3099 FF76"));

ok(decompose(hexU("1E14 AC01"), 1), hexU("0045 0304 0300 1100 1161 11A8"));
ok(decompose(hexU("AC00 AE00"), 1), hexU("1100 1161 1100 1173 11AF"));
ok(decompose(hexU("304C FF76"), 1), hexU("304B 3099 30AB"));

# don't modify the source
my $sDec = "\x{FA19}";
ok(decompose($sDec), "\x{795E}");
ok($sDec, "\x{FA19}");

ok(reorder(""), "");
ok(reorder("A"), "A");
ok(reorder(hexU("0041 0300 0315 0313 031b 0061")),
	   hexU("0041 031b 0300 0313 0315 0061"));
ok(reorder(hexU("00C1 0300 0315 0313 031b 0061 309A 3099")),
	   hexU("00C1 031b 0300 0313 0315 0061 309A 3099"));

# don't modify the source
my $sReord = "\x{3000}\x{300}\x{31b}";
ok(reorder($sReord), "\x{3000}\x{31b}\x{300}");
ok($sReord, "\x{3000}\x{300}\x{31b}");

ok(compose(""), "");
ok(compose("A"), "A");
ok(compose(hexU("0061 0300")),      hexU("00E0"));
ok(compose(hexU("0061 0300 031B")), hexU("00E0 031B"));
ok(compose(hexU("0061 0300 0315")), hexU("00E0 0315"));
ok(compose(hexU("0061 0300 0313")), hexU("00E0 0313"));
ok(compose(hexU("0061 031B 0300")), hexU("00E0 031B"));
ok(compose(hexU("0061 0315 0300")), hexU("0061 0315 0300"));
ok(compose(hexU("0061 0313 0300")), hexU("0061 0313 0300"));

# don't modify the source
my $sCom = "\x{304B}\x{3099}";
ok(compose($sCom), "\x{304C}");
ok($sCom, "\x{304B}\x{3099}");

ok(composeContiguous(""), "");
ok(composeContiguous("A"), "A");
ok(composeContiguous(hexU("0061 0300")),      hexU("00E0"));
ok(composeContiguous(hexU("0061 0300 031B")), hexU("00E0 031B"));
ok(composeContiguous(hexU("0061 0300 0315")), hexU("00E0 0315"));
ok(composeContiguous(hexU("0061 0300 0313")), hexU("00E0 0313"));
ok(composeContiguous(hexU("0061 031B 0300")), hexU("0061 031B 0300"));
ok(composeContiguous(hexU("0061 0315 0300")), hexU("0061 0315 0300"));
ok(composeContiguous(hexU("0061 0313 0300")), hexU("0061 0313 0300"));

# don't modify the source
my $sCtg = "\x{30DB}\x{309A}";
ok(composeContiguous($sCtg), "\x{30DD}");
ok($sCtg, "\x{30DB}\x{309A}");

sub answer { defined $_[0] ? $_[0] ? "YES" : "NO" : "MAYBE" }

ok(answer(checkNFD("")),  "YES");
ok(answer(checkNFC("")),  "YES");
ok(answer(checkNFKD("")), "YES");
ok(answer(checkNFKC("")), "YES");
ok(answer(check("NFD", "")), "YES");
ok(answer(check("NFC", "")), "YES");
ok(answer(check("NFKD","")), "YES");
ok(answer(check("NFKC","")), "YES");

# U+0000 to U+007F are prenormalized in all the normalization forms.
ok(answer(checkNFD("AZaz\t12!#`")),  "YES");
ok(answer(checkNFC("AZaz\t12!#`")),  "YES");
ok(answer(checkNFKD("AZaz\t12!#`")), "YES");
ok(answer(checkNFKC("AZaz\t12!#`")), "YES");
ok(answer(check("D", "AZaz\t12!#`")), "YES");
ok(answer(check("C", "AZaz\t12!#`")), "YES");
ok(answer(check("KD","AZaz\t12!#`")), "YES");
ok(answer(check("KC","AZaz\t12!#`")), "YES");

ok(answer(checkNFD(NFD(_pack_U(0xC1, 0x1100, 0x1173, 0x11AF)))), "YES");
ok(answer(checkNFD(_pack_U(0x20, 0xC1, 0x1100, 0x1173, 0x11AF))), "NO");
ok(answer(checkNFC(_pack_U(0x20, 0xC1, 0x1173, 0x11AF))), "MAYBE");
ok(answer(checkNFC(_pack_U(0x20, 0xC1, 0xAE00, 0x1100))), "YES");
ok(answer(checkNFC(_pack_U(0x20, 0xC1, 0xAE00, 0x1100, 0x300))), "MAYBE");
ok(answer(checkNFC(_pack_U(0x20, 0xC1, 0xFF71, 0x2025))),    "YES");
ok(answer(check("NFC", _pack_U(0x20, 0xC1, 0x212B, 0x300))), "NO");
ok(answer(checkNFKD(_pack_U(0x20, 0xC1, 0xFF71, 0x2025))),   "NO");
ok(answer(checkNFKC(_pack_U(0x20, 0xC1, 0xAE00, 0x2025))),   "NO");

"012ABC" =~ /(\d+)(\w+)/;
ok("012" eq NFC $1 && "ABC" eq NFC $2);

ok(normalize('C', $1), "012");
ok(normalize('C', $2), "ABC");

ok(normalize('NFC', $1), "012");
ok(normalize('NFC', $2), "ABC");
 # s/^NF// in normalize() must not prevent using $1, $&, etc.

# a string with initial zero should be treated like a number

# LATIN CAPITAL LETTER A WITH GRAVE
ok(getCombinClass("0192"), 0);
ok(getCanon ("0192"), _pack_U(0x41, 0x300));
ok(getCompat("0192"), _pack_U(0x41, 0x300));
ok(getComposite("065", "0768"), 192);
ok(isNFD_NO ("0192"));
ok(isNFKD_NO("0192"));

# DEVANAGARI LETTER QA
ok(isExclusion("02392"));
ok(isComp_Ex  ("02392"));
ok(isNFC_NO   ("02392"));
ok(isNFKC_NO  ("02392"));
ok(isNFD_NO   ("02392"));
ok(isNFKD_NO  ("02392"));

# ANGSTROM SIGN
ok(isSingleton("08491"));
ok(isComp_Ex  ("08491"));
ok(isNFC_NO   ("08491"));
ok(isNFKC_NO  ("08491"));
ok(isNFD_NO   ("08491"));
ok(isNFKD_NO  ("08491"));

# COMBINING GREEK DIALYTIKA TONOS
ok(isNonStDecomp("0836"));
ok(isComp_Ex    ("0836"));
ok(isNFC_NO     ("0836"));
ok(isNFKC_NO    ("0836"));
ok(isNFD_NO     ("0836"));
ok(isNFKD_NO    ("0836"));

# COMBINING GRAVE ACCENT
ok(getCombinClass("0768"), 230);
ok(isComp2nd   ("0768"));
ok(isNFC_MAYBE ("0768"));
ok(isNFKC_MAYBE("0768"));

# HANGUL SYLLABLE GA
ok(getCombinClass("044032"), 0);
ok(getCanon("044032"),  _pack_U(0x1100, 0x1161));
ok(getCompat("044032"), _pack_U(0x1100, 0x1161));
ok(getComposite("04352", "04449"), 0xAC00);

# string with 22 combining characters: (0x300..0x315)
my $str_cc22 = _pack_U(0x3041, 0x300..0x315, 0x3042);
ok(decompose($str_cc22), $str_cc22);
ok(reorder($str_cc22), $str_cc22);
ok(compose($str_cc22), $str_cc22);
ok(composeContiguous($str_cc22), $str_cc22);
ok(NFD($str_cc22), $str_cc22);
ok(NFC($str_cc22), $str_cc22);
ok(NFKD($str_cc22), $str_cc22);
ok(NFKC($str_cc22), $str_cc22);
ok(FCD($str_cc22), $str_cc22);
ok(FCC($str_cc22), $str_cc22);

# string with 40 combining characters of the same class: (0x300..0x313)x2
my $str_cc40 = _pack_U(0x3041, 0x300..0x313, 0x300..0x313, 0x3042);
ok(decompose($str_cc40), $str_cc40);
ok(reorder($str_cc40), $str_cc40);
ok(compose($str_cc40), $str_cc40);
ok(composeContiguous($str_cc40), $str_cc40);
ok(NFD($str_cc40), $str_cc40);
ok(NFC($str_cc40), $str_cc40);
ok(NFKD($str_cc40), $str_cc40);
ok(NFKC($str_cc40), $str_cc40);
ok(FCD($str_cc40), $str_cc40);
ok(FCC($str_cc40), $str_cc40);

my $precomp = hexU("304C 304E 3050 3052 3054");
my $combseq = hexU("304B 3099 304D 3099 304F 3099 3051 3099 3053 3099");
ok(decompose($precomp x 5),  $combseq x 5);
ok(decompose($precomp x 10), $combseq x 10);
ok(decompose($precomp x 20), $combseq x 20);

my $hangsyl = hexU("AC00 B098 B2E4 B77C B9C8");
my $jamoseq = hexU("1100 1161 1102 1161 1103 1161 1105 1161 1106 1161");
ok(decompose($hangsyl x 5), $jamoseq x 5);
ok(decompose($hangsyl x 10), $jamoseq x 10);
ok(decompose($hangsyl x 20), $jamoseq x 20);

my $notcomp = hexU("304B 304D 304F 3051 3053");
ok(decompose($precomp . $notcomp),     $combseq . $notcomp);
ok(decompose($precomp . $notcomp x 5), $combseq . $notcomp x 5);
ok(decompose($precomp . $notcomp x10), $combseq . $notcomp x10);
Commit	Line	Data
ac5ea531	1
4a2e806c	2	BEGIN {
1efaba7f	3	unless ("A" eq pack('U', 0x41)) {
9f1f04a1	4	print "1..0 # Unicode::Normalize " .
9f1f04a1	5	"cannot stringify a Unicode code point\n";
f027f502	6	exit 0;
4a2e806c	7	}
	8	}
	9
6c941e0c	10	BEGIN {
	11	if ($ENV{PERL_CORE}) {
	12	chdir('t') if -d 't';
9f1f04a1	13	@INC = $^O eq 'MacOS' ? qw(::lib) : qw(../lib);
6c941e0c	14	}
	15	}
	16
ac5ea531	17	#########################
	18
	19	use Test;
	20	use strict;
	21	use warnings;
fe067ad9	22	BEGIN { plan tests => 202 };
ac5ea531	23	use Unicode::Normalize qw(:all);
	24	ok(1); # If we made it this far, we're ok.
	25
fe067ad9	26	sub _pack_U { Unicode::Normalize::pack_U(@_) }
fe067ad9	27	sub hexU { _pack_U map hex, split ' ', shift }
6c941e0c	28
ac5ea531	29	#########################
ac5ea531	30
fe067ad9	31	ok(getCombinClass( 0), 0);
	32	ok(getCombinClass( 41), 0);
	33	ok(getCombinClass( 65), 0);
	34	ok(getCombinClass( 768), 230);
	35	ok(getCombinClass(1809), 36);
	36
	37	ok(getCanon( 0), undef);
	38	ok(getCanon(0x29), undef);
	39	ok(getCanon(0x41), undef);
	40	ok(getCanon(0x00C0), _pack_U(0x0041, 0x0300));
	41	ok(getCanon(0x00EF), _pack_U(0x0069, 0x0308));
	42	ok(getCanon(0x304C), _pack_U(0x304B, 0x3099));
	43	ok(getCanon(0x1EA4), _pack_U(0x0041, 0x0302, 0x0301));
	44	ok(getCanon(0x1F82), _pack_U(0x03B1, 0x0313, 0x0300, 0x0345));
	45	ok(getCanon(0x1FAF), _pack_U(0x03A9, 0x0314, 0x0342, 0x0345));
	46	ok(getCanon(0xAC00), _pack_U(0x1100, 0x1161));
	47	ok(getCanon(0xAE00), _pack_U(0x1100, 0x1173, 0x11AF));
	48	ok(getCanon(0x212C), undef);
	49	ok(getCanon(0x3243), undef);
	50	ok(getCanon(0xFA2D), _pack_U(0x9DB4));
	51
	52	ok(getCompat( 0), undef);
	53	ok(getCompat(0x29), undef);
	54	ok(getCompat(0x41), undef);
	55	ok(getCompat(0x00C0), _pack_U(0x0041, 0x0300));
	56	ok(getCompat(0x00EF), _pack_U(0x0069, 0x0308));
	57	ok(getCompat(0x304C), _pack_U(0x304B, 0x3099));
	58	ok(getCompat(0x1EA4), _pack_U(0x0041, 0x0302, 0x0301));
	59	ok(getCompat(0x1F82), _pack_U(0x03B1, 0x0313, 0x0300, 0x0345));
	60	ok(getCompat(0x1FAF), _pack_U(0x03A9, 0x0314, 0x0342, 0x0345));
	61	ok(getCompat(0x212C), _pack_U(0x0042));
	62	ok(getCompat(0x3243), _pack_U(0x0028, 0x81F3, 0x0029));
	63	ok(getCompat(0xAC00), _pack_U(0x1100, 0x1161));
	64	ok(getCompat(0xAE00), _pack_U(0x1100, 0x1173, 0x11AF));
	65	ok(getCompat(0xFA2D), _pack_U(0x9DB4));
	66
	67	ok(getComposite( 0, 0), undef);
	68	ok(getComposite( 0, 0x29), undef);
	69	ok(getComposite(0x29, 0), undef);
	70	ok(getComposite(0x29, 0x29), undef);
	71	ok(getComposite( 0, 0x41), undef);
	72	ok(getComposite(0x41, 0), undef);
	73	ok(getComposite(0x41, 0x41), undef);
	74	ok(getComposite(12, 0x0300), undef);
	75	ok(getComposite(0x0055, 0xFF00), undef);
	76	ok(getComposite(0x0041, 0x0300), 0x00C0);
	77	ok(getComposite(0x0055, 0x0300), 0x00D9);
	78	ok(getComposite(0x0112, 0x0300), 0x1E14);
	79	ok(getComposite(0x1100, 0x1161), 0xAC00);
	80	ok(getComposite(0x1100, 0x1173), 0xADF8);
	81	ok(getComposite(0x1100, 0x11AF), undef);
	82	ok(getComposite(0x1173, 0x11AF), undef);
	83	ok(getComposite(0xAC00, 0x11A7), undef);
	84	ok(getComposite(0xAC00, 0x11A8), 0xAC01);
	85	ok(getComposite(0xADF8, 0x11AF), 0xAE00);
	86
	87	sub uprops {
	88	my $uv = shift;
	89	my $r = "";
	90	$r .= isExclusion($uv) ? 'X' : 'x';
	91	$r .= isSingleton($uv) ? 'S' : 's';
	92	$r .= isNonStDecomp($uv) ? 'N' : 'n'; # Non-Starter Decomposition
	93	$r .= isComp_Ex($uv) ? 'F' : 'f'; # Full exclusion (X + S + N)
	94	$r .= isComp2nd($uv) ? 'B' : 'b'; # B = M = Y
95	$r .= isNFD_NO($uv) ? 'D' : 'd';
96	$r .= isNFC_MAYBE($uv) ? 'M' : 'm'; # Maybe
97	$r .= isNFC_NO($uv) ? 'C' : 'c';
98	$r .= isNFKD_NO($uv) ? 'K' : 'k';
99	$r .= isNFKC_MAYBE($uv) ? 'Y' : 'y'; # maYbe
100	$r .= isNFKC_NO($uv) ? 'G' : 'g';
101	return $r;
102	}
103
104	ok(uprops(0x0000), 'xsnfbdmckyg');
105	ok(uprops(0x0029), 'xsnfbdmckyg');
106	ok(uprops(0x0041), 'xsnfbdmckyg');
107	ok(uprops(0x00A0), 'xsnfbdmcKyG'); # NO-BREAK SPACE
108	ok(uprops(0x00C0), 'xsnfbDmcKyg'); # LATIN CAPITAL LETTER A WITH GRAVE
109	ok(uprops(0x0300), 'xsnfBdMckYg'); # COMBINING GRAVE ACCENT
110	ok(uprops(0x0344), 'xsNFbDmCKyG'); # COMBINING GREEK DIALYTIKA TONOS
111	ok(uprops(0x0387), 'xSnFbDmCKyG'); # GREEK ANO TELEIA
112	ok(uprops(0x0958), 'XsnFbDmCKyG'); # DEVANAGARI LETTER QA
113	ok(uprops(0x0F43), 'XsnFbDmCKyG'); # TIBETAN LETTER GHA
114	ok(uprops(0x1100), 'xsnfbdmckyg'); # HANGUL CHOSEONG KIYEOK
115	ok(uprops(0x1161), 'xsnfBdMckYg'); # HANGUL JUNGSEONG A
116	ok(uprops(0x11AF), 'xsnfBdMckYg'); # HANGUL JONGSEONG RIEU
117	ok(uprops(0x212B), 'xSnFbDmCKyG'); # ANGSTROM SIGN
118	ok(uprops(0xAC00), 'xsnfbDmcKyg'); # HANGUL SYLLABLE GA
119	ok(uprops(0xF900), 'xSnFbDmCKyG'); # CJK COMPATIBILITY IDEOGRAPH-F900
120	ok(uprops(0xFB4E), 'XsnFbDmCKyG'); # HEBREW LETTER PE WITH RAFE
121	ok(uprops(0xFF71), 'xsnfbdmcKyG'); # HALFWIDTH KATAKANA LETTER A
122
123	ok(decompose(""), "");
124	ok(decompose("A"), "A");
125	ok(decompose("", 1), "");
126	ok(decompose("A", 1), "A");
127
128	ok(decompose(hexU("1E14 AC01")), hexU("0045 0304 0300 1100 1161 11A8"));
129	ok(decompose(hexU("AC00 AE00")), hexU("1100 1161 1100 1173 11AF"));
130	ok(decompose(hexU("304C FF76")), hexU("304B 3099 FF76"));
131
132	ok(decompose(hexU("1E14 AC01"), 1), hexU("0045 0304 0300 1100 1161 11A8"));
133	ok(decompose(hexU("AC00 AE00"), 1), hexU("1100 1161 1100 1173 11AF"));
134	ok(decompose(hexU("304C FF76"), 1), hexU("304B 3099 30AB"));
135
136	# don't modify the source
137	my $sDec = "\x{FA19}";
138	ok(decompose($sDec), "\x{795E}");
139	ok($sDec, "\x{FA19}");
140
141	ok(reorder(""), "");
142	ok(reorder("A"), "A");
143	ok(reorder(hexU("0041 0300 0315 0313 031b 0061")),
144	hexU("0041 031b 0300 0313 0315 0061"));
145	ok(reorder(hexU("00C1 0300 0315 0313 031b 0061 309A 3099")),
146	hexU("00C1 031b 0300 0313 0315 0061 309A 3099"));
147
148	# don't modify the source
149	my $sReord = "\x{3000}\x{300}\x{31b}";
150	ok(reorder($sReord), "\x{3000}\x{31b}\x{300}");
151	ok($sReord, "\x{3000}\x{300}\x{31b}");
152
153	ok(compose(""), "");
154	ok(compose("A"), "A");
155	ok(compose(hexU("0061 0300")), hexU("00E0"));
156	ok(compose(hexU("0061 0300 031B")), hexU("00E0 031B"));
157	ok(compose(hexU("0061 0300 0315")), hexU("00E0 0315"));
158	ok(compose(hexU("0061 0300 0313")), hexU("00E0 0313"));
159	ok(compose(hexU("0061 031B 0300")), hexU("00E0 031B"));
160	ok(compose(hexU("0061 0315 0300")), hexU("0061 0315 0300"));
161	ok(compose(hexU("0061 0313 0300")), hexU("0061 0313 0300"));
162
163	# don't modify the source
164	my $sCom = "\x{304B}\x{3099}";
165	ok(compose($sCom), "\x{304C}");
166	ok($sCom, "\x{304B}\x{3099}");
167
168	ok(composeContiguous(""), "");
169	ok(composeContiguous("A"), "A");
170	ok(composeContiguous(hexU("0061 0300")), hexU("00E0"));
171	ok(composeContiguous(hexU("0061 0300 031B")), hexU("00E0 031B"));
172	ok(composeContiguous(hexU("0061 0300 0315")), hexU("00E0 0315"));
173	ok(composeContiguous(hexU("0061 0300 0313")), hexU("00E0 0313"));
174	ok(composeContiguous(hexU("0061 031B 0300")), hexU("0061 031B 0300"));
175	ok(composeContiguous(hexU("0061 0315 0300")), hexU("0061 0315 0300"));
176	ok(composeContiguous(hexU("0061 0313 0300")), hexU("0061 0313 0300"));
177
178	# don't modify the source
179	my $sCtg = "\x{30DB}\x{309A}";
180	ok(composeContiguous($sCtg), "\x{30DD}");
181	ok($sCtg, "\x{30DB}\x{309A}");
8f118dcd	182
	183	sub answer { defined $_[0] ? $_[0] ? "YES" : "NO" : "MAYBE" }
	184
fe067ad9	185	ok(answer(checkNFD("")), "YES");
	186	ok(answer(checkNFC("")), "YES");
	187	ok(answer(checkNFKD("")), "YES");
	188	ok(answer(checkNFKC("")), "YES");
	189	ok(answer(check("NFD", "")), "YES");
	190	ok(answer(check("NFC", "")), "YES");
	191	ok(answer(check("NFKD","")), "YES");
	192	ok(answer(check("NFKC","")), "YES");
	193
8f118dcd	194	# U+0000 to U+007F are prenormalized in all the normalization forms.
fe067ad9	195	ok(answer(checkNFD("AZaz\t12!#`")), "YES");
	196	ok(answer(checkNFC("AZaz\t12!#`")), "YES");
	197	ok(answer(checkNFKD("AZaz\t12!#`")), "YES");
	198	ok(answer(checkNFKC("AZaz\t12!#`")), "YES");
	199	ok(answer(check("D", "AZaz\t12!#`")), "YES");
	200	ok(answer(check("C", "AZaz\t12!#`")), "YES");
	201	ok(answer(check("KD","AZaz\t12!#`")), "YES");
	202	ok(answer(check("KC","AZaz\t12!#`")), "YES");
	203
	204	ok(answer(checkNFD(NFD(_pack_U(0xC1, 0x1100, 0x1173, 0x11AF)))), "YES");
	205	ok(answer(checkNFD(_pack_U(0x20, 0xC1, 0x1100, 0x1173, 0x11AF))), "NO");
	206	ok(answer(checkNFC(_pack_U(0x20, 0xC1, 0x1173, 0x11AF))), "MAYBE");
	207	ok(answer(checkNFC(_pack_U(0x20, 0xC1, 0xAE00, 0x1100))), "YES");
	208	ok(answer(checkNFC(_pack_U(0x20, 0xC1, 0xAE00, 0x1100, 0x300))), "MAYBE");
	209	ok(answer(checkNFC(_pack_U(0x20, 0xC1, 0xFF71, 0x2025))), "YES");
	210	ok(answer(check("NFC", _pack_U(0x20, 0xC1, 0x212B, 0x300))), "NO");
	211	ok(answer(checkNFKD(_pack_U(0x20, 0xC1, 0xFF71, 0x2025))), "NO");
	212	ok(answer(checkNFKC(_pack_U(0x20, 0xC1, 0xAE00, 0x2025))), "NO");
f027f502	213
f027f502	214	"012ABC" =~ /(\d+)(\w+)/;
fe067ad9	215	ok("012" eq NFC $1 && "ABC" eq NFC $2);
f027f502	216
fe067ad9	217	ok(normalize('C', $1), "012");
fe067ad9	218	ok(normalize('C', $2), "ABC");
f027f502	219
fe067ad9	220	ok(normalize('NFC', $1), "012");
fe067ad9	221	ok(normalize('NFC', $2), "ABC");
f027f502	222	# s/^NF// in normalize() must not prevent using $1, $&, etc.
f027f502	223
fe067ad9	224	# a string with initial zero should be treated like a number
	225
	226	# LATIN CAPITAL LETTER A WITH GRAVE
	227	ok(getCombinClass("0192"), 0);
	228	ok(getCanon ("0192"), _pack_U(0x41, 0x300));
	229	ok(getCompat("0192"), _pack_U(0x41, 0x300));
	230	ok(getComposite("065", "0768"), 192);
	231	ok(isNFD_NO ("0192"));
	232	ok(isNFKD_NO("0192"));
	233
	234	# DEVANAGARI LETTER QA
	235	ok(isExclusion("02392"));
	236	ok(isComp_Ex ("02392"));
	237	ok(isNFC_NO ("02392"));
	238	ok(isNFKC_NO ("02392"));
	239	ok(isNFD_NO ("02392"));
	240	ok(isNFKD_NO ("02392"));
	241
	242	# ANGSTROM SIGN
	243	ok(isSingleton("08491"));
	244	ok(isComp_Ex ("08491"));
	245	ok(isNFC_NO ("08491"));
	246	ok(isNFKC_NO ("08491"));
	247	ok(isNFD_NO ("08491"));
	248	ok(isNFKD_NO ("08491"));
	249
	250	# COMBINING GREEK DIALYTIKA TONOS
	251	ok(isNonStDecomp("0836"));
	252	ok(isComp_Ex ("0836"));
	253	ok(isNFC_NO ("0836"));
	254	ok(isNFKC_NO ("0836"));
	255	ok(isNFD_NO ("0836"));
	256	ok(isNFKD_NO ("0836"));
	257
	258	# COMBINING GRAVE ACCENT
	259	ok(getCombinClass("0768"), 230);
	260	ok(isComp2nd ("0768"));
	261	ok(isNFC_MAYBE ("0768"));
	262	ok(isNFKC_MAYBE("0768"));
	263
	264	# HANGUL SYLLABLE GA
	265	ok(getCombinClass("044032"), 0);
	266	ok(getCanon("044032"), _pack_U(0x1100, 0x1161));
	267	ok(getCompat("044032"), _pack_U(0x1100, 0x1161));
	268	ok(getComposite("04352", "04449"), 0xAC00);
	269
	270	# string with 22 combining characters: (0x300..0x315)
	271	my $str_cc22 = _pack_U(0x3041, 0x300..0x315, 0x3042);
	272	ok(decompose($str_cc22), $str_cc22);
	273	ok(reorder($str_cc22), $str_cc22);
	274	ok(compose($str_cc22), $str_cc22);
	275	ok(composeContiguous($str_cc22), $str_cc22);
	276	ok(NFD($str_cc22), $str_cc22);
	277	ok(NFC($str_cc22), $str_cc22);
	278	ok(NFKD($str_cc22), $str_cc22);
	279	ok(NFKC($str_cc22), $str_cc22);
	280	ok(FCD($str_cc22), $str_cc22);
	281	ok(FCC($str_cc22), $str_cc22);
	282
	283	# string with 40 combining characters of the same class: (0x300..0x313)x2
	284	my $str_cc40 = _pack_U(0x3041, 0x300..0x313, 0x300..0x313, 0x3042);
	285	ok(decompose($str_cc40), $str_cc40);
	286	ok(reorder($str_cc40), $str_cc40);
	287	ok(compose($str_cc40), $str_cc40);
288	ok(composeContiguous($str_cc40), $str_cc40);
289	ok(NFD($str_cc40), $str_cc40);
290	ok(NFC($str_cc40), $str_cc40);
291	ok(NFKD($str_cc40), $str_cc40);
292	ok(NFKC($str_cc40), $str_cc40);
293	ok(FCD($str_cc40), $str_cc40);
294	ok(FCC($str_cc40), $str_cc40);
295
296	my $precomp = hexU("304C 304E 3050 3052 3054");
297	my $combseq = hexU("304B 3099 304D 3099 304F 3099 3051 3099 3053 3099");
298	ok(decompose($precomp x 5), $combseq x 5);
299	ok(decompose($precomp x 10), $combseq x 10);
300	ok(decompose($precomp x 20), $combseq x 20);
301
302	my $hangsyl = hexU("AC00 B098 B2E4 B77C B9C8");
303	my $jamoseq = hexU("1100 1161 1102 1161 1103 1161 1105 1161 1106 1161");
304	ok(decompose($hangsyl x 5), $jamoseq x 5);
305	ok(decompose($hangsyl x 10), $jamoseq x 10);
306	ok(decompose($hangsyl x 20), $jamoseq x 20);
307
308	my $notcomp = hexU("304B 304D 304F 3051 3053");
309	ok(decompose($precomp . $notcomp), $combseq . $notcomp);
310	ok(decompose($precomp . $notcomp x 5), $combseq . $notcomp x 5);
311	ok(decompose($precomp . $notcomp x10), $combseq . $notcomp x10);
312
313