3 # This script generates "unfcan.h", "unfcpt.h", "unfcmb.h",
4 # "unfcmp.h", and "unfexc.h"
5 # from CombiningClass.pl, Decomposition.pl, CompositionExclusions.txt
6 # in lib/unicore or unicode directory
7 # for Unicode::Normalize.xs. (cf. Makefile.PL)
9 # Usage: <perl mkheader> in command line
10 # or <do 'mkheader'> in perl
18 our $IsEBCDIC = ord("A") != 0x41;
20 our $PACKAGE = 'Unicode::Normalize, mkheader';
22 our $Combin = do "unicore/CombiningClass.pl"
23 || do "unicode/CombiningClass.pl"
24 || croak "$PACKAGE: CombiningClass.pl not found";
26 our $Decomp = do "unicore/Decomposition.pl"
27 || do "unicode/Decomposition.pl"
28 || croak "$PACKAGE: Decomposition.pl not found";
30 our %Combin; # $codepoint => $number : combination class
31 our %Canon; # $codepoint => \@codepoints : canonical decomp.
32 our %Compat; # $codepoint => \@codepoints : compat. decomp.
33 # after _U_stringify(), ($codepoint => $hexstring) for %Canon and %Compat
34 our %Exclus; # $codepoint => 1 : composition exclusions
35 our %Single; # $codepoint => 1 : singletons
36 our %NonStD; # $codepoint => 1 : non-starter decompositions
38 our %Comp1st; # $codepoint => $listname : may be composed with a next char.
39 our %Comp2nd; # $codepoint => 1 : may be composed with a prev char.
40 our %CompList; # $listname,$2nd => $codepoint : composite
43 our $structname = "${prefix}complist";
45 ########## definition of Hangul constants ##########
46 use constant SBase => 0xAC00;
47 use constant SFinal => 0xD7A3; # SBase -1 + SCount
48 use constant SCount => 11172; # LCount * NCount
49 use constant NCount => 588; # VCount * TCount
50 use constant LBase => 0x1100;
51 use constant LFinal => 0x1112;
52 use constant LCount => 19;
53 use constant VBase => 0x1161;
54 use constant VFinal => 0x1175;
55 use constant VCount => 21;
56 use constant TBase => 0x11A7;
57 use constant TFinal => 0x11C2;
58 use constant TCount => 28;
61 my $SIndex = $_[0] - SBase;
62 my $LIndex = int( $SIndex / NCount);
63 my $VIndex = int(($SIndex % NCount) / TCount);
64 my $TIndex = $SIndex % TCount;
68 $TIndex ? (TBase + $TIndex) : (),
70 wantarray ? @ret : pack('U*', @ret);
71 # any element in @ret greater than 0xFF, so no need of u2n conversion.
74 ########## getting full decomposion ##########
77 foreach my $d (@INC) {
78 $f = File::Spec->catfile($d, "unicore", "CompositionExclusions.txt");
79 last if open($fh, $f);
80 $f = File::Spec->catfile($d, "unicode", "CompExcl.txt");
81 last if open($fh, $f);
84 croak "$PACKAGE: neither unicore/CompositionExclusions.txt "
85 . "nor unicode/CompExcl.txt is found in @INC" unless defined $f;
90 $Exclus{ hex($1) } = 1 if /([0-9A-Fa-f]+)/;
96 ## converts string "hhhh hhhh hhhh" to a numeric list
98 sub _getHexArray { map hex, $_[0] =~ /([0-9A-Fa-f]+)/g }
100 while ($Combin =~ /(.+)/g) {
101 my @tab = split /\t/, $1;
102 my $ini = hex $tab[0];
104 $Combin{ $ini } = $tab[2];
106 $Combin{ $_ } = $tab[2] foreach $ini .. hex($tab[1]);
110 while ($Decomp =~ /(.+)/g) {
111 my @tab = split /\t/, $1;
112 my $compat = $tab[2] =~ s/<[^>]+>//;
113 my $dec = [ _getHexArray($tab[2]) ]; # decomposition
114 my $ini = hex($tab[0]); # initial decomposable character
117 @$dec == 2 ? sprintf("${structname}_%06x", $dec->[0]) : 'USELESS';
118 # %04x is bad since it'd place _3046 after _1d157.
121 $Compat{ $ini } = $dec;
124 $Canon{ $ini } = $dec;
127 if ($Combin{ $dec->[0] }) {
130 $CompList{ $listname }{ $dec->[1] } = $ini;
131 $Comp1st{ $dec->[0] } = $listname;
132 $Comp2nd{ $dec->[1] } = 1 if ! $Exclus{$ini};
134 } elsif (@$dec == 1) {
137 croak("Weird Canonical Decomposition of U+$tab[0]");
141 foreach my $u ($ini .. hex($tab[1])) {
142 $Compat{ $u } = $dec;
148 if ($Combin{ $dec->[0] }) {
151 $CompList{ $listname }{ $dec->[1] } = $u;
152 $Comp1st{ $dec->[0] } = $listname;
153 $Comp2nd{ $dec->[1] } = 1 if ! $Exclus{$u};
155 } elsif (@$dec == 1) {
158 croak("Weird Canonical Decomposition of U+$tab[0]");
165 # modern HANGUL JUNGSEONG and HANGUL JONGSEONG jamo
166 foreach my $j (0x1161..0x1175, 0x11A8..0x11C2) {
173 (SBase <= $_ && $_ <= SFinal) ? decomposeHangul($_)
174 : $Canon{$_} ? @{ $Canon{$_} } : $_
176 return join(" ",@src) eq join(" ",@dec) ? @dec : getCanonList(@dec);
177 # condition @src == @dec is not ok.
183 (SBase <= $_ && $_ <= SFinal) ? decomposeHangul($_)
184 : $Compat{$_} ? @{ $Compat{$_} } : $_
186 return join(" ",@src) eq join(" ",@dec) ? @dec : getCompatList(@dec);
187 # condition @src == @dec is not ok.
190 # exhaustive decomposition
191 foreach my $key (keys %Canon) {
192 $Canon{$key} = [ getCanonList($key) ];
195 # exhaustive decomposition
196 foreach my $key (keys %Compat) {
197 $Compat{$key} = [ getCompatList($key) ];
201 sprintf '"%s"', join '',
202 map sprintf("\\x%02x", $_), unpack 'C*',
204 ? pack('U*', map utf8::unicode_to_native($_), @_)
208 foreach my $hash (\%Canon, \%Compat) {
209 foreach my $key (keys %$hash) {
210 $hash->{$key} = _U_stringify( @{ $hash->{$key} } );
214 ########## writing header files ##########
228 name => "NonStDecomp",
239 my $file = "unfexc.h";
240 open FH, ">$file" or croak "$PACKAGE: $file can't be made";
241 binmode FH; select FH;
245 * This file is auto-generated by mkheader.
246 * Any changes here will be lost!
250 foreach my $tbl (@boolfunc) {
251 my @temp = sort {$a <=> $b} keys %{$tbl->{hash}};
252 my $type = $tbl->{type};
253 my $name = $tbl->{name};
254 print "$type is$name (UV uv)\n{\nreturn\n\t";
257 my $cur = shift @temp;
258 if (@temp && $cur + 1 == $temp[0]) {
259 print "($cur <= uv && uv <= ";
260 while (@temp && $cur + 1 == $temp[0]) {
264 print "\n\t|| " if @temp;
267 print "\n\t|| " if @temp;
270 print "\n\t? TRUE : FALSE;\n}\n\n";
275 ####################################
278 "typedef struct { UV nextchar; UV composite; } $structname;\n\n";
280 foreach my $i (sort keys %CompList) {
281 $compinit .= "$structname $i [] = {\n";
282 $compinit .= join ",\n",
283 map sprintf("\t{ %d, %d }", $_, $CompList{$i}{$_}),
284 sort {$a <=> $b } keys %{ $CompList{$i} };
285 $compinit .= ",\n{0,0}\n};\n\n"; # with sentinel
313 type => "$structname *",
320 foreach my $tbl (@tripletable) {
321 my $file = "$tbl->{file}.h";
322 my $head = "${prefix}$tbl->{name}";
323 my $type = $tbl->{type};
324 my $hash = $tbl->{hash};
325 my $null = $tbl->{null};
326 my $init = $tbl->{init};
328 open FH, ">$file" or croak "$PACKAGE: $file can't be made";
329 binmode FH; select FH;
334 * This file is auto-generated by mkheader.
335 * Any changes here will be lost!
339 print $init if defined $init;
341 foreach my $uv (keys %$hash) {
342 croak sprintf("a Unicode code point 0x%04X over 0x10FFFF.", $uv)
343 unless $uv <= 0x10FFFF;
344 my @c = unpack 'CCCC', pack 'N', $uv;
345 $val{ $c[1] }{ $c[2] }{ $c[3] } = $hash->{$uv};
348 foreach my $p (sort { $a <=> $b } keys %val) {
349 next if ! $val{ $p };
350 for (my $r = 0; $r < 256; $r++) {
351 next if ! $val{ $p }{ $r };
352 printf "$type ${head}_%02x_%02x [256] = {\n", $p, $r;
353 for (my $c = 0; $c < 256; $c++) {
354 print "\t", defined $val{$p}{$r}{$c}
355 ? "($type)".$val{$p}{$r}{$c}
357 print ',' if $c != 255;
358 print "\n" if $c % 8 == 7;
363 foreach my $p (sort { $a <=> $b } keys %val) {
364 next if ! $val{ $p };
365 printf "$type* ${head}_%02x [256] = {\n", $p;
366 for (my $r = 0; $r < 256; $r++) {
367 print $val{ $p }{ $r }
368 ? sprintf("${head}_%02x_%02x", $p, $r)
370 print ',' if $r != 255;
371 print "\n" if $val{ $p }{ $r } || ($r+1) % 8 == 0;
375 print "$type** $head [] = {\n";
376 for (my $p = 0; $p <= 0x10; $p++) {
377 print $val{ $p } ? sprintf("${head}_%02x", $p) : "NULL";
378 print ',' if $p != 0x10;