5 $UnicodeData = "Unicode.txt";
6 $SyllableData = "syllables.txt";
7 $PropData = "PropList.txt";
10 # Note: we try to keep filenames unique within first 8 chars. Using
11 # subdirectories for the following helps.
19 # 005F: SPACING UNDERSCROE
20 ['IsWord', '$cat =~ /^[LMN]/ or $code eq "005F"', ''],
21 ['IsAlnum', '$cat =~ /^[LMN]/', ''],
22 ['IsAlpha', '$cat =~ /^[LM]/', ''],
23 # 0009: HORIZONTAL TABULATION
25 # 000B: VERTICAL TABULATION
27 # 000D: CARRIAGE RETURN
29 ['IsSpace', '$cat =~ /^Z/ ||
30 $code =~ /^(0009|000A|000B|000C|000D)$/', ''],
33 $code =~ /^(0009|000A|000C|000D)$/', ''],
34 ['IsBlank', '$code =~ /^(0020|0009)$/ ||
35 $cat =~ /^Z[^lp]$/', ''],
36 ['IsDigit', '$cat =~ /^Nd$/', ''],
37 ['IsUpper', '$cat =~ /^L[ut]$/', ''],
38 ['IsLower', '$cat =~ /^Ll$/', ''],
39 ['IsASCII', '$code le "007f"', ''],
40 ['IsCntrl', '$cat =~ /^C/', ''],
41 ['IsGraph', '$cat =~ /^([LMNPS]|Co)/', ''],
42 ['IsPrint', '$cat =~ /^([LMNPS]|Co|Zs)/', ''],
43 ['IsPunct', '$cat =~ /^P/', ''],
44 # 003[0-9]: DIGIT ZERO..NINE, 00[46][1-6]: A..F, a..f
45 ['IsXDigit', '$code =~ /^00(3[0-9]|[46][1-6])$/', ''],
46 ['ToUpper', '$up', '$up'],
47 ['ToLower', '$down', '$down'],
48 ['ToTitle', '$title', '$title'],
49 ['ToDigit', '$dec ne ""', '$dec'],
53 ['Name', '$name', '$name'],
57 ['Category', '$cat', '$cat'],
61 ['IsM', '$cat =~ /^M/', ''], # Mark
62 ['IsMn', '$cat eq "Mn"', ''], # Mark, Non-Spacing
63 ['IsMc', '$cat eq "Mc"', ''], # Mark, Combining
64 ['IsMe', '$cat eq "Me"', ''], # Mark, Enclosing
66 ['IsN', '$cat =~ /^N/', ''], # Number
67 ['IsNd', '$cat eq "Nd"', ''], # Number, Decimal Digit
68 ['IsNo', '$cat eq "No"', ''], # Number, Other
69 ['IsNl', '$cat eq "Nl"', ''], # Number, Letter
71 ['IsZ', '$cat =~ /^Z/', ''], # Separator
72 ['IsZs', '$cat eq "Zs"', ''], # Separator, Space
73 ['IsZl', '$cat eq "Zl"', ''], # Separator, Line
74 ['IsZp', '$cat eq "Zp"', ''], # Separator, Paragraph
76 ['IsC', '$cat =~ /^C/', ''], # Crazy
77 ['IsCc', '$cat eq "Cc"', ''], # Other, Control or Format
78 ['IsCo', '$cat eq "Co"', ''], # Other, Private Use
79 ['IsCn', '$cat eq "Cn"', ''], # Other, Not Assigned
80 ['IsCf', '$cat eq "Cf"', ''], # Other, Format
81 ['IsCs', '$cat eq "Cs"', ''], # Other, Surrogate
82 ['IsCn', 'Unassigned Code Value',$PropData], # Other, Not Assigned
86 ['IsL', '$cat =~ /^L/', ''], # Letter
87 ['IsLu', '$cat eq "Lu"', ''], # Letter, Uppercase
88 ['IsLl', '$cat eq "Ll"', ''], # Letter, Lowercase
89 ['IsLt', '$cat eq "Lt"', ''], # Letter, Titlecase
90 ['IsLm', '$cat eq "Lm"', ''], # Letter, Modifier
91 ['IsLo', '$cat eq "Lo"', ''], # Letter, Other
93 ['IsP', '$cat =~ /^P/', ''], # Punctuation
94 ['IsPd', '$cat eq "Pd"', ''], # Punctuation, Dash
95 ['IsPs', '$cat eq "Ps"', ''], # Punctuation, Open
96 ['IsPe', '$cat eq "Pe"', ''], # Punctuation, Close
97 ['IsPo', '$cat eq "Po"', ''], # Punctuation, Other
98 ['IsPc', '$cat eq "Pc"', ''], # Punctuation, Connector
99 ['IsPi', '$cat eq "Pi"', ''], # Punctuation, Initial quote
100 ['IsPf', '$cat eq "Pf"', ''], # Punctuation, Final quote
102 ['IsS', '$cat =~ /^S/', ''], # Symbol
103 ['IsSm', '$cat eq "Sm"', ''], # Symbol, Math
104 ['IsSk', '$cat eq "Sk"', ''], # Symbol, Modifier
105 ['IsSc', '$cat eq "Sc"', ''], # Symbol, Currency
106 ['IsSo', '$cat eq "So"', ''], # Symbol, Other
109 ['CombiningClass', '$comb', '$comb'],
111 # BIDIRECTIONAL PROPERTIES
113 ['Bidirectional', '$bid', '$bid'],
117 ['IsBidiL', '$bid eq "L"', ''], # Left-Right; Most alphabetic,
118 # syllabic, and logographic
119 # characters (e.g., CJK
121 ['IsBidiR', '$bid eq "R"', ''], # Right-Left; Arabic, Hebrew,
122 # and punctuation specific to
125 ['IsBidiLRE', '$bid eq "LRE"', ''], # Left-to-Right Embedding
126 ['IsBidiLRO', '$bid eq "LRO"', ''], # Left-to-Right Override
127 ['IsBidiAL', '$bid eq "AL"', ''], # Right-to-Left Arabic
128 ['IsBidiRLE', '$bid eq "RLE"', ''], # Right-to-Left Embedding
129 ['IsBidiRLO', '$bid eq "RLO"', ''], # Right-to-Left Override
130 ['IsBidiPDF', '$bid eq "PDF"', ''], # Pop Directional Format
131 ['IsBidiNSM', '$bid eq "NSM"', ''], # Non-Spacing Mark
132 ['IsBidiBN', '$bid eq "BN"', ''], # Boundary Neutral
136 ['IsBidiEN','$bid eq "EN"', ''], # European Number
137 ['IsBidiES','$bid eq "ES"', ''], # European Number Separator
138 ['IsBidiET','$bid eq "ET"', ''], # European Number Terminator
139 ['IsBidiAN','$bid eq "AN"', ''], # Arabic Number
140 ['IsBidiCS','$bid eq "CS"', ''], # Common Number Separator
144 ['IsBidiB', '$bid eq "B"', ''], # Block Separator
145 ['IsBidiS', '$bid eq "S"', ''], # Segment Separator
149 ['IsBidiWS','$bid eq "WS"', ''], # Whitespace
150 ['IsBidiON','$bid eq "ON"', ''], # Other Neutrals ; All other
151 # characters: punctuation,
156 ['Decomposition', '$decomp', '$decomp'],
157 ['IsDecoCanon', '$decomp && $decomp !~ /^</', ''],
158 ['IsDecoCompat', '$decomp =~ /^</', ''],
159 ['IsDCfont', '$decomp =~ /^<font>/', ''],
160 ['IsDCnoBreak', '$decomp =~ /^<noBreak>/', ''],
161 ['IsDCinitial', '$decomp =~ /^<initial>/', ''],
162 ['IsDCmedial', '$decomp =~ /^<medial>/', ''],
163 ['IsDCfinal', '$decomp =~ /^<final>/', ''],
164 ['IsDCisolated', '$decomp =~ /^<isolated>/', ''],
165 ['IsDCcircle', '$decomp =~ /^<circle>/', ''],
166 ['IsDCsuper', '$decomp =~ /^<super>/', ''],
167 ['IsDCsub', '$decomp =~ /^<sub>/', ''],
168 ['IsDCvertical', '$decomp =~ /^<vertical>/', ''],
169 ['IsDCwide', '$decomp =~ /^<wide>/', ''],
170 ['IsDCnarrow', '$decomp =~ /^<narrow>/', ''],
171 ['IsDCsmall', '$decomp =~ /^<small>/', ''],
172 ['IsDCsquare', '$decomp =~ /^<square>/', ''],
173 ['IsDCfraction', '$decomp =~ /^<fraction>/', ''],
174 ['IsDCcompat', '$decomp =~ /^<compat>/', ''],
178 ['Number', '$num ne ""', '$num'],
182 ['IsMirrored', '$mir eq "Y"', ''],
186 ['ArabLink', '1', '$link'],
187 ['ArabLnkGrp', '1', '$linkgroup'],
191 ['JamoShort', '1', '$short'],
197 # Line break properties - Normative
199 ['IsLbrkBK','$brk eq "BK"', ''], # Mandatory Break
200 ['IsLbrkCR','$brk eq "CR"', ''], # Carriage Return
201 ['IsLbrkLF','$brk eq "LF"', ''], # Line Feed
202 ['IsLbrkCM','$brk eq "CM"', ''], # Attached Characters and Combining Marks
203 ['IsLbrkSG','$brk eq "SG"', ''], # Surrogates
204 ['IsLbrkGL','$brk eq "GL"', ''], # Non-breaking (Glue)
205 ['IsLbrkCB','$brk eq "CB"', ''], # Contingent Break Opportunity
206 ['IsLbrkSP','$brk eq "SP"', ''], # Space
207 ['IsLbrkZW','$brk eq "ZW"', ''], # Zero Width Space
209 # Line break properties - Informative
210 ['IsLbrkXX','$brk eq "XX"', ''], # Unknown
211 ['IsLbrkOP','$brk eq "OP"', ''], # Opening Punctuation
212 ['IsLbrkCL','$brk eq "CL"', ''], # Closing Punctuation
213 ['IsLbrkQU','$brk eq "QU"', ''], # Ambiguous Quotation
214 ['IsLbrkNS','$brk eq "NS"', ''], # Non Starter
215 ['IsLbrkEX','$brk eq "EX"', ''], # Exclamation/Interrogation
216 ['IsLbrkSY','$brk eq "SY"', ''], # Symbols Allowing Breaks
217 ['IsLbrkIS','$brk eq "IS"', ''], # Infix Separator (Numeric)
218 ['IsLbrkPR','$brk eq "PR"', ''], # Prefix (Numeric)
219 ['IsLbrkPO','$brk eq "PO"', ''], # Postfix (Numeric)
220 ['IsLbrkNU','$brk eq "NU"', ''], # Numeric
221 ['IsLbrkAL','$brk eq "AL"', ''], # Ordinary Alphabetic and Symbol Characters
222 ['IsLbrkID','$brk eq "ID"', ''], # Ideographic
223 ['IsLbrkIN','$brk eq "IN"', ''], # Inseparable
224 ['IsLbrkHY','$brk eq "HY"', ''], # Hyphen
225 ['IsLbrkBB','$brk eq "BB"', ''], # Break Opportunity Before
226 ['IsLbrkBA','$brk eq "BA"', ''], # Break Opportunity After
227 ['IsLbrkSA','$brk eq "SA"', ''], # Complex Context (South East Asian)
228 ['IsLbrkAI','$brk eq "AI"', ''], # Ambiguous (Alphabetic or Ideographic)
229 ['IsLbrkB2','$brk eq "B2"', ''], # Break Opportunity Before and After
232 # This is not written for speed...
238 foreach $file (@todo) {
239 my ($table, $wanted, $val) = @$file;
240 next if @ARGV and not grep { $_ eq $table } @ARGV;
243 if ($table =~ /^(Is|To)(.+)/) {
244 open(OUT, ">$1/$2.pl") or die "Can't create $1/$2.pl: $!\n";
247 open(OUT, ">$table.pl") or die "Can't create $table.pl: $!\n";
250 # !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
251 # This file is built by $0 from e.g. $UnicodeData.
252 # Any changes made here will be lost!
257 print OUT proplist($table, $wanted, $val);
263 open(UD, 'Scripts.txt') or die "Can't open Scripts.txt: $!\n";
264 open(OUT, ">Scripts.pl") or die "Can't create Scripts.pl: $!\n";
266 # !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
267 # This file is built by $0 from e.g. $UnicodeData.
268 # Any changes made here will be lost!
282 ($code, $last, $name) = /^([0-9a-f]+)(?:\.\.([0-9a-f]+))?\s+;\s+(.+)\s+\#/i;
286 unless (exists $InIdScript{$InName}) {
288 $id = $Scripts{$InName} = $InIdScript{$InName} = $InId++;
289 open(SCRIPT, ">In/$id.pl") or die "create In/$id.pl: $!\n";
291 # !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
292 # This file is built by $0 from e.g. $UnicodeData.
293 # Any changes made here will be lost!
298 $id = $InIdScript{$InName};
300 $last = "" unless defined $last;
301 print OUT "$code\t$last\t$name\t# In/$id.pl\n";
302 open(SCRIPT, ">>In/$id.pl");
308 my $firsti = hex($code);
309 my $lasti = $last ? hex($last) : $firsti;
310 for my $i ($firsti..$lasti) {
311 vec($ScriptsVec, $i, 1) = 1;
313 $lastlast = $lasti if $lasti > $lastlast;
314 print "\t\t$code..$last\n";
317 for my $id (values %InIdScript) {
318 open(SCRIPT, ">>In/$id.pl");
328 # Must treat blocks specially.
330 exit if @ARGV and not grep { $_ eq Block } @ARGV;
332 open(UD, 'Blocks.txt') or die "Can't open Blocks.txt: $!\n";
333 open(OUT, ">Blocks.pl") or die "Can't create Blocks.pl: $!\n";
335 # !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
336 # This file is built by $0 from e.g. $UnicodeData.
337 # Any changes made here will be lost!
347 ($code, $last, $name) = /^([0-9a-f]+)\.\.([0-9a-f]+); (.+?)\s*$/i;
352 # TODO: only the first one of Private Use blocks qualifies
353 unless (exists $InIdBlock{$InName}) {
354 $InIdBlock{$InName} = $InId++;
356 $id = $InIdBlock{$InName};
357 open(BLOCK, ">In/$id.pl") or die "create In/$id.pl: $!\n";
358 print OUT "$code\t$last\t$name\t# In/$id.pl\n";
360 # !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
361 # This file is built by $0 from e.g. $UnicodeData.
362 # Any changes made here will be lost!
364 print BLOCK <<"END2";
377 my $CommonId = $Scripts{Common} = $InIdScript{Common} = $InId++;
378 open(SCRIPT, ">In/$CommonId.pl") or die "create In/$CommonId.pl: $!\n";
380 # !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
381 # This file is built by $0 from e.g. $UnicodeData.
382 # Any changes made here will be lost!
388 if (defined $first) {
390 $last = $last == $first ? "" : sprintf("%04X", $last);
391 printf SCRIPT "%04X\t$last\tCommon\t# In/$CommonId.pl\n", $first;
392 printf "\t\t%04X..$last\n", $first;
396 for my $i (0..$lastlast) {
397 if (vec($ScriptsVec, $i, 1)) {
398 defined $first && flushzerorange($i);
400 $first = $i unless defined $first;
403 flushzerorange($lastlast+1);
404 print SCRIPT "END\n";
407 open(INID, ">In.pl");
410 # !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
411 # This file is built by $0 from e.g. $UnicodeData.
412 # Any changes made here will be lost!
416 my %InIdScriptById = reverse %InIdScript;
417 my %InIdBlockById = reverse %InIdBlock;
419 my @InIdScriptById = sort { $a <=> $b } keys %InIdScriptById;
420 my @InIdBlockById = sort { $a <=> $b } keys %InIdBlockById;
425 for my $id (@InIdScriptById) {
426 my $name = $InIdScriptById{$id};
427 my $lcname = lc($name);
429 $IdIdLcName{$lcname} = $id;
432 for my $id (@InIdBlockById) {
433 my $name = $InIdBlockById{$id};
434 my $lcname = lc($name);
435 if (exists $IdIdLcName{$lcname}) {
436 $InId{"$name Block"} = $id;
440 $IdIdLcName{$lcname} = $id;
443 my @InId = sort { $InId{$a} <=> $InId{$b} } keys %InId;
447 foreach my $in (@InId) {
449 $inpat =~ s/([- _])/(?:[-_]|\\s+)?/g;
450 push @{$InIdPrefix{lc(substr($in, 0, 3))}}, [ $in, $inpat ];
451 printf INID "%-45s => %3d,\n", "'$in'", $InId{$in};
460 foreach my $prefix (sort keys %InIdPrefix) {
461 printf INID "'$prefix' => {\n";
462 foreach my $ininpat (@{$InIdPrefix{$prefix}}) {
463 my ($in, $inpat) = @$ininpat;
464 printf INID "\t'$inpat' => '$in',\n";
473 ##################################################
476 my ($table, $wanted, $val) = @_;
481 return listFromPropFile($wanted) if $val eq $PropData;
483 if ($table =~ /^Arab/) {
484 open(UD, "ArabShap.txt") or warn "Can't open $table: $!";
486 $split = '($code, $name, $link, $linkgroup) = split(/; */);';
488 elsif ($table =~ /^Jamo/) {
489 open(UD, "Jamo.txt") or warn "Can't open $table: $!";
491 $split = '($code, $short, $name) = split(/; */); $code =~ s/^U\+//;';
493 elsif ($table =~ /^IsSyl/) {
494 open(UD, $SyllableData) or warn "Can't open $table: $!";
496 $split = '($code, $short, $syl) = split(/; */); $code =~ s/^U\+//;';
498 elsif ($table =~ /^IsLbrk/) {
499 open(UD, "LineBrk.txt") or warn "Can't open $table: $!";
501 $split = '($code, $brk, $name) = /^([0-9a-f]+);(\w+) # (.+)/i;';
504 open(UD, $UnicodeData) or warn "Can't open $UnicodeData: $!";
506 $split = '($code, $name, $cat, $comb, $bid, $decomp, $dec, $dig, $num, $mir, $uni1,
507 $comment, $up, $down, $title) = split(/;/);';
510 if ($table =~ /^(?:To|Is)[A-Z]/) {
518 push(\@wanted, [hex \$code, hex $val, \$name =~ /, First>\$/]);
525 $beg = shift @wanted;
527 while (@wanted and $wanted[0]->[0] == $last->[0] + 1 and
528 (not $val or $wanted[0]->[1] == $last->[1] + 1)) {
529 $last = shift @wanted;
531 $out .= sprintf "%04x", $beg->[0];
533 $last = shift @wanted;
539 $out .= sprintf "\t%04x", $last->[0];
541 $out .= sprintf "\t%04x", $beg->[1] if $val;
553 push(\@wanted, [hex \$code, $val, \$name =~ /, First>\$/]);
560 $beg = shift @wanted;
562 while (@wanted and $wanted[0]->[0] == $last->[0] + 1 and
563 ($wanted[0]->[1] eq $last->[1])) {
564 $last = shift @wanted;
566 $out .= sprintf "%04x", $beg->[0];
568 $last = shift @wanted;
574 $out .= sprintf "\t%04x", $last->[0];
576 $out .= sprintf "\t%s\n", $beg->[1];
582 sub listFromPropFile {
586 open (UD, $PropData) or die "Can't open $PropData: $!\n";
587 local($/) = "\n" . '*' x 43 . "\n\nProperty dump for:"; # not 42?
592 if (s/0x[\d\w]+\s+\((.*?)\)// and $wanted eq $1) {
609 open (SD, $SyllableData) or die "Can't open $SyllableData: $!\n";
613 ($code, $name, $syl) = split /; */;
615 push (@defs, ["IsSyl$syl", qq{\$syl eq "$syl"}, ''])
616 unless $seen{$syl}++;