5 $UnicodeData = "Unicode.txt";
6 $SyllableData = "syllables.txt";
7 $PropData = "PropList.txt";
10 # Note: we try to keep filenames unique within first 8 chars. Using
11 # subdirectories for the following helps.
19 # 005F: SPACING UNDERSCROE
20 ['IsWord', '$cat =~ /^[LMN]/ or $code eq "005F"', ''],
21 ['IsAlnum', '$cat =~ /^[LMN]/', ''],
22 ['IsAlpha', '$cat =~ /^[LM]/', ''],
23 # 0009: HORIZONTAL TABULATION
25 # 000B: VERTICAL TABULATION
27 # 000D: CARRIAGE RETURN
29 ['IsSpace', '$cat =~ /^Z/ ||
30 $code =~ /^(0009|000A|000B|000C|000D)$/', ''],
33 $code =~ /^(0009|000A|000C|000D)$/', ''],
34 ['IsBlank', '$code =~ /^(0020|0009)$/ ||
35 $cat =~ /^Z[^lp]$/', ''],
36 ['IsDigit', '$cat =~ /^Nd$/', ''],
37 ['IsUpper', '$cat =~ /^L[ut]$/', ''],
38 ['IsLower', '$cat =~ /^Ll$/', ''],
39 ['IsASCII', '$code le "007f"', ''],
40 ['IsCntrl', '$cat =~ /^C/', ''],
41 ['IsGraph', '$cat =~ /^([LMNPS]|Co)/', ''],
42 ['IsPrint', '$cat =~ /^([LMNPS]|Co|Zs)/', ''],
43 ['IsPunct', '$cat =~ /^P/', ''],
44 # 003[0-9]: DIGIT ZERO..NINE, 00[46][1-6]: A..F, a..f
45 ['IsXDigit', '$code =~ /^00(3[0-9]|[46][1-6])$/', ''],
46 ['ToUpper', '$up', '$up'],
47 ['ToLower', '$down', '$down'],
48 ['ToTitle', '$title', '$title'],
49 ['ToDigit', '$dec ne ""', '$dec'],
53 ['Name', '$name', '$name'],
57 ['Category', '$cat', '$cat'],
61 ['IsM', '$cat =~ /^M/', ''], # Mark
62 ['IsMn', '$cat eq "Mn"', ''], # Mark, Non-Spacing
63 ['IsMc', '$cat eq "Mc"', ''], # Mark, Combining
64 ['IsMe', '$cat eq "Me"', ''], # Mark, Enclosing
66 ['IsN', '$cat =~ /^N/', ''], # Number
67 ['IsNd', '$cat eq "Nd"', ''], # Number, Decimal Digit
68 ['IsNo', '$cat eq "No"', ''], # Number, Other
69 ['IsNl', '$cat eq "Nl"', ''], # Number, Letter
71 ['IsZ', '$cat =~ /^Z/', ''], # Separator
72 ['IsZs', '$cat eq "Zs"', ''], # Separator, Space
73 ['IsZl', '$cat eq "Zl"', ''], # Separator, Line
74 ['IsZp', '$cat eq "Zp"', ''], # Separator, Paragraph
76 ['IsC', '$cat =~ /^C/', ''], # Crazy
77 ['IsCc', '$cat eq "Cc"', ''], # Other, Control or Format
78 ['IsCo', '$cat eq "Co"', ''], # Other, Private Use
79 ['IsCn', '$cat eq "Cn"', ''], # Other, Not Assigned
80 ['IsCf', '$cat eq "Cf"', ''], # Other, Format
81 ['IsCs', '$cat eq "Cs"', ''], # Other, Surrogate
82 ['IsCn', 'Unassigned Code Value',$PropData], # Other, Not Assigned
86 ['IsL', '$cat =~ /^L/', ''], # Letter
87 ['IsLu', '$cat eq "Lu"', ''], # Letter, Uppercase
88 ['IsLl', '$cat eq "Ll"', ''], # Letter, Lowercase
89 ['IsLt', '$cat eq "Lt"', ''], # Letter, Titlecase
90 ['IsLm', '$cat eq "Lm"', ''], # Letter, Modifier
91 ['IsLo', '$cat eq "Lo"', ''], # Letter, Other
93 ['IsP', '$cat =~ /^P/', ''], # Punctuation
94 ['IsPd', '$cat eq "Pd"', ''], # Punctuation, Dash
95 ['IsPs', '$cat eq "Ps"', ''], # Punctuation, Open
96 ['IsPe', '$cat eq "Pe"', ''], # Punctuation, Close
97 ['IsPo', '$cat eq "Po"', ''], # Punctuation, Other
98 ['IsPc', '$cat eq "Pc"', ''], # Punctuation, Connector
99 ['IsPi', '$cat eq "Pi"', ''], # Punctuation, Initial quote
100 ['IsPf', '$cat eq "Pf"', ''], # Punctuation, Final quote
102 ['IsS', '$cat =~ /^S/', ''], # Symbol
103 ['IsSm', '$cat eq "Sm"', ''], # Symbol, Math
104 ['IsSk', '$cat eq "Sk"', ''], # Symbol, Modifier
105 ['IsSc', '$cat eq "Sc"', ''], # Symbol, Currency
106 ['IsSo', '$cat eq "So"', ''], # Symbol, Other
109 ['CombiningClass', '$comb', '$comb'],
111 # BIDIRECTIONAL PROPERTIES
113 ['Bidirectional', '$bid', '$bid'],
117 ['IsBidiL', '$bid eq "L"', ''], # Left-Right; Most alphabetic,
118 # syllabic, and logographic
119 # characters (e.g., CJK
121 ['IsBidiR', '$bid eq "R"', ''], # Right-Left; Arabic, Hebrew,
122 # and punctuation specific to
125 ['IsBidiLRE', '$bid eq "LRE"', ''], # Left-to-Right Embedding
126 ['IsBidiLRO', '$bid eq "LRO"', ''], # Left-to-Right Override
127 ['IsBidiAL', '$bid eq "AL"', ''], # Right-to-Left Arabic
128 ['IsBidiRLE', '$bid eq "RLE"', ''], # Right-to-Left Embedding
129 ['IsBidiRLO', '$bid eq "RLO"', ''], # Right-to-Left Override
130 ['IsBidiPDF', '$bid eq "PDF"', ''], # Pop Directional Format
131 ['IsBidiNSM', '$bid eq "NSM"', ''], # Non-Spacing Mark
132 ['IsBidiBN', '$bid eq "BN"', ''], # Boundary Neutral
136 ['IsBidiEN','$bid eq "EN"', ''], # European Number
137 ['IsBidiES','$bid eq "ES"', ''], # European Number Separator
138 ['IsBidiET','$bid eq "ET"', ''], # European Number Terminator
139 ['IsBidiAN','$bid eq "AN"', ''], # Arabic Number
140 ['IsBidiCS','$bid eq "CS"', ''], # Common Number Separator
144 ['IsBidiB', '$bid eq "B"', ''], # Block Separator
145 ['IsBidiS', '$bid eq "S"', ''], # Segment Separator
149 ['IsBidiWS','$bid eq "WS"', ''], # Whitespace
150 ['IsBidiON','$bid eq "ON"', ''], # Other Neutrals ; All other
151 # characters: punctuation,
156 ['Decomposition', '$decomp', '$decomp'],
157 ['IsDecoCanon', '$decomp && $decomp !~ /^</', ''],
158 ['IsDecoCompat', '$decomp =~ /^</', ''],
159 ['IsDCfont', '$decomp =~ /^<font>/', ''],
160 ['IsDCnoBreak', '$decomp =~ /^<noBreak>/', ''],
161 ['IsDCinitial', '$decomp =~ /^<initial>/', ''],
162 ['IsDCmedial', '$decomp =~ /^<medial>/', ''],
163 ['IsDCfinal', '$decomp =~ /^<final>/', ''],
164 ['IsDCisolated', '$decomp =~ /^<isolated>/', ''],
165 ['IsDCcircle', '$decomp =~ /^<circle>/', ''],
166 ['IsDCsuper', '$decomp =~ /^<super>/', ''],
167 ['IsDCsub', '$decomp =~ /^<sub>/', ''],
168 ['IsDCvertical', '$decomp =~ /^<vertical>/', ''],
169 ['IsDCwide', '$decomp =~ /^<wide>/', ''],
170 ['IsDCnarrow', '$decomp =~ /^<narrow>/', ''],
171 ['IsDCsmall', '$decomp =~ /^<small>/', ''],
172 ['IsDCsquare', '$decomp =~ /^<square>/', ''],
173 ['IsDCfraction', '$decomp =~ /^<fraction>/', ''],
174 ['IsDCcompat', '$decomp =~ /^<compat>/', ''],
178 ['Number', '$num ne ""', '$num'],
182 ['IsMirrored', '$mir eq "Y"', ''],
186 ['ArabLink', '1', '$link'],
187 ['ArabLnkGrp', '1', '$linkgroup'],
191 ['JamoShort', '1', '$short'],
197 # Line break properties - Normative
199 ['IsLbrkBK','$brk eq "BK"', ''], # Mandatory Break
200 ['IsLbrkCR','$brk eq "CR"', ''], # Carriage Return
201 ['IsLbrkLF','$brk eq "LF"', ''], # Line Feed
202 ['IsLbrkCM','$brk eq "CM"', ''], # Attached Characters and Combining Marks
203 ['IsLbrkSG','$brk eq "SG"', ''], # Surrogates
204 ['IsLbrkGL','$brk eq "GL"', ''], # Non-breaking (Glue)
205 ['IsLbrkCB','$brk eq "CB"', ''], # Contingent Break Opportunity
206 ['IsLbrkSP','$brk eq "SP"', ''], # Space
207 ['IsLbrkZW','$brk eq "ZW"', ''], # Zero Width Space
209 # Line break properties - Informative
210 ['IsLbrkXX','$brk eq "XX"', ''], # Unknown
211 ['IsLbrkOP','$brk eq "OP"', ''], # Opening Punctuation
212 ['IsLbrkCL','$brk eq "CL"', ''], # Closing Punctuation
213 ['IsLbrkQU','$brk eq "QU"', ''], # Ambiguous Quotation
214 ['IsLbrkNS','$brk eq "NS"', ''], # Non Starter
215 ['IsLbrkEX','$brk eq "EX"', ''], # Exclamation/Interrogation
216 ['IsLbrkSY','$brk eq "SY"', ''], # Symbols Allowing Breaks
217 ['IsLbrkIS','$brk eq "IS"', ''], # Infix Separator (Numeric)
218 ['IsLbrkPR','$brk eq "PR"', ''], # Prefix (Numeric)
219 ['IsLbrkPO','$brk eq "PO"', ''], # Postfix (Numeric)
220 ['IsLbrkNU','$brk eq "NU"', ''], # Numeric
221 ['IsLbrkAL','$brk eq "AL"', ''], # Ordinary Alphabetic and Symbol Characters
222 ['IsLbrkID','$brk eq "ID"', ''], # Ideographic
223 ['IsLbrkIN','$brk eq "IN"', ''], # Inseparable
224 ['IsLbrkHY','$brk eq "HY"', ''], # Hyphen
225 ['IsLbrkBB','$brk eq "BB"', ''], # Break Opportunity Before
226 ['IsLbrkBA','$brk eq "BA"', ''], # Break Opportunity After
227 ['IsLbrkSA','$brk eq "SA"', ''], # Complex Context (South East Asian)
228 ['IsLbrkAI','$brk eq "AI"', ''], # Ambiguous (Alphabetic or Ideographic)
229 ['IsLbrkB2','$brk eq "B2"', ''], # Break Opportunity Before and After
232 # This is not written for speed...
237 foreach $file (@todo) {
238 my ($table, $wanted, $val) = @$file;
239 next if @ARGV and not grep { $_ eq $table } @ARGV;
242 if ($table =~ /^In(.+)/) {
244 unless (exists $InId{$1}) {
248 open(OUT, ">In/$id.pl") or die "Can't create In/$id.pl: $!\n";
249 print OUT "# In/$id.pl $1\n";
251 elsif ($table =~ /^(Is|To)(.+)/) {
252 open(OUT, ">$1/$2.pl") or die "Can't create $1/$2.pl: $!\n";
255 open(OUT, ">$table.pl") or die "Can't create $table.pl: $!\n";
258 # !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
259 # This file is built by $0 from e.g. $UnicodeData.
260 # Any changes made here will be lost!
265 print OUT proplist($table, $wanted, $val);
270 # Must treat blocks specially.
272 exit if @ARGV and not grep { $_ eq Block } @ARGV;
274 open(UD, 'Blocks.txt') or die "Can't open Blocks.txt: $!\n";
275 open(OUT, ">Blocks.pl") or die "Can't create Blocks.pl: $!\n";
277 # !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
278 # This file is built by $0 from e.g. $UnicodeData.
279 # Any changes made here will be lost!
289 ($code, $last, $name) = /^([0-9a-f]+)\.\.([0-9a-f]+); (.+)/i;
291 print OUT "$code $last $name\n";
294 unless (exists $InId{$name}) {
295 $InId{$name} = $InId++;
298 open(BLOCK, ">In/$id.pl");
299 print OUT "# In/$id.pl $name\n";
301 # !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
302 # This file is built by $0 from e.g. $UnicodeData.
303 # Any changes made here will be lost!
305 print BLOCK <<"END2";
317 open(INID, ">In.pl");
320 # !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
321 # This file is built by $0 from e.g. $UnicodeData.
322 # Any changes made here will be lost!
326 # Order doesn't matter but let's prettyprint anyway.
327 foreach my $in (sort { $InId{$a} <=> $InId{$b} } keys %InId) {
328 printf INID "%-40s => %2d,\n", "'$in'", $InId{$in};
335 ##################################################
338 my ($table, $wanted, $val) = @_;
343 return listFromPropFile($wanted) if $val eq $PropData;
345 if ($table =~ /^Arab/) {
346 open(UD, "ArabShap.txt") or warn "Can't open $table: $!";
348 $split = '($code, $name, $link, $linkgroup) = split(/; */);';
350 elsif ($table =~ /^Jamo/) {
351 open(UD, "Jamo.txt") or warn "Can't open $table: $!";
353 $split = '($code, $short, $name) = split(/; */); $code =~ s/^U\+//;';
355 elsif ($table =~ /^IsSyl/) {
356 open(UD, $SyllableData) or warn "Can't open $table: $!";
358 $split = '($code, $short, $syl) = split(/; */); $code =~ s/^U\+//;';
360 elsif ($table =~ /^IsLbrk/) {
361 open(UD, "LineBrk.txt") or warn "Can't open $table: $!";
363 $split = '($code, $brk, $name) = /^([0-9a-f]+);(\w+) # (.+)/i;';
366 open(UD, $UnicodeData) or warn "Can't open $UnicodeData: $!";
368 $split = '($code, $name, $cat, $comb, $bid, $decomp, $dec, $dig, $num, $mir, $uni1,
369 $comment, $up, $down, $title) = split(/;/);';
372 if ($table =~ /^(?:To|Is)[A-Z]/) {
380 push(\@wanted, [hex \$code, hex $val, \$name =~ /, First>\$/]);
387 $beg = shift @wanted;
389 while (@wanted and $wanted[0]->[0] == $last->[0] + 1 and
390 (not $val or $wanted[0]->[1] == $last->[1] + 1)) {
391 $last = shift @wanted;
393 $out .= sprintf "%04x", $beg->[0];
395 $last = shift @wanted;
401 $out .= sprintf "\t%04x", $last->[0];
403 $out .= sprintf "\t%04x", $beg->[1] if $val;
415 push(\@wanted, [hex \$code, $val, \$name =~ /, First>\$/]);
422 $beg = shift @wanted;
424 while (@wanted and $wanted[0]->[0] == $last->[0] + 1 and
425 ($wanted[0]->[1] eq $last->[1])) {
426 $last = shift @wanted;
428 $out .= sprintf "%04x", $beg->[0];
430 $last = shift @wanted;
436 $out .= sprintf "\t%04x", $last->[0];
438 $out .= sprintf "\t%s\n", $beg->[1];
444 sub listFromPropFile {
448 open (UD, $PropData) or die "Can't open $PropData: $!\n";
449 local($/) = "\n" . '*' x 43 . "\n\nProperty dump for:"; # not 42?
454 if (s/0x[\d\w]+\s+\((.*?)\)// and $wanted eq $1) {
471 open (SD, $SyllableData) or die "Can't open $SyllableData: $!\n";
475 ($code, $name, $syl) = split /; */;
477 push (@defs, ["IsSyl$syl", qq{\$syl eq "$syl"}, ''])
478 unless $seen{$syl}++;