#!../../miniperl
-$UnicodeData = "UnicodeData-Latest.txt";
+$UnicodeData = "Unicode.300";
# Note: we try to keep filenames unique within first 8 chars. Using
# subdirectories for the following helps.
mkdir "In", 0777;
mkdir "Is", 0777;
mkdir "To", 0777;
-mkdir "Eq", 0777;
-
-open(UNICODEDATA, $UnicodeData) || die "$0: $UnicodeData: $!\n";
-
-while (<UNICODEDATA>) {
- ($code, $name) = split /;/;
-
- $code{$name} = $code;
- $name{$code} = $name;
-
- next unless $name =~ /^(.+? LETTER .+?) WITH .+( \w+ FORM)?$/;
-
- push @base, [ $code, $1 ];
- push @base, [ $code, $1.$2 ] if $2 ne '';
-
- # Before this "diacritics stripping" phase (and for Arabic, also
- # "form stripping" phase) all ligatures could be decomposed into
- # their constituent letters.
- #
- # For example the ligature
- # ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH ALEF ISOLATED FORM
- # would go first through ligature decomposition producing the two letters
- # ARABIC LETTER YEH WITH HAMZA ABOVE ISOLATED FORM
- # ARABIC LETTER ALEF WITH HAMZA ABOVE ISOLATED FORM
- # and those with diacritics stripping
- # ARABIC LETTER YEH ISOLATED FORM
- # ARABIC LETTER ALEF ISOLATED FORM
- # and those with the Arabic form stripping
- # ARABIC LETTER YEH
- # ARABIC LETTER ALEF ISOLATED FORM
- # ARABIC LETTER YEH
- # ARABIC LETTER ALEF ISOLATED FORM
- #
- # Similarly for ligatures from other scripts.
- # Effectively this would mean that ligatures turn into categories
- # (Unicodese for character classes).
-}
-
-foreach my $b (@base) {
- ($code, $base) = @$b;
- next unless exists $code{$base};
- push @{$unicode{$code{$base}}}, $code;
-# print "$code: $name{$code} -> $base\n",
-}
-
-@unicode = sort keys %unicode;
-
-print "Eq/Unicode\n";
-if (open(EQ_UNICODE, ">Eq/Unicode")) {
- foreach my $c (@unicode) {
- print EQ_UNICODE "$c @{$unicode{$c}}\n";
- }
- close EQ_UNICODE;
-} else {
- die "$0: failed to open Eq/Unicode for writing: $!\n";
-}
-
-print "Eq/Latin1\n";
-if (open(EQ_LATIN1, ">Eq/Latin1")) {
- foreach my $c (@unicode) {
- last if hex($c) > 255;
- my @c = grep { hex($_) <= 255 } @{$unicode{$c}};
- next unless @c;
- print EQ_LATIN1 "$c @c\n";
- }
- close EQ_LATIN1;
-} else {
- die "$0: failed to open Eq/Latin1 for writing: $!\n";
-}
@todo = (
# typical
# Jamo
['JamoShort', '1', '$short'],
+
+# Syllables
+
+ ['IsSylV', '$syl eq "V"', ''],
+ ['IsSylU', '$syl eq "U"', ''],
+ ['IsSylI', '$syl eq "I"', ''],
+ ['IsSylA', '$syl eq "A"', ''],
+ ['IsSylE', '$syl eq "E"', ''],
+ ['IsSylC', '$syl eq "C"', ''],
+ ['IsSylO', '$syl eq "O"', ''],
+ ['IsSylWV', '$syl eq "V"', ''],
+ ['IsSylWI', '$syl eq "I"', ''],
+ ['IsSylWA', '$syl eq "A"', ''],
+ ['IsSylWE', '$syl eq "E"', ''],
+ ['IsSylWC', '$syl eq "C"', ''],
+
+# Line break properties - Normative
+
+ ['IsLbrkBK','$brk eq "BK"', ''], # Mandatory Break
+ ['IsLbrkCR','$brk eq "CR"', ''], # Carriage Return
+ ['IsLbrkLF','$brk eq "LF"', ''], # Line Feed
+ ['IsLbrkCM','$brk eq "CM"', ''], # Attached Characters and Combining Marks
+ ['IsLbrkSG','$brk eq "SG"', ''], # Surrogates
+ ['IsLbrkGL','$brk eq "GL"', ''], # Non-breaking (Glue)
+ ['IsLbrkCB','$brk eq "CB"', ''], # Contingent Break Opportunity
+ ['IsLbrkSP','$brk eq "SP"', ''], # Space
+ ['IsLbrkZW','$brk eq "ZW"', ''], # Zero Width Space
+
+# Line break properties - Informative
+ ['IsLbrkXX','$brk eq "XX"', ''], # Unknown
+ ['IsLbrkOP','$brk eq "OP"', ''], # Opening Punctuation
+ ['IsLbrkCL','$brk eq "CL"', ''], # Closing Punctuation
+ ['IsLbrkQU','$brk eq "QU"', ''], # Ambiguous Quotation
+ ['IsLbrkNS','$brk eq "NS"', ''], # Non Starter
+ ['IsLbrkEX','$brk eq "EX"', ''], # Exclamation/Interrogation
+ ['IsLbrkSY','$brk eq "SY"', ''], # Symbols Allowing Breaks
+ ['IsLbrkIS','$brk eq "IS"', ''], # Infix Separator (Numeric)
+ ['IsLbrkPR','$brk eq "PR"', ''], # Prefix (Numeric)
+ ['IsLbrkPO','$brk eq "PO"', ''], # Postfix (Numeric)
+ ['IsLbrkNU','$brk eq "NU"', ''], # Numeric
+ ['IsLbrkAL','$brk eq "AL"', ''], # Ordinary Alphabetic and Symbol Characters
+ ['IsLbrkID','$brk eq "ID"', ''], # Ideographic
+ ['IsLbrkIN','$brk eq "IN"', ''], # Inseparable
+ ['IsLbrkHY','$brk eq "HY"', ''], # Hyphen
+ ['IsLbrkBB','$brk eq "BB"', ''], # Break Opportunity Before
+ ['IsLbrkBA','$brk eq "BA"', ''], # Break Opportunity After
+ ['IsLbrkSA','$brk eq "SA"', ''], # Complex Context (South East Asian)
+ ['IsLbrkAI','$brk eq "AI"', ''], # Ambiguous (Alphabetic or Ideographic)
+ ['IsLbrkB2','$brk eq "B2"', ''], # Break Opportunity Before and After
);
# This is not written for speed...
else {
open(OUT, ">$table.pl") or die "Can't create $table.pl: $!\n";
}
+ print OUT <<EOH;
+# !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
+# This file is built by $0 from e.g. $UnicodeData.
+# Any changes made here will be lost!
+EOH
print OUT <<"END";
return <<'END';
END
exit if @ARGV and not grep { $_ eq Block } @ARGV;
print "Block\n";
-open(UD, 'blocks.txt') or die "Can't open blocks.txt: $!\n";
+open(UD, 'Blocks.txt') or die "Can't open blocks.txt: $!\n";
open(OUT, ">Block.pl") or die "Can't create $table.pl: $!\n";
+print OUT <<EOH;
+# !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
+# This file is built by $0 from e.g. $UnicodeData.
+# Any changes made here will be lost!
+EOH
print OUT <<"END";
return <<'END';
END
print OUT "$code $last $name\n";
$name =~ s/\s+//g;
open(BLOCK, ">In/$name.pl");
+ print BLOCK <<EOH;
+# !!!!!!! DO NOT EDIT THIS FILE !!!!!!!
+# This file is built by $0 from e.g. $UnicodeData.
+# Any changes made here will be lost!
+EOH
print BLOCK <<"END2";
return <<'END';
$code $last
my $split;
if ($table =~ /^Arab/) {
- open(UD, "arabshp.txt") or warn "Can't open $table: $!";
+ open(UD, "ArabShap.txt") or warn "Can't open $table: $!";
$split = '($code, $name, $link, $linkgroup) = split(/; */);';
}
elsif ($table =~ /^Jamo/) {
- open(UD, "jamo2.txt") or warn "Can't open $table: $!";
+ open(UD, "Jamo.txt") or warn "Can't open $table: $!";
$split = '($code, $short, $name) = split(/; */); $code =~ s/^U\+//;';
}
+ elsif ($table =~ /^IsSyl/) {
+ open(UD, "syllables.txt") or warn "Can't open $table: $!";
+
+ $split = '($code, $short, $syl) = split(/; */); $code =~ s/^U\+//;';
+ }
+ elsif ($table =~ /^IsLbrk/) {
+ open(UD, "LineBrk.txt") or warn "Can't open $table: $!";
+
+ $split = '($code, $brk, $name) = split(/;/);';
+ }
else {
open(UD, $UnicodeData) or warn "Can't open $UnicodeData: $!";
$out;
}
-# Create the equivalence mappings.
-
-
+# eof
projects / p5sagit/p5-mst-13.2.git / blobdiff