} elsif ($arg eq '-w') {
$AlwaysWrite = 1; # update the files even if they havent changed
$FileList = "";
+ } elsif ($arg eq '-check') {
+ my $this = shift @ARGV;
+ my $ok = shift @ARGV;
+ if ($this ne $ok) {
+ print "Skipping as check params are not the same.\n";
+ exit(0);
+ }
} elsif ($arg eq '-maketest') {
$MakeTestScript = 1;
} elsif ($arg eq '-makelist') {
" -maketest : Make test script\n",
" -makelist : Rewrite the file list based on current setup\n",
" -L filelist : Use this file list, (defaults to $0)\n",
- " -C dir : Change to this directory before proceding\n";
+ " -C dir : Change to this directory before proceeding\n",
+ " -check A B : Executes only if A and B are the same\n";
}
}
return $Table;
}
-##
-## Returns true if the Table has no code points
-##
-sub Table::IsEmpty
-{
- my $Table = shift; #self
- return not @$Table;
-}
-
-##
-## Returns true if the Table has code points
-##
-sub Table::NotEmpty
-{
- my $Table = shift; #self
- return @$Table;
-}
##
## Returns the maximum code point currently in the table.
##
sub Table::Max
{
- my $Table = shift; #self
- confess "oops" if $Table->IsEmpty; ## must have code points to have a max
- return $Table->[-1]->[RANGE_END];
+ my $last = $_[0]->[-1]; ## last code point
+ confess "oops" unless $last; ## must have code points to have a max
+ return $last->[RANGE_END];
}
##
## Given a new code point, make the last range of the Table extend to
## include the new (and all intervening) code points.
##
+## Takes the time to make sure that the extension is valid.
+##
sub Table::Extend
{
my $Table = shift; #self
confess "oops ($codepoint <= $PrevMax)" if $codepoint <= $PrevMax;
- $Table->[-1]->[RANGE_END] = $codepoint;
+ $Table->ExtendNoCheck($codepoint);
+}
+
+
+##
+## Given a new code point, make the last range of the Table extend to
+## include the new (and all intervening) code points.
+##
+## Does NOT check that the extension is valid. Assumes that the caller
+## has already made this check.
+##
+sub Table::ExtendNoCheck
+{
+ ## Optmized adding: Assumes $Table and $codepoint as parms
+ $_[0]->[-1]->[RANGE_END] = $_[1];
}
##
## If we've already got a range working, and this code point is the next
## one in line, and if the name is the same, just extend the current range.
##
- if ($Table->NotEmpty
+ my $last = $Table->[-1];
+ if ($last
and
- $Table->Max == $codepoint - 1
+ $last->[RANGE_END] == $codepoint - 1
and
- $Table->[-1]->[RANGE_NAME] eq $name)
+ $last->[RANGE_NAME] eq $name)
{
- $Table->Extend($codepoint);
+ $Table->ExtendNoCheck($codepoint);
}
else
{
if ($start > $New->Max) {
$New->AppendRange($start, $end);
} elsif ($end > $New->Max) {
- $New->Extend($end);
+ $New->ExtendNoCheck($end);
}
}
{
my $Table = shift; #self
- return 0x1234 if $Table->IsEmpty();
+ return 0x1234 if not @$Table;
for my $set (@$Table)
{
Table->New(Is => 'Word', Desc => "[[:Word:]]", Fuzzy => 0);
$Cat{SpacePerl} =
Table->New(Is => 'SpacePerl', Desc => '\s', Fuzzy => 0);
-
+ $Cat{VertSpace} =
+ Table->New(Is => 'VertSpace', Desc => '\v', Fuzzy => 0);
+ $Cat{HorizSpace} =
+ Table->New(Is => 'HorizSpace', Desc => '\h', Fuzzy => 0);
my %To;
$To{Upper} = Table->New();
$To{Lower} = Table->New();
$Cat{SpacePerl}->$op($code) if $isspace
&& $code != 0x000B; # Backward compat.
+ $Cat{VertSpace}->$op($code) if grep {$code == $_}
+ ( 0x0A..0x0D,0x85,0x2028,0x2029 );
+
+ $Cat{HorizSpace}->$op($code) if grep {$code == $_} (
+ 0x09, 0x20, 0xa0, 0x1680, 0x180e, 0x2000, 0x2001, 0x2002,
+ 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200a,
+ 0x202f, 0x205f, 0x3000
+ );
+
$Cat{Blank}->$op($code) if $isspace
&& !($code == 0x000A ||
$code == 0x000B ||
);
}
- # Alphabetic is L and Other_Alphabetic.
+ # Alphabetic is L, Nl, and Other_Alphabetic.
New_Prop(Is => 'Alphabetic',
- Table->Merge($Cat{L}, $Prop{Other_Alphabetic}),
- Desc => '[\p{L}\p{OtherAlphabetic}]', # use canonical names here
+ Table->Merge($Cat{L}, $Cat{Nl}, $Prop{Other_Alphabetic}),
+ Desc => '[\p{L}\p{Nl}\p{OtherAlphabetic}]', # canonical names
Fuzzy => 1);
# Lowercase is Ll and Other_Lowercase.
New_Prop(Is => 'Lowercase',
Table->Merge($Cat{Ll}, $Prop{Other_Lowercase}),
- Desc => '[\p{Ll}\p{OtherLowercase}]', # use canonical names here
+ Desc => '[\p{Ll}\p{OtherLowercase}]', # canonical names
Fuzzy => 1);
# Uppercase is Lu and Other_Uppercase.
New_Prop(Is => 'Uppercase',
Table->Merge($Cat{Lu}, $Prop{Other_Uppercase}),
- Desc => '[\p{Lu}\p{Other_Uppercase}]', # use canonical names here
+ Desc => '[\p{Lu}\p{OtherUppercase}]', # canonical names
Fuzzy => 1);
# Math is Sm and Other_Math.
New_Prop(Is => 'Math',
Table->Merge($Cat{Sm}, $Prop{Other_Math}),
- Desc => '[\p{Sm}\p{OtherMath}]', # use canonical names here
+ Desc => '[\p{Sm}\p{OtherMath}]', # canonical names
Fuzzy => 1);
- # ID_Start is Ll, Lu, Lt, Lm, Lo, and Nl.
+ # ID_Start is Ll, Lu, Lt, Lm, Lo, Nl, and Other_ID_Start.
New_Prop(Is => 'ID_Start',
- Table->Merge(@Cat{qw[Ll Lu Lt Lm Lo Nl]}),
- Desc => '[\p{Ll}\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{Nl}]',
+ Table->Merge(@Cat{qw[Ll Lu Lt Lm Lo Nl]}, $Prop{Other_ID_Start}),
+ Desc => '[\p{Ll}\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{Nl}\p{OtherIDStart}]',
Fuzzy => 1);
- # ID_Continue is ID_Start, Mn, Mc, Nd, and Pc.
+ # ID_Continue is ID_Start, Mn, Mc, Nd, Pc, and Other_ID_Continue.
New_Prop(Is => 'ID_Continue',
- Table->Merge(@Cat{qw[Ll Lu Lt Lm Lo Nl Mn Mc Nd Pc ]}),
- Desc => '[\p{ID_Start}\p{Mn}\p{Mc}\p{Nd}\p{Pc}]',
+ Table->Merge(@Cat{qw[Ll Lu Lt Lm Lo Nl Mn Mc Nd Pc ]},
+ @Prop{qw[Other_ID_Start Other_ID_Continue]}),
+ Desc => '[\p{ID_Start}\p{Mn}\p{Mc}\p{Nd}\p{Pc}\p{OtherIDContinue}]',
Fuzzy => 1);
+
+ # Default_Ignorable_Code_Point = Other_Default_Ignorable_Code_Point
+ # + Cf + Cc + Cs + Noncharacter + Variation_Selector
+ # - WhiteSpace - FFF9..FFFB (Annotation Characters)
+
+ my $Annotation = Table->New();
+ $Annotation->RawAppendRange(0xFFF9, 0xFFFB);
+
+ New_Prop(Is => 'Default_Ignorable_Code_Point',
+ Table->Merge(@Cat{qw[Cf Cc Cs]},
+ $Prop{Noncharacter_Code_Point},
+ $Prop{Variation_Selector},
+ $Prop{Other_Default_Ignorable_Code_Point})
+ ->Invert
+ ->Merge($Prop{White_Space}, $Annotation)
+ ->Invert,
+ Desc => '(?![\p{WhiteSpace}\x{FFF9}-\x{FFFB}])[\p{Cf}\p{Cc}'.
+ '\p{Cs}\p{NoncharacterCodePoint}\p{VariationSelector}'.
+ '\p{OtherDefaultIgnorableCodePoint}]',
+ Fuzzy => 1);
+
}
projects / p5sagit/p5-mst-13.2.git / blobdiff