Commit | Line | Data |
017e2add |
1 | #!../../perl -w |
2f2b4ff2 |
2 | BEGIN { @INC = '../../lib' }; |
017e2add |
3 | use strict; |
afdae191 |
4 | use Getopt::Std; |
5 | my @orig_ARGV = @ARGV; |
017e2add |
6 | |
7 | sub encode_U |
8 | { |
9b37254d |
9 | # UTF-8 encode long hand - only covers part of perl's range |
017e2add |
10 | my $uv = shift; |
11 | if ($uv < 0x80) |
12 | { |
13 | return chr($uv) |
14 | } |
15 | if ($uv < 0x800) |
16 | { |
17 | return chr(($uv >> 6) | 0xC0). |
18 | chr(($uv & 0x3F) | 0x80); |
19 | } |
20 | return chr(($uv >> 12) | 0xE0). |
21 | chr((($uv >> 6) & 0x3F) | 0x80). |
22 | chr(($uv & 0x3F) | 0x80); |
23 | } |
24 | |
25 | sub encode_S |
26 | { |
14a8264b |
27 | # encode single byte |
017e2add |
28 | my ($ch,$page) = @_; |
29 | return chr($ch); |
30 | } |
31 | |
32 | sub encode_D |
33 | { |
14a8264b |
34 | # encode double byte MS byte first |
017e2add |
35 | my ($ch,$page) = @_; |
36 | return chr($page).chr($ch); |
37 | } |
38 | |
39 | sub encode_M |
40 | { |
14a8264b |
41 | # encode Multi-byte - single for 0..255 otherwise double |
017e2add |
42 | my ($ch,$page) = @_; |
43 | return &encode_D if $page; |
44 | return &encode_S; |
45 | } |
46 | |
14a8264b |
47 | # Win32 does not expand globs on command line |
252a8565 |
48 | eval "\@ARGV = map(glob(\$_),\@ARGV)" if ($^O eq 'MSWin32'); |
18b7339f |
49 | |
afdae191 |
50 | my %opt; |
e03ac092 |
51 | getopts('qo:F:n:',\%opt); |
afdae191 |
52 | my $cname = (exists $opt{'o'}) ? $opt{'o'} : shift(@ARGV); |
2f2b4ff2 |
53 | chmod(0666,$cname) if -f $cname && !-w $cname; |
017e2add |
54 | open(C,">$cname") || die "Cannot open $cname:$!"; |
afdae191 |
55 | |
56 | |
2f2b4ff2 |
57 | my $dname = $cname; |
58 | $dname =~ s/(\.[^\.]*)?$/.def/; |
e0c49a6b |
59 | |
60 | my ($doC,$doEnc,$doUcm); |
61 | |
62 | if ($cname =~ /\.(c|xs)$/) |
63 | { |
64 | $doC = 1; |
65 | chmod(0666,$dname) if -f $cname && !-w $dname; |
66 | open(D,">$dname") || die "Cannot open $dname:$!"; |
67 | my $hname = $cname; |
68 | $hname =~ s/(\.[^\.]*)?$/.h/; |
69 | chmod(0666,$hname) if -f $cname && !-w $hname; |
70 | open(H,">$hname") || die "Cannot open $hname:$!"; |
71 | |
72 | foreach my $fh (\*C,\*D,\*H) |
73 | { |
afdae191 |
74 | print $fh <<"END" unless $opt{'q'}; |
14a8264b |
75 | /* |
76 | !!!!!!! DO NOT EDIT THIS FILE !!!!!!! |
77 | This file was autogenerated by: |
afdae191 |
78 | $^X $0 $cname @orig_ARGV |
14a8264b |
79 | */ |
80 | END |
e0c49a6b |
81 | } |
14a8264b |
82 | |
e0c49a6b |
83 | if ($cname =~ /(\w+)\.xs$/) |
84 | { |
85 | print C "#include <EXTERN.h>\n"; |
86 | print C "#include <perl.h>\n"; |
87 | print C "#include <XSUB.h>\n"; |
88 | print C "#define U8 U8\n"; |
89 | } |
90 | print C "#include \"encode.h\"\n"; |
91 | } |
92 | elsif ($cname =~ /\.enc$/) |
93 | { |
94 | $doEnc = 1; |
95 | } |
96 | elsif ($cname =~ /\.ucm$/) |
2f2b4ff2 |
97 | { |
e0c49a6b |
98 | $doUcm = 1; |
2f2b4ff2 |
99 | } |
017e2add |
100 | |
afdae191 |
101 | my @encfiles; |
102 | if (exists $opt{'F'}) |
103 | { |
104 | # -F is followed by name of file containing list of filenames |
105 | my $flist = $opt{'F'}; |
106 | open(FLIST,$flist) || die "Cannot open $flist:$!"; |
107 | chomp(@encfiles = <FLIST>); |
108 | close(FLIST); |
109 | } |
110 | else |
111 | { |
112 | @encfiles = @ARGV; |
113 | } |
c6fdb90a |
114 | |
017e2add |
115 | my %encoding; |
116 | my %strings; |
117 | |
2f2b4ff2 |
118 | sub cmp_name |
119 | { |
120 | if ($a =~ /^.*-(\d+)/) |
121 | { |
122 | my $an = $1; |
123 | if ($b =~ /^.*-(\d+)/) |
124 | { |
125 | my $r = $an <=> $1; |
126 | return $r if $r; |
127 | } |
128 | } |
129 | return $a cmp $b; |
130 | } |
131 | |
c6fdb90a |
132 | foreach my $enc (sort cmp_name @encfiles) |
017e2add |
133 | { |
9b37254d |
134 | my ($name,$sfx) = $enc =~ /^.*?([\w-]+)\.(enc|ucm)$/; |
e03ac092 |
135 | $name = delete $opt{'n'} if exists $opt{'n'}; |
017e2add |
136 | if (open(E,$enc)) |
137 | { |
9b37254d |
138 | if ($sfx eq 'enc') |
139 | { |
140 | compile_enc(\*E,lc($name),\*C); |
141 | } |
142 | else |
143 | { |
144 | compile_ucm(\*E,lc($name),\*C); |
145 | } |
017e2add |
146 | } |
147 | else |
148 | { |
149 | warn "Cannot open $enc for $name:$!"; |
150 | } |
151 | } |
152 | |
e0c49a6b |
153 | if ($doC) |
2f2b4ff2 |
154 | { |
e0c49a6b |
155 | foreach my $enc (sort cmp_name keys %encoding) |
156 | { |
157 | my $sym = "${enc}_encoding"; |
158 | $sym =~ s/\W+/_/g; |
159 | print C "encode_t $sym = \n"; |
160 | print C " {",join(',',"\"$enc\"",@{$encoding{$enc}}),"};\n\n"; |
161 | } |
2f2b4ff2 |
162 | |
e0c49a6b |
163 | foreach my $enc (sort cmp_name keys %encoding) |
164 | { |
165 | my $sym = "${enc}_encoding"; |
166 | $sym =~ s/\W+/_/g; |
167 | print H "extern encode_t $sym;\n"; |
168 | print D " Encode_Define(aTHX_ &$sym);\n"; |
169 | } |
017e2add |
170 | |
e0c49a6b |
171 | if ($cname =~ /(\w+)\.xs$/) |
172 | { |
173 | my $mod = $1; |
174 | print C "\nMODULE = Encode::$mod\tPACKAGE = Encode::$mod\n\n"; |
175 | print C "BOOT:\n{\n"; |
176 | print C "#include \"$dname\"\n"; |
177 | print C "}\n"; |
178 | } |
179 | close(D); |
180 | close(H); |
2f2b4ff2 |
181 | } |
017e2add |
182 | close(C); |
183 | |
9b37254d |
184 | |
185 | sub compile_ucm |
186 | { |
187 | my ($fh,$name,$ch) = @_; |
188 | my $e2u = {}; |
189 | my $u2e = {}; |
190 | my $cs; |
191 | my %attr; |
192 | while (<$fh>) |
193 | { |
194 | s/#.*$//; |
195 | last if /^\s*CHARMAP\s*$/i; |
196 | if (/^\s*<(\w+)>\s+"?([^"]*)"?\s*$/i) |
197 | { |
198 | $attr{$1} = $2; |
199 | } |
200 | } |
201 | if (!defined($cs = $attr{'code_set_name'})) |
202 | { |
203 | warn "No <code_set_name> in $name\n"; |
204 | } |
205 | else |
206 | { |
e0c49a6b |
207 | # $name = lc($cs); |
9b37254d |
208 | } |
209 | my $erep; |
210 | my $urep; |
afdae191 |
211 | my $max_el; |
212 | my $min_el; |
9b37254d |
213 | if (exists $attr{'subchar'}) |
214 | { |
afdae191 |
215 | my @byte; |
216 | $attr{'subchar'} =~ /^\s*/cg; |
217 | push(@byte,$1) while $attr{'subchar'} =~ /\G\\x([0-9a-f]+)/icg; |
218 | $erep = join('',map(chr(hex($_)),@byte)); |
9b37254d |
219 | } |
e0c49a6b |
220 | warn "Scanning $name ($cs)\n"; |
9b37254d |
221 | my $nfb = 0; |
222 | my $hfb = 0; |
223 | while (<$fh>) |
224 | { |
225 | s/#.*$//; |
226 | last if /^\s*END\s+CHARMAP\s*$/i; |
227 | next if /^\s*$/; |
afdae191 |
228 | my ($u,@byte); |
229 | my $fb = ''; |
230 | $u = $1 if (/^<U([0-9a-f]+)>\s+/igc); |
231 | push(@byte,$1) while /\G\\x([0-9a-f]+)/igc; |
232 | $fb = $1 if /\G\s*(\|[0-3])/gc; |
233 | # warn "$_: $u @byte | $fb\n"; |
234 | die "Bad line:$_" unless /\G\s*(#.*)?$/gc; |
9b37254d |
235 | if (defined($u)) |
236 | { |
237 | my $uch = encode_U(hex($u)); |
e0c49a6b |
238 | my $ech = join('',map(chr(hex($_)),@byte)); |
afdae191 |
239 | my $el = length($ech); |
240 | $max_el = $el if (!defined($max_el) || $el > $max_el); |
241 | $min_el = $el if (!defined($min_el) || $el < $min_el); |
9b37254d |
242 | if (length($fb)) |
243 | { |
244 | $fb = substr($fb,1); |
245 | $hfb++; |
246 | } |
247 | else |
248 | { |
249 | $nfb++; |
250 | $fb = '0'; |
251 | } |
252 | # $fb is fallback flag |
253 | # 0 - round trip safe |
254 | # 1 - fallback for unicode -> enc |
255 | # 2 - skip sub-char mapping |
256 | # 3 - fallback enc -> unicode |
257 | enter($u2e,$uch,$ech,$u2e,$fb+0) if ($fb =~ /[01]/); |
258 | enter($e2u,$ech,$uch,$e2u,$fb+0) if ($fb =~ /[03]/); |
259 | } |
260 | else |
261 | { |
262 | warn $_; |
263 | } |
9b37254d |
264 | } |
265 | if ($nfb && $hfb) |
266 | { |
267 | die "$nfb entries without fallback, $hfb entries with\n"; |
268 | } |
e0c49a6b |
269 | if ($doC) |
270 | { |
271 | output($ch,$name.'_utf8',$e2u); |
272 | output($ch,'utf8_'.$name,$u2e); |
273 | $encoding{$name} = [$e2u->{Cname},$u2e->{Cname}, |
274 | outstring($ch,$e2u->{Cname}.'_def',$erep),length($erep)]; |
275 | } |
276 | elsif ($doEnc) |
277 | { |
278 | output_enc($ch,$name,$e2u); |
279 | } |
280 | elsif ($doUcm) |
281 | { |
afdae191 |
282 | output_ucm($ch,$name,$u2e,$erep,$min_el,$max_el); |
e0c49a6b |
283 | } |
9b37254d |
284 | } |
285 | |
14a8264b |
286 | sub compile_enc |
017e2add |
287 | { |
288 | my ($fh,$name,$ch) = @_; |
289 | my $e2u = {}; |
290 | my $u2e = {}; |
291 | |
292 | my $type; |
293 | while ($type = <$fh>) |
294 | { |
295 | last if $type !~ /^\s*#/; |
296 | } |
297 | chomp($type); |
298 | return if $type eq 'E'; |
299 | my ($def,$sym,$pages) = split(/\s+/,scalar(<$fh>)); |
14a8264b |
300 | warn "$type encoded $name\n"; |
017e2add |
301 | my $rep = ''; |
afdae191 |
302 | my $min_el; |
303 | my $max_el; |
017e2add |
304 | { |
305 | my $v = hex($def); |
306 | no strict 'refs'; |
307 | $rep = &{"encode_$type"}($v & 0xFF, ($v >> 8) & 0xffe); |
308 | } |
e03ac092 |
309 | my %seen; |
017e2add |
310 | while ($pages--) |
311 | { |
312 | my $line = <$fh>; |
313 | chomp($line); |
314 | my $page = hex($line); |
315 | my $ch = 0; |
316 | for (my $i = 0; $i < 16; $i++) |
317 | { |
318 | my $line = <$fh>; |
319 | for (my $j = 0; $j < 16; $j++) |
320 | { |
321 | no strict 'refs'; |
322 | my $ech = &{"encode_$type"}($ch,$page); |
323 | my $val = hex(substr($line,0,4,'')); |
e03ac092 |
324 | next if $val == 0xFFFD; |
017e2add |
325 | if ($val || (!$ch && !$page)) |
326 | { |
afdae191 |
327 | my $el = length($ech); |
328 | $max_el = $el if (!defined($max_el) || $el > $max_el); |
329 | $min_el = $el if (!defined($min_el) || $el < $min_el); |
017e2add |
330 | my $uch = encode_U($val); |
e03ac092 |
331 | if (exists $seen{$uch}) |
332 | { |
333 | warn sprintf("U%04X is %02X%02X and %02X%02X\n", |
334 | $val,$page,$ch,@{$seen{$uch}}); |
335 | } |
336 | else |
337 | { |
338 | $seen{$uch} = [$page,$ch]; |
339 | } |
9b37254d |
340 | enter($e2u,$ech,$uch,$e2u,0); |
341 | enter($u2e,$uch,$ech,$u2e,0); |
017e2add |
342 | } |
343 | else |
344 | { |
345 | # No character at this position |
346 | # enter($e2u,$ech,undef,$e2u); |
347 | } |
348 | $ch++; |
349 | } |
350 | } |
351 | } |
e0c49a6b |
352 | if ($doC) |
353 | { |
354 | output($ch,$name.'_utf8',$e2u); |
355 | output($ch,'utf8_'.$name,$u2e); |
356 | $encoding{$name} = [$e2u->{Cname},$u2e->{Cname}, |
357 | outstring($ch,$e2u->{Cname}.'_def',$rep),length($rep)]; |
358 | } |
359 | elsif ($doEnc) |
360 | { |
361 | output_enc($ch,$name,$e2u); |
362 | } |
363 | elsif ($doUcm) |
364 | { |
afdae191 |
365 | output_ucm($ch,$name,$u2e,$rep,$min_el,$max_el); |
e0c49a6b |
366 | } |
017e2add |
367 | } |
368 | |
369 | sub enter |
370 | { |
9b37254d |
371 | my ($a,$s,$d,$t,$fb) = @_; |
017e2add |
372 | $t = $a if @_ < 4; |
373 | my $b = substr($s,0,1); |
374 | my $e = $a->{$b}; |
375 | unless ($e) |
376 | { # 0 1 2 3 4 5 |
9b37254d |
377 | $e = [$b,$b,'',{},length($s),0,$fb]; |
017e2add |
378 | $a->{$b} = $e; |
379 | } |
380 | if (length($s) > 1) |
381 | { |
9b37254d |
382 | enter($e->[3],substr($s,1),$d,$t,$fb); |
017e2add |
383 | } |
384 | else |
385 | { |
386 | $e->[2] = $d; |
387 | $e->[3] = $t; |
388 | $e->[5] = length($d); |
389 | } |
390 | } |
391 | |
017e2add |
392 | sub outstring |
393 | { |
394 | my ($fh,$name,$s) = @_; |
395 | my $sym = $strings{$s}; |
396 | unless ($sym) |
397 | { |
2f2b4ff2 |
398 | foreach my $o (keys %strings) |
017e2add |
399 | { |
2f2b4ff2 |
400 | my $i = index($o,$s); |
401 | if ($i >= 0) |
017e2add |
402 | { |
2f2b4ff2 |
403 | $sym = $strings{$o}; |
404 | $sym .= sprintf("+0x%02x",$i) if ($i); |
405 | return $sym; |
017e2add |
406 | } |
407 | } |
408 | $strings{$s} = $sym = $name; |
14a8264b |
409 | printf $fh "\nstatic const U8 %s[%d] =\n",$name,length($s); |
2f2b4ff2 |
410 | # Do in chunks of 16 chars to constrain line length |
411 | # Assumes ANSI C adjacent string litteral concatenation |
017e2add |
412 | while (length($s)) |
413 | { |
414 | my $c = substr($s,0,16,''); |
415 | print $fh '"',join('',map(sprintf('\x%02x',ord($_)),split(//,$c))),'"'; |
416 | print $fh "\n" if length($s); |
417 | } |
14a8264b |
418 | printf $fh ";\n"; |
017e2add |
419 | } |
420 | return $sym; |
421 | } |
422 | |
14a8264b |
423 | sub process |
017e2add |
424 | { |
14a8264b |
425 | my ($name,$a) = @_; |
017e2add |
426 | $name =~ s/\W+/_/g; |
427 | $a->{Cname} = $name; |
428 | my @keys = grep(ref($a->{$_}),sort keys %$a); |
017e2add |
429 | my $l; |
430 | my @ent; |
431 | foreach my $b (@keys) |
432 | { |
433 | my ($s,$f,$out,$t,$end) = @{$a->{$b}}; |
434 | if (defined($l) && |
435 | ord($b) == ord($a->{$l}[1])+1 && |
436 | $a->{$l}[3] == $a->{$b}[3] && |
437 | $a->{$l}[4] == $a->{$b}[4] && |
9b37254d |
438 | $a->{$l}[5] == $a->{$b}[5] && |
439 | $a->{$l}[6] == $a->{$b}[6] |
2f2b4ff2 |
440 | # && length($a->{$l}[2]) < 16 |
441 | ) |
017e2add |
442 | { |
443 | my $i = ord($b)-ord($a->{$l}[0]); |
444 | $a->{$l}[1] = $b; |
445 | $a->{$l}[2] .= $a->{$b}[2]; |
446 | } |
447 | else |
448 | { |
449 | $l = $b; |
450 | push(@ent,$b); |
451 | } |
14a8264b |
452 | if (exists $t->{Cname}) |
017e2add |
453 | { |
14a8264b |
454 | $t->{'Forward'} = 1 if $t != $a; |
455 | } |
456 | else |
457 | { |
458 | process(sprintf("%s_%02x",$name,ord($s)),$t); |
017e2add |
459 | } |
460 | } |
461 | if (ord($keys[-1]) < 255) |
462 | { |
463 | my $t = chr(ord($keys[-1])+1); |
464 | $a->{$t} = [$t,chr(255),undef,$a,0,0]; |
465 | push(@ent,$t); |
466 | } |
14a8264b |
467 | $a->{'Entries'} = \@ent; |
468 | } |
469 | |
470 | sub outtable |
471 | { |
472 | my ($fh,$a) = @_; |
473 | my $name = $a->{'Cname'}; |
017e2add |
474 | # String tables |
14a8264b |
475 | foreach my $b (@{$a->{'Entries'}}) |
017e2add |
476 | { |
477 | next unless $a->{$b}[5]; |
478 | my $s = ord($a->{$b}[0]); |
479 | my $e = ord($a->{$b}[1]); |
480 | outstring($fh,sprintf("%s__%02x_%02x",$name,$s,$e),$a->{$b}[2]); |
481 | } |
14a8264b |
482 | if ($a->{'Forward'}) |
483 | { |
484 | print $fh "\nstatic encpage_t $name\[",scalar(@{$a->{'Entries'}}),"];\n"; |
485 | } |
486 | $a->{'Done'} = 1; |
487 | foreach my $b (@{$a->{'Entries'}}) |
488 | { |
489 | my ($s,$e,$out,$t,$end,$l) = @{$a->{$b}}; |
490 | outtable($fh,$t) unless $t->{'Done'}; |
491 | } |
492 | print $fh "\nstatic encpage_t $name\[",scalar(@{$a->{'Entries'}}),"] = {\n"; |
493 | foreach my $b (@{$a->{'Entries'}}) |
017e2add |
494 | { |
9b37254d |
495 | my ($s,$e,$out,$t,$end,$l,$fb) = @{$a->{$b}}; |
017e2add |
496 | my $sc = ord($s); |
497 | my $ec = ord($e); |
e0c49a6b |
498 | $end |= 0x80 if $fb; |
017e2add |
499 | print $fh "{"; |
500 | if ($l) |
501 | { |
502 | printf $fh outstring($fh,'',$out); |
503 | } |
504 | else |
505 | { |
506 | print $fh "0"; |
507 | } |
508 | print $fh ",",$t->{Cname}; |
2f2b4ff2 |
509 | printf $fh ",0x%02x,0x%02x,$l,$end},\n",$sc,$ec; |
017e2add |
510 | } |
14a8264b |
511 | print $fh "};\n"; |
512 | } |
513 | |
514 | sub output |
515 | { |
516 | my ($fh,$name,$a) = @_; |
517 | process($name,$a); |
518 | # Sub-tables |
519 | outtable($fh,$a); |
017e2add |
520 | } |
521 | |
e0c49a6b |
522 | sub output_enc |
523 | { |
524 | my ($fh,$name,$a) = @_; |
525 | foreach my $b (sort keys %$a) |
526 | { |
527 | my ($s,$e,$out,$t,$end,$l,$fb) = @{$a->{$b}}; |
528 | } |
529 | } |
530 | |
531 | sub decode_U |
532 | { |
533 | my $s = shift; |
e0c49a6b |
534 | } |
535 | |
e0c49a6b |
536 | sub output_ucm_page |
537 | { |
538 | my ($fh,$a,$t,$pre) = @_; |
539 | # warn sprintf("Page %x\n",$pre); |
540 | foreach my $b (sort keys %$t) |
541 | { |
542 | my ($s,$e,$out,$n,$end,$l,$fb) = @{$t->{$b}}; |
543 | die "oops $s $e" unless $s eq $e; |
544 | my $u = ord($s); |
545 | if ($n != $a && $n != $t) |
546 | { |
547 | output_ucm_page($fh,$a,$n,(($pre|($u &0x3F)) << 6)&0xFFFF); |
548 | } |
549 | elsif (length($out)) |
550 | { |
551 | if ($pre) |
552 | { |
553 | $u = $pre|($u &0x3f); |
554 | } |
555 | printf $fh "<U%04X> ",$u; |
556 | foreach my $c (split(//,$out)) |
557 | { |
558 | printf $fh "\\x%02X",ord($c); |
559 | } |
560 | printf $fh " |%d\n",($fb ? 1 : 0); |
561 | } |
562 | else |
563 | { |
564 | warn join(',',@{$t->{$b}},$a,$t); |
565 | } |
566 | } |
567 | } |
568 | |
569 | sub output_ucm |
570 | { |
afdae191 |
571 | my ($fh,$name,$a,$rep,$min_el,$max_el) = @_; |
572 | print $fh "# Written by $0 @orig_ARGV\n" unless $opt{'q'}; |
573 | print $fh "<code_set_name> \"$name\"\n"; |
574 | if (defined $min_el) |
575 | { |
576 | print $fh "<mb_cur_min> $min_el\n"; |
577 | } |
578 | if (defined $max_el) |
579 | { |
580 | print $fh "<mb_cur_max> $max_el\n"; |
581 | } |
582 | if (defined $rep) |
583 | { |
584 | print $fh "<subchar> "; |
585 | foreach my $c (split(//,$rep)) |
586 | { |
587 | printf $fh "\\x%02X",ord($c); |
588 | } |
589 | print $fh "\n"; |
590 | } |
591 | print $fh "#\nCHARMAP\n"; |
e0c49a6b |
592 | output_ucm_page($fh,$a,$a,0); |
593 | print $fh "END CHARMAP\n"; |
594 | } |
017e2add |
595 | |