Commit | Line | Data |
61ad1ccd |
1 | #!./perl -w |
7aa207d6 |
2 | |
ba77e4cc |
3 | print "1..3980\n"; |
61ad1ccd |
4 | my $test = 0; |
7aa207d6 |
5 | |
5c7da53c |
6 | my %templates = ( |
7 | utf8 => 'C0U', |
8 | utf16be => 'n', |
9 | utf16le => 'v', |
10 | ); |
11 | |
12 | sub bytes_to_utf { |
13 | my ($enc, $content, $do_bom) = @_; |
14 | my $template = $templates{$enc}; |
15 | die "Unsupported encoding $enc" unless $template; |
b3766b12 |
16 | my @chars = unpack "U*", $content; |
17 | if ($enc ne 'utf8') { |
18 | # Make surrogate pairs |
19 | my @remember_that_utf_16_is_variable_length; |
20 | foreach my $ord (@chars) { |
21 | if ($ord < 0x10000) { |
22 | push @remember_that_utf_16_is_variable_length, |
23 | $ord; |
24 | } else { |
25 | $ord -= 0x10000; |
26 | push @remember_that_utf_16_is_variable_length, |
27 | (0xD800 | ($ord >> 10)), (0xDC00 | ($ord & 0x3FF)); |
28 | } |
29 | } |
30 | @chars = @remember_that_utf_16_is_variable_length; |
31 | } |
32 | return pack "$template*", ($do_bom ? 0xFEFF : ()), @chars; |
5c7da53c |
33 | } |
7aa207d6 |
34 | |
35 | sub test { |
02512a66 |
36 | my ($enc, $write, $expect, $bom, $nl, $name) = @_; |
5c7da53c |
37 | open my $fh, ">", "utf$$.pl" or die "utf.pl: $!"; |
38 | binmode $fh; |
02512a66 |
39 | print $fh bytes_to_utf($enc, $write . ($nl ? "\n" : ''), $bom); |
5c7da53c |
40 | close $fh or die $!; |
2d90ac95 |
41 | my $got = do "./utf$$.pl"; |
61ad1ccd |
42 | $test = $test + 1; |
43 | if (!defined $got) { |
02512a66 |
44 | print "not ok $test # $enc $bom $nl $name; got undef\n"; |
45 | } elsif ($got ne $expect) { |
46 | print "not ok $test # $enc $bom $nl $name; got '$got'\n"; |
61ad1ccd |
47 | } else { |
02512a66 |
48 | print "ok $test # $enc $bom $nl $name\n"; |
61ad1ccd |
49 | } |
7aa207d6 |
50 | } |
51 | |
386ac4df |
52 | for my $bom (0, 1) { |
53 | for my $enc (qw(utf16le utf16be utf8)) { |
02512a66 |
54 | for my $nl (1, 0) { |
55 | for my $value (123, 1234, 12345) { |
56 | test($enc, $value, $value, $bom, $nl, $value); |
57 | } |
58 | next if $enc eq 'utf8'; |
59 | # Arguably a bug that currently string literals from UTF-8 file |
60 | # handles are not implicitly "use utf8", but don't FIXME that |
61 | # right now, as here we're testing the input filter itself. |
62 | |
63 | for my $expect ("N", "\xFF", "\x{100}", "\x{010a}", "\x{0a23}", |
b3766b12 |
64 | "\x{10000}", "\x{64321}", "\x{10FFFD}", |
ba77e4cc |
65 | "\x{1000a}", # 0xD800 0xDC0A |
66 | "\x{12800}", # 0xD80A 0xDC00 |
02512a66 |
67 | ) { |
b3766b12 |
68 | # A space so that the UTF-16 heuristic triggers - " '" gives two |
02512a66 |
69 | # characters of ASCII. |
70 | my $write = " '$expect'"; |
71 | my $name = 'chrs ' . join ', ', map {ord $_} split '', $expect; |
72 | test($enc, $write, $expect, $bom, $nl, $name); |
c28d6105 |
73 | } |
ba77e4cc |
74 | |
75 | # This is designed to try to trip over the end of the buffer, |
76 | # with similar results to U-1000A and U-12800 above. |
77 | for my $pad (2 .. 162) { |
78 | for my $chr ("\x{10000}", "\x{1000a}", "\x{12800}") { |
79 | my $padding = ' ' x $pad; |
80 | # Need 4 octets that were from 2 ASCII characters to trigger |
81 | # the heuristic that detects UTF-16 without a BOM. For |
82 | # UTF-16BE, one space and the newline will do, as the |
83 | # newline's high octet comes first. But for UTF-16LE, a |
84 | # newline is "\n\0", so it doesn't trigger it. |
85 | test($enc, " \n$padding'$chr'", $chr, $bom, $nl, |
86 | sprintf "'\\x{%x}' with $pad spaces before it", ord $chr); |
87 | } |
88 | } |
386ac4df |
89 | } |
90 | } |
91 | } |
7aa207d6 |
92 | |
93 | END { |
2d90ac95 |
94 | 1 while unlink "utf$$.pl"; |
7aa207d6 |
95 | } |