Re: Unicode/EBCDIC

Message-ID: <Pine.OSF.4.10.10103081617390.377472-100000@aspara.forte.com>

p4raw-id: //depot/perl@9082

diff --git a/perl.c b/perl.c
index 03b2ec1..0920a41 100644 (file)
--- a/perl.c
+++ b/perl.c
@@ -253,9 +253,9 @@ perl_construct(pTHXx)
        if (PERL_REVISION > 127 || PERL_VERSION > 127 || PERL_SUBVERSION > 127)
            SvGROW(PL_patchlevel, UTF8_MAXLEN*3+1);
        s = (U8*)SvPVX(PL_patchlevel);
-       s = uv_to_utf8(s, (UV)PERL_REVISION);
-       s = uv_to_utf8(s, (UV)PERL_VERSION);
-       s = uv_to_utf8(s, (UV)PERL_SUBVERSION);
+       s = uv_to_utf8(s, (UV)(ASCII_TO_NATIVE(PERL_REVISION)));
+       s = uv_to_utf8(s, (UV)(ASCII_TO_NATIVE(PERL_VERSION)));
+       s = uv_to_utf8(s, (UV)(ASCII_TO_NATIVE(PERL_SUBVERSION)));
        *s = '\0';
        SvCUR_set(PL_patchlevel, s - (U8*)SvPVX(PL_patchlevel));
        SvPOK_on(PL_patchlevel);

diff --git a/perl.h b/perl.h
index 2b66473..4ee33cc 100644 (file)
--- a/perl.h
+++ b/perl.h
@@ -2436,10 +2436,220 @@ EXT char *PL_sig_name[];
 EXT int   PL_sig_num[];
 #endif
 
-/* fast case folding tables */
+/* fast conversion and case folding tables */
 
 #ifdef DOINIT
 #ifdef EBCDIC
+#if '^' == 106  /* if defined(_OSD_POSIX) POSIX-BC */
+EXT unsigned char PL_e2a[] = { /* ASCII (ISO8859-1) to EBCDIC (POSIX-BC) */
+      0,      1,      2,      3,     55,     45,     46,     47,
+     22,      5,     21,     11,     12,     13,     14,     15,
+     16,     17,     18,     19,     60,     61,     50,     38,
+     24,     25,     63,     39,     28,     29,     30,     31,
+     64,     90,    127,    123,     91,    108,     80,    125,
+     77,     93,     92,     78,    107,     96,     75,     97,
+    240,    241,    242,    243,    244,    245,    246,    247,
+    248,    249,    122,     94,     76,    126,    110,    111,
+    124,    193,    194,    195,    196,    197,    198,    199,
+    200,    201,    209,    210,    211,    212,    213,    214,
+    215,    216,    217,    226,    227,    228,    229,    230,
+    231,    232,    233,    187,    188,    189,    106,    109,
+     74,    129,    130,    131,    132,    133,    134,    135,
+    136,    137,    145,    146,    147,    148,    149,    150,
+    151,    152,    153,    162,    163,    164,    165,    166,
+    167,    168,    169,    251,     79,    253,    255,      7,
+     32,     33,     34,     35,     36,     37,      6,     23,
+     40,     41,     42,     43,     44,      9,     10,     27,
+     48,     49,     26,     51,     52,     53,     54,      8,
+     56,     57,     58,     59,      4,     20,     62,     95,
+     65,    170,    176,    177,    159,    178,    208,    181,
+    121,    180,    154,    138,    186,    202,    175,    161,
+    144,    143,    234,    250,    190,    160,    182,    179,
+    157,    218,    155,    139,    183,    184,    185,    171,
+    100,    101,     98,    102,     99,    103,    158,    104,
+    116,    113,    114,    115,    120,    117,    118,    119,
+    172,    105,    237,    238,    235,    239,    236,    191,
+    128,    224,    254,    221,    252,    173,    174,     89,
+     68,     69,     66,     70,     67,     71,    156,     72,
+     84,     81,     82,     83,     88,     85,     86,     87,
+    140,     73,    205,    206,    203,    207,    204,    225,
+    112,    192,    222,    219,    220,    141,    142,    223
+};
+EXT unsigned char PL_a2e[] = { /* EBCDIC (POSIX-BC) to ASCII (ISO8859-1) */
+      0,      1,      2,      3,    156,      9,    134,    127,
+    151,    141,    142,     11,     12,     13,     14,     15,
+     16,     17,     18,     19,    157,     10,      8,    135,
+     24,     25,    146,    143,     28,     29,     30,     31,
+    128,    129,    130,    131,    132,    133,     23,     27,
+    136,    137,    138,    139,    140,      5,      6,      7,
+    144,    145,     22,    147,    148,    149,    150,      4,
+    152,    153,    154,    155,     20,     21,    158,     26,
+     32,    160,    226,    228,    224,    225,    227,    229,
+    231,    241,     96,     46,     60,     40,     43,    124,
+     38,    233,    234,    235,    232,    237,    238,    239,
+    236,    223,     33,     36,     42,     41,     59,    159,
+     45,     47,    194,    196,    192,    193,    195,    197,
+    199,    209,     94,     44,     37,     95,     62,     63,
+    248,    201,    202,    203,    200,    205,    206,    207,
+    204,    168,     58,     35,     64,     39,     61,     34,
+    216,     97,     98,     99,    100,    101,    102,    103,
+    104,    105,    171,    187,    240,    253,    254,    177,
+    176,    106,    107,    108,    109,    110,    111,    112,
+    113,    114,    170,    186,    230,    184,    198,    164,
+    181,    175,    115,    116,    117,    118,    119,    120,
+    121,    122,    161,    191,    208,    221,    222,    174,
+    162,    163,    165,    183,    169,    167,    182,    188,
+    189,    190,    172,     91,     92,     93,    180,    215,
+    249,     65,     66,     67,     68,     69,     70,     71,
+     72,     73,    173,    244,    246,    242,    243,    245,
+    166,     74,     75,     76,     77,     78,     79,     80,
+     81,     82,    185,    251,    252,    219,    250,    255,
+    217,    247,     83,     84,     85,     86,     87,     88,
+     89,     90,    178,    212,    214,    210,    211,    213,
+     48,     49,     50,     51,     52,     53,     54,     55,
+     56,     57,    179,    123,    220,    125,    218,    126
+};
+#endif          /* POSIX-BC */
+#if '^' == 176  /* if defined(??) (OS/400?) 037 */
+EXT unsigned char PL_e2a[] = { /* ASCII (ISO8859-1) to EBCDIC (IBM-037) */
+      0,      1,      2,      3,     55,     45,     46,     47,
+     22,      5,     37,     11,     12,     13,     14,     15,
+     16,     17,     18,     19,     60,     61,     50,     38,
+     24,     25,     63,     39,     28,     29,     30,     31,
+     64,     90,    127,    123,     91,    108,     80,    125,
+     77,     93,     92,     78,    107,     96,     75,     97,
+    240,    241,    242,    243,    244,    245,    246,    247,
+    248,    249,    122,     94,     76,    126,    110,    111,
+    124,    193,    194,    195,    196,    197,    198,    199,
+    200,    201,    209,    210,    211,    212,    213,    214,
+    215,    216,    217,    226,    227,    228,    229,    230,
+    231,    232,    233,    186,    224,    187,    176,    109,
+    121,    129,    130,    131,    132,    133,    134,    135,
+    136,    137,    145,    146,    147,    148,    149,    150,
+    151,    152,    153,    162,    163,    164,    165,    166,
+    167,    168,    169,    192,     79,    208,    161,      7,
+     32,     33,     34,     35,     36,     21,      6,     23,
+     40,     41,     42,     43,     44,      9,     10,     27,
+     48,     49,     26,     51,     52,     53,     54,      8,
+     56,     57,     58,     59,      4,     20,     62,    255,
+     65,    170,     74,    177,    159,    178,    106,    181,
+    189,    180,    154,    138,     95,    202,    175,    188,
+    144,    143,    234,    250,    190,    160,    182,    179,
+    157,    218,    155,    139,    183,    184,    185,    171,
+    100,    101,     98,    102,     99,    103,    158,    104,
+    116,    113,    114,    115,    120,    117,    118,    119,
+    172,    105,    237,    238,    235,    239,    236,    191,
+    128,    253,    254,    251,    252,    173,    174,     89,
+     68,     69,     66,     70,     67,     71,    156,     72,
+     84,     81,     82,     83,     88,     85,     86,     87,
+    140,     73,    205,    206,    203,    207,    204,    225,
+    112,    221,    222,    219,    220,    141,    142,    223
+};
+EXT unsigned char PL_a2e[] = { /* EBCDIC (IBM-037) to ASCII (ISO8859-1) */
+      0,      1,      2,      3,    156,      9,    134,    127,
+    151,    141,    142,     11,     12,     13,     14,     15,
+     16,     17,     18,     19,    157,    133,      8,    135,
+     24,     25,    146,    143,     28,     29,     30,     31,
+    128,    129,    130,    131,    132,     10,     23,     27,
+    136,    137,    138,    139,    140,      5,      6,      7,
+    144,    145,     22,    147,    148,    149,    150,      4,
+    152,    153,    154,    155,     20,     21,    158,     26,
+     32,    160,    226,    228,    224,    225,    227,    229,
+    231,    241,    162,     46,     60,     40,     43,    124,
+     38,    233,    234,    235,    232,    237,    238,    239,
+    236,    223,     33,     36,     42,     41,     59,    172,
+     45,     47,    194,    196,    192,    193,    195,    197,
+    199,    209,    166,     44,     37,     95,     62,     63,
+    248,    201,    202,    203,    200,    205,    206,    207,
+    204,     96,     58,     35,     64,     39,     61,     34,
+    216,     97,     98,     99,    100,    101,    102,    103,
+    104,    105,    171,    187,    240,    253,    254,    177,
+    176,    106,    107,    108,    109,    110,    111,    112,
+    113,    114,    170,    186,    230,    184,    198,    164,
+    181,    126,    115,    116,    117,    118,    119,    120,
+    121,    122,    161,    191,    208,    221,    222,    174,
+     94,    163,    165,    183,    169,    167,    182,    188,
+    189,    190,     91,     93,    175,    168,    180,    215,
+    123,     65,     66,     67,     68,     69,     70,     71,
+     72,     73,    173,    244,    246,    242,    243,    245,
+    125,     74,     75,     76,     77,     78,     79,     80,
+     81,     82,    185,    251,    252,    249,    250,    255,
+     92,    247,     83,     84,     85,     86,     87,     88,
+     89,     90,    178,    212,    214,    210,    211,    213,
+     48,     49,     50,     51,     52,     53,     54,     55,
+     56,     57,    179,    219,    220,    217,    218,    159
+};
+#endif          /* 037 */
+#if '^' == 95   /* if defined(__MVS__) || defined(??) (VM/ESA?) 1047 */
+EXT unsigned char PL_e2a[] = { /* ASCII (ISO8859-1) to EBCDIC (IBM-1047) */
+    0,      1,      2,      3,      55,     45,     46,     47,
+    22,     5,      21,     11,     12,     13,     14,     15,
+    16,     17,     18,     19,     60,     61,     50,     38,
+    24,     25,     63,     39,     28,     29,     30,     31,
+    64,     90,     127,    123,    91,     108,    80,     125,
+    77,     93,     92,     78,     107,    96,     75,     97,
+    240,    241,    242,    243,    244,    245,    246,    247,
+    248,    249,    122,    94,     76,     126,    110,    111,
+    124,    193,    194,    195,    196,    197,    198,    199,
+    200,    201,    209,    210,    211,    212,    213,    214,
+    215,    216,    217,    226,    227,    228,    229,    230,
+    231,    232,    233,    173,    224,    189,    95,     109,
+    121,    129,    130,    131,    132,    133,    134,    135,
+    136,    137,    145,    146,    147,    148,    149,    150,
+    151,    152,    153,    162,    163,    164,    165,    166,
+    167,    168,    169,    192,    79,     208,    161,    7,
+    32,     33,     34,     35,     36,     37,     6,      23,
+    40,     41,     42,     43,     44,     9,      10,     27,
+    48,     49,     26,     51,     52,     53,     54,     8,
+    56,     57,     58,     59,     4,      20,     62,     255,
+    65,     170,    74,     177,    159,    178,    106,    181,
+    187,    180,    154,    138,    176,    202,    175,    188,
+    144,    143,    234,    250,    190,    160,    182,    179,
+    157,    218,    155,    139,    183,    184,    185,    171,
+    100,    101,    98,     102,    99,     103,    158,    104,
+    116,    113,    114,    115,    120,    117,    118,    119,
+    172,    105,    237,    238,    235,    239,    236,    191,
+    128,    253,    254,    251,    252,    186,    174,    89,
+    68,     69,     66,     70,     67,     71,     156,    72,
+    84,     81,     82,     83,     88,     85,     86,     87,
+    140,    73,     205,    206,    203,    207,    204,    225,
+    112,    221,    222,    219,    220,    141,    142,    223
+};
+EXT unsigned char PL_a2e[] = { /* EBCDIC (IBM-1047) to ASCII (ISO8859-1) */
+    0,      1,      2,      3,      156,    9,      134,    127,
+    151,    141,    142,    11,     12,     13,     14,     15,
+    16,     17,     18,     19,     157,    10,     8,      135,
+    24,     25,     146,    143,    28,     29,     30,     31,
+    128,    129,    130,    131,    132,    133,    23,     27,
+    136,    137,    138,    139,    140,    5,      6,      7,
+    144,    145,    22,     147,    148,    149,    150,    4,
+    152,    153,    154,    155,    20,     21,     158,    26,
+    32,     160,    226,    228,    224,    225,    227,    229,
+    231,    241,    162,    46,     60,     40,     43,     124,
+    38,     233,    234,    235,    232,    237,    238,    239,
+    236,    223,    33,     36,     42,     41,     59,     94,
+    45,     47,     194,    196,    192,    193,    195,    197,
+    199,    209,    166,    44,     37,     95,     62,     63,
+    248,    201,    202,    203,    200,    205,    206,    207,
+    204,    96,     58,     35,     64,     39,     61,     34,
+    216,    97,     98,     99,     100,    101,    102,    103,
+    104,    105,    171,    187,    240,    253,    254,    177,
+    176,    106,    107,    108,    109,    110,    111,    112,
+    113,    114,    170,    186,    230,    184,    198,    164,
+    181,    126,    115,    116,    117,    118,    119,    120,
+    121,    122,    161,    191,    208,    91,     222,    174,
+    172,    163,    165,    183,    169,    167,    182,    188,
+    189,    190,    221,    168,    175,    93,     180,    215,
+    123,    65,     66,     67,     68,     69,     70,     71,
+    72,     73,     173,    244,    246,    242,    243,    245,
+    125,    74,     75,     76,     77,     78,     79,     80,
+    81,     82,     185,    251,    252,    249,    250,    255,
+    92,     247,    83,     84,     85,     86,     87,     88,
+    89,     90,     178,    212,    214,    210,    211,    213,
+    48,     49,    50,      51,     52,     53,     54,     55,
+    56,     57,    179,     219,    220,    217,    218,    159
+};
+#endif          /* 1047 */
 EXT unsigned char PL_fold[] = { /* fast EBCDIC case folding table */
     0,      1,      2,      3,      4,      5,      6,      7,
     8,      9,      10,     11,     12,     13,     14,     15,
@@ -2512,6 +2722,10 @@ EXTCONST  unsigned char PL_fold[] = {
 #endif  /* !EBCDIC */
 #else
 EXTCONST unsigned char PL_fold[];
+#ifdef EBCDIC
+EXTCONST unsigned char PL_e2a[];
+EXTCONST unsigned char PL_a2e[];
+#endif /* EBCDIC */
 #endif
 
 #ifdef DOINIT

diff --git a/sv.c b/sv.c
index d47a2f5..20b4f2a 100644 (file)
--- a/sv.c
+++ b/sv.c
@@ -7188,6 +7188,8 @@ Perl_sv_vcatpvfn(pTHX_ SV *sv, const char *pat, STRLEN patlen, va_list *args, SV
                    iv = *vecstr;
                    ulen = 1;
                }
+               if (iv <256) 
+               iv = NATIVE_TO_ASCII(iv); /* v-strings are codepoints */
                vecstr += ulen;
                veclen -= ulen;
            }
@@ -7268,6 +7270,8 @@ Perl_sv_vcatpvfn(pTHX_ SV *sv, const char *pat, STRLEN patlen, va_list *args, SV
                    uv = *vecstr;
                    ulen = 1;
                }
+               if (uv <256) 
+               uv = NATIVE_TO_ASCII(uv); /* v-strings are codepoints */
                vecstr += ulen;
                veclen -= ulen;
            }

diff --git a/toke.c b/toke.c
index 4e90201..2bb9282 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -7273,6 +7273,10 @@ vstring:
                                            "Integer overflow in decimal number");
                        }
                    }
+                   /* THIS IS EVIL */
+                   if (rev < 256) 
+                       rev = ASCII_TO_NATIVE(rev);
+
                    tmpend = uv_to_utf8(tmpbuf, rev);
                    if (rev > revmax)
                        revmax = rev;
@@ -7289,11 +7293,11 @@ vstring:
 
                SvPOK_on(sv);
                SvREADONLY_on(sv);
-               if (revmax > 127) {
-                   SvUTF8_on(sv);
+               /* if (revmax > 127) { */
+                   SvUTF8_on(sv); /*
                    if (revmax < 256)
                      sv_utf8_downgrade(sv, TRUE);
-               }
+               } */
            }
        }
        break;

diff --git a/utf8.c b/utf8.c
index f00659a..55a8c7c 100644 (file)
--- a/utf8.c
+++ b/utf8.c
@@ -46,6 +46,8 @@ is the recommended Unicode-aware way of saying
 U8 *
 Perl_uv_to_utf8(pTHX_ U8 *d, UV uv)
 {
+    if (uv < 0x100) 
+    uv = NATIVE_TO_ASCII(uv);
     if (uv < 0x80) {
        *d++ = uv;
        return d;
@@ -254,7 +256,7 @@ Perl_utf8_to_uv(pTHX_ U8* s, STRLEN curlen, STRLEN* retlen, U32 flags)
     if (UTF8_IS_ASCII(uv)) {
        if (retlen)
            *retlen = 1;
-       return *s;
+       return ASCII_TO_NATIVE(*s);
     }
 
     if (UTF8_IS_CONTINUATION(uv) &&

diff --git a/utf8.h b/utf8.h
index 8b0c8c3..4917811 100644 (file)
--- a/utf8.h
+++ b/utf8.h
@@ -131,3 +131,22 @@ END_EXTERN_C
 #endif
 #define isIDFIRST_lazy(p)      isIDFIRST_lazy_if(p,1)
 #define isALNUM_lazy(p)                isALNUM_lazy_if(p,1)
+
+/* EBCDIC-happy ways of converting native code to UTF8; the reverse
+   process is taken care of in utf8_to_uv */
+
+#ifdef EBCDIC
+#define NATIVE_TO_ASCII(ch)                  PL_e2a[(ch)]
+#define ASCII_TO_NATIVE(ch)                  PL_a2e[(ch)]
+#else
+#define NATIVE_TO_ASCII(ch)                  (ch)
+#define ASCII_TO_NATIVE(ch)                  (ch)
+#endif
+
+#define UTF8_NEEDS_UPGRADE(ch)        (NATIVE_TO_ASCII(ch) & 0x80)
+#define NATIVE_TO_UTF8(ch, string)    STMT_START { \
+                                        if (!UTF8_NEEDS_UPGRADE(ch)) \
+                                            *(string)++ = NATIVE_TO_ASCII(ch); \
+                                        else /*  uv_to_utf8 is EBCDIC-aware */ \
+                                           string = uv_to_utf8(string, ch); \
+                                      } STMT_END