Infrastructure to use UTF-EBCDIC rather than UTF-8 as the internal

encoding on EBCDIC platforms. This has property that U+0000..U+009F i.e.
a superset of ASCII are invariant under the encoding. This is EBCDIC
friendly as an encoded string can be looked at as being EBCDIC by lexer
sprintf("%d",...) etc. in same manner that a UTF-8 string be considered
ASCII on ASCII machines.

 - re-arrange utf8.h to get ASCII specific vs Unicode generic bits
   seperate.
 - Add some more macros to comprehend different shift amounts and
   possible swizzle in UTF-EBCDIC vs UTF-8. Change utf8.c to use them.
 - add utfebcdic.h which provides UTF-EBCDIC versions of the macros,
   and conditionally #include it.

EBCDIC build as yet untested.  ASCII still fails the one test.

p4raw-id: //depot/perlio@9185

diff --git a/utf8.c b/utf8.c
index 227dcba..71a76e3 100644 (file)
--- a/utf8.c
+++ b/utf8.c
@@ -46,10 +46,22 @@ is the recommended Unicode-aware way of saying
 U8 *
 Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv)
 {
-    if (uv < 0x80) {
+    if (UTF8_IS_INVARIANT(uv)) {
        *d++ = uv;
        return d;
     }
+#if defined(EBCDIC) || 1 /* always for testing */
+    else {
+       STRLEN len  = UNISKIP(uv);
+       U8 *p = d+len-1;
+       while (p > d) {
+           *p-- = UTF_TO_NATIVE((uv & UTF_CONTINUATION_MASK) | UTF_CONTINUATION_MARK);
+           uv >>= UTF_ACCUMULATION_SHIFT;
+       }
+       *p = UTF_TO_NATIVE((uv & UTF_START_MASK(len)) | UTF_START_MARK(len));
+       return d+len;
+    }
+#else /* Non loop style */
     if (uv < 0x800) {
        *d++ = (( uv >>  6)         | 0xc0);
        *d++ = (( uv        & 0x3f) | 0x80);
@@ -116,6 +128,7 @@ Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv)
        return d;
     }
 #endif
+#endif /* Loop style */
 }
 
 /*
@@ -148,7 +161,7 @@ Perl_uvchr_to_utf8(pTHX_ U8 *d, UV uv)
 =for apidoc A|STRLEN|is_utf8_char|U8 *s
 
 Tests if some arbitrary number of bytes begins in a valid UTF-8
-character.  Note that an ASCII character is a valid UTF-8 character.
+character.  Note that an INVARIANT (i.e. ASCII) character is a valid UTF-8 character.
 The actual number of bytes in the UTF-8 character will be returned if
 it is valid, otherwise 0.
 
@@ -160,7 +173,7 @@ Perl_is_utf8_char(pTHX_ U8 *s)
     STRLEN slen, len;
     UV uv, ouv;
 
-    if (UTF8_IS_ASCII(u))
+    if (UTF8_IS_INVARIANT(u))
        return 1;
 
     if (!UTF8_IS_START(u))
@@ -173,7 +186,8 @@ Perl_is_utf8_char(pTHX_ U8 *s)
 
     slen = len - 1;
     s++;
-    uv = u;
+    /* The initial value is dubious */
+    uv  = u;
     ouv = uv;
     while (slen--) {
        if (!UTF8_IS_CONTINUATION(*s))
@@ -276,7 +290,7 @@ Perl_utf8n_to_uvuni(pTHX_ U8* s, STRLEN curlen, STRLEN* retlen, U32 flags)
        goto malformed;
     }
 
-    if (UTF8_IS_ASCII(uv)) {
+    if (UTF8_IS_INVARIANT(uv)) {
        if (retlen)
            *retlen = 1;
        return (UV) (*s);
@@ -294,20 +308,29 @@ Perl_utf8n_to_uvuni(pTHX_ U8* s, STRLEN curlen, STRLEN* retlen, U32 flags)
        goto malformed;
     }
 
+#ifdef EBCDIC
+    uv = NATIVE_TO_UVF(uv);
+#else
     if ((uv == 0xfe || uv == 0xff) &&
        !(flags & UTF8_ALLOW_FE_FF)) {
        warning = UTF8_WARN_FE_FF;
        goto malformed;
     }
-       
+#endif
+
     if      (!(uv & 0x20))     { len =  2; uv &= 0x1f; }
     else if (!(uv & 0x10))     { len =  3; uv &= 0x0f; }
     else if (!(uv & 0x08))     { len =  4; uv &= 0x07; }
     else if (!(uv & 0x04))     { len =  5; uv &= 0x03; }
+#ifdef EBCDIC
+    else if (!(uv & 0x02))     { len =  6; uv &= 0x01; }
+    else                       { len =  7; uv &= 0x01; }
+#else
     else if (!(uv & 0x02))     { len =  6; uv &= 0x01; }
     else if (!(uv & 0x01))     { len =  7; uv = 0; }
-    else                       { len = 13; uv = 0; } /* whoa! */
-       
+    else                       { len = 13; uv = 0; } /* whoa! */
+#endif
+
     if (retlen)
        *retlen = len;
 
@@ -634,9 +657,9 @@ Perl_utf8_to_bytes(pTHX_ U8* s, STRLEN *len)
     for (send = s + *len; s < send; ) {
         U8 c = *s++;
 
-        if (c >= 0x80 &&
-            ((s >= send) ||
-            ((*s++ & 0xc0) != 0x80) || ((c & 0xfe) != 0xc2))) {
+        if (!UTF8_IS_INVARIANT(c) &&
+            (!UTF8_IS_DOWNGRADEABLE_START(c) || (s >= send)
+            || !(c = *s++) || !UTF8_IS_CONTINUATION(c))) {
             *len = -1;
             return 0;
         }
@@ -679,7 +702,7 @@ Perl_bytes_from_utf8(pTHX_ U8* s, STRLEN *len, bool *is_utf8)
     /* ensure valid UTF8 and chars < 256 before converting string */
     for (send = s + *len; s < send;) {
        U8 c = *s++;
-       if (!UTF8_IS_ASCII(c)) {
+       if (!UTF8_IS_INVARIANT(c)) {
            if (UTF8_IS_DOWNGRADEABLE_START(c) && s < send &&
                 (c = *s++) && UTF8_IS_CONTINUATION(c))
                count++;
@@ -700,7 +723,7 @@ Perl_bytes_from_utf8(pTHX_ U8* s, STRLEN *len, bool *is_utf8)
     s = start; start = d;
     while (s < send) {
        U8 c = *s++;
-       if (!UTF8_IS_ASCII(c))
+       if (!UTF8_IS_INVARIANT(c))
            c = UTF8_ACCUMULATE(c, *s++);
        *d++ = ASCII_TO_NATIVE(c);
     }
@@ -732,7 +755,7 @@ Perl_bytes_to_utf8(pTHX_ U8* s, STRLEN *len)
 
     while (s < send) {
         UV uv = NATIVE_TO_ASCII(*s++);
-        if (UTF8_IS_ASCII(uv))
+        if (UTF8_IS_INVARIANT(uv))
             *d++ = uv;
         else {
             *d++ = UTF8_EIGHT_BIT_HI(uv);

diff --git a/utf8.h b/utf8.h
index 67b3d53..a606397 100644 (file)
--- a/utf8.h
+++ b/utf8.h
@@ -7,6 +7,13 @@
  *
  */
 
+#ifdef EBCDIC
+/* The equivalent of these macros but implementing UTF-EBCDIC
+   are in the following header file:
+ */
+
+#include "utfebcdic.h"
+#else
 START_EXTERN_C
 
 #ifdef DOINIT
@@ -26,47 +33,20 @@ EXTCONST unsigned char PL_utf8skip[];
 #endif
 
 END_EXTERN_C
-
-#define UTF8_MAXLEN 13 /* how wide can a single UTF8 encoded character become */
-
-/* #define IN_UTF8 (PL_curcop->op_private & HINT_UTF8) */
-#define IN_BYTE (PL_curcop->op_private & HINT_BYTE)
-#ifdef USE_BYTES_DOWNGRADES
-#define DO_UTF8(sv) (SvUTF8(sv) && !(IN_BYTE && sv_utf8_downgrade(sv,0)))
-#else
-#define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTE)
-#endif
-
-#define UTF8_ALLOW_EMPTY               0x0001
-#define UTF8_ALLOW_CONTINUATION                0x0002
-#define UTF8_ALLOW_NON_CONTINUATION    0x0004
-#define UTF8_ALLOW_FE_FF               0x0008
-#define UTF8_ALLOW_SHORT               0x0010
-#define UTF8_ALLOW_SURROGATE           0x0020
-#define UTF8_ALLOW_BOM                 0x0040
-#define UTF8_ALLOW_FFFF                        0x0080
-#define UTF8_ALLOW_LONG                        0x0100
-#define UTF8_ALLOW_ANYUV               (UTF8_ALLOW_EMPTY|UTF8_ALLOW_FE_FF|\
-                                        UTF8_ALLOW_SURROGATE|UTF8_ALLOW_BOM|\
-                                        UTF8_ALLOW_FFFF|UTF8_ALLOW_LONG)
-#define UTF8_ALLOW_ANY                 0x00ff
-#define UTF8_CHECK_ONLY                        0x0100
-
-#define UNICODE_SURROGATE_FIRST                0xd800
-#define UNICODE_SURROGATE_LAST         0xdfff
-#define UNICODE_REPLACEMENT            0xfffd
-#define UNICODE_BYTER_ORDER_MARK       0xfffe
-#define UNICODE_ILLEGAL                        0xffff
-
-#define UNICODE_IS_SURROGATE(c)                ((c) >= UNICODE_SURROGATE_FIRST && \
-                                        (c) <= UNICODE_SURROGATE_LAST)
-#define UNICODE_IS_REPLACEMENT(c)      ((c) == UNICODE_REPLACMENT)
-#define UNICODE_IS_BYTE_ORDER_MARK(c)  ((c) == UNICODE_BYTER_ORDER_MARK)
-#define UNICODE_IS_ILLEGAL(c)          ((c) == UNICODE_ILLEGAL)
-
 #define UTF8SKIP(s) PL_utf8skip[*(U8*)s]
 
-#define UTF8_QUAD_MAX  UINT64_C(0x1000000000)
+/* Native character to iso-8859-1 */
+#define NATIVE_TO_ASCII(ch)      (ch)
+#define ASCII_TO_NATIVE(ch)      (ch)
+/* Transform after encoding */
+#define NATIVE_TO_UTF(ch)        (ch)
+#define UTF_TO_NATIVE(ch)        (ch)
+/* Transforms in wide UV chars */
+#define UNI_TO_NATIVE(ch)        (ch)
+#define NATIVE_TO_UNI(ch)        (ch)
+/* Transforms in invariant space */
+#define NATIVE_TO_NEED(enc,ch)   (ch)
+#define ASCII_TO_NEED(enc,ch)    (ch)
 
 /*
 
@@ -84,18 +64,22 @@ END_EXTERN_C
 
  */
 
-#define UTF8_IS_ASCII(c)               (((U8)c) <  0x80)
+#define UTF8_IS_INVARIANT(c)           (((UV)c) <  0x80)
 #define UTF8_IS_START(c)               (((U8)c) >= 0xc0 && (((U8)c) <= 0xfd))
 #define UTF8_IS_CONTINUATION(c)                (((U8)c) >= 0x80 && (((U8)c) <= 0xbf))
 #define UTF8_IS_CONTINUED(c)           (((U8)c) &  0x80)
 #define UTF8_IS_DOWNGRADEABLE_START(c) (((U8)c & 0xfc) == 0xc0)
 
-#define UTF8_CONTINUATION_MASK         ((U8)0x3f)
-#define UTF8_ACCUMULATION_SHIFT                6
-#define UTF8_ACCUMULATE(old, new)      ((old) << UTF8_ACCUMULATION_SHIFT | (((U8)new) & UTF8_CONTINUATION_MASK))
+#define UTF_START_MARK(len) ((len >  7) ? 0xFF : (0xFE << (7-len)))
+#define UTF_START_MASK(len) ((len >= 7) ? 0x00 : (0x1F >> (len-2)))
+
+#define UTF_CONTINUATION_MARK          0x80
+#define UTF_ACCUMULATION_SHIFT         6
+#define UTF_CONTINUATION_MASK          ((U8)0x3f)
+#define UTF8_ACCUMULATE(old, new)      ((old) << UTF_ACCUMULATION_SHIFT | (((U8)new) & UTF_CONTINUATION_MASK))
 
-#define UTF8_EIGHT_BIT_HI(c)   ( (((U8)(c))>>6)      |0xc0)
-#define UTF8_EIGHT_BIT_LO(c)   (((((U8)(c))   )&0x3f)|0x80)
+#define UTF8_EIGHT_BIT_HI(c)   ((((U8)(c))>>UTF_ACCUMULATION_SHIFT)|UTF_START_MARK(2))
+#define UTF8_EIGHT_BIT_LO(c)   (((((U8)(c)))&UTF_CONTINUATION_MASK)|UTF_CONTINUATION_MARK)
 
 #ifdef HAS_QUAD
 #define UNISKIP(uv) ( (uv) < 0x80           ? 1 : \
@@ -115,43 +99,60 @@ END_EXTERN_C
                      (uv) < 0x80000000     ? 6 : 7 )
 #endif
 
-
 /*
  * Note: we try to be careful never to call the isXXX_utf8() functions
  * unless we're pretty sure we've seen the beginning of a UTF-8 character
  * (that is, the two high bits are set).  Otherwise we risk loading in the
  * heavy-duty SWASHINIT and SWASHGET routines unnecessarily.
  */
-#ifdef EBCDIC
-#define isIDFIRST_lazy_if(p,c) isIDFIRST(*(p))
-#define isALNUM_lazy_if(p,c)   isALNUM(*(p))
-#else
 #define isIDFIRST_lazy_if(p,c) ((IN_BYTE || (!c || (*((U8*)p) < 0xc0))) \
                                ? isIDFIRST(*(p)) \
                                : isIDFIRST_utf8((U8*)p))
 #define isALNUM_lazy_if(p,c)   ((IN_BYTE || (!c || (*((U8*)p) < 0xc0))) \
                                ? isALNUM(*(p)) \
                                : isALNUM_utf8((U8*)p))
-#endif
+
+
+#endif /* EBCDIC vs ASCII */
+
+/* Rest of these are attributes of Unicode and perl's internals rather than the encoding */
+
 #define isIDFIRST_lazy(p)      isIDFIRST_lazy_if(p,1)
 #define isALNUM_lazy(p)                isALNUM_lazy_if(p,1)
 
-/* EBCDIC-happy ways of converting native code to UTF8 */
+#define UTF8_MAXLEN 13 /* how wide can a single UTF8 encoded character become */
 
-#ifdef EBCDIC
-#define NATIVE_TO_ASCII(ch)      PL_e2a[(ch)&255]
-#define ASCII_TO_NATIVE(ch)      PL_a2e[(ch)&255]
-#define NATIVE_TO_UNI(ch)        (((ch) > 255) ? (ch) : (UV) PL_e2a[(ch)])
-#define UNI_TO_NATIVE(ch)        (((ch) > 255) ? (ch) : (UV) PL_a2e[(ch)])
-#define NATIVE_TO_NEED(enc,ch)   ((enc) ? NATIVE_TO_ASCII(ch) : (ch))
-#define ASCII_TO_NEED(enc,ch)    ((enc) ? (ch) : ASCII_TO_NATIVE(ch))
-#else
-#define NATIVE_TO_ASCII(ch)      (ch)
-#define ASCII_TO_NATIVE(ch)      (ch)
-#define UNI_TO_NATIVE(ch)        (ch)
-#define NATIVE_TO_UNI(ch)        (ch)
-#define NATIVE_TO_NEED(enc,ch)   (ch)
-#define ASCII_TO_NEED(enc,ch)    (ch)
-#endif
+/* #define IN_UTF8 (PL_curcop->op_private & HINT_UTF8) */
+#define IN_BYTE (PL_curcop->op_private & HINT_BYTE)
+#define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTE)
+
+#define UTF8_ALLOW_EMPTY               0x0001
+#define UTF8_ALLOW_CONTINUATION                0x0002
+#define UTF8_ALLOW_NON_CONTINUATION    0x0004
+#define UTF8_ALLOW_FE_FF               0x0008
+#define UTF8_ALLOW_SHORT               0x0010
+#define UTF8_ALLOW_SURROGATE           0x0020
+#define UTF8_ALLOW_BOM                 0x0040
+#define UTF8_ALLOW_FFFF                        0x0080
+#define UTF8_ALLOW_LONG                        0x0100
+#define UTF8_ALLOW_ANYUV               (UTF8_ALLOW_EMPTY|UTF8_ALLOW_FE_FF|\
+                                        UTF8_ALLOW_SURROGATE|UTF8_ALLOW_BOM|\
+                                        UTF8_ALLOW_FFFF|UTF8_ALLOW_LONG)
+#define UTF8_ALLOW_ANY                 0x00ff
+#define UTF8_CHECK_ONLY                        0x0100
+
+#define UNICODE_SURROGATE_FIRST                0xd800
+#define UNICODE_SURROGATE_LAST         0xdfff
+#define UNICODE_REPLACEMENT            0xfffd
+#define UNICODE_BYTER_ORDER_MARK       0xfffe
+#define UNICODE_ILLEGAL                        0xffff
+
+#define UNICODE_IS_SURROGATE(c)                ((c) >= UNICODE_SURROGATE_FIRST && \
+                                        (c) <= UNICODE_SURROGATE_LAST)
+#define UNICODE_IS_REPLACEMENT(c)      ((c) == UNICODE_REPLACMENT)
+#define UNICODE_IS_BYTE_ORDER_MARK(c)  ((c) == UNICODE_BYTER_ORDER_MARK)
+#define UNICODE_IS_ILLEGAL(c)          ((c) == UNICODE_ILLEGAL)
+
+#define UTF8_QUAD_MAX  UINT64_C(0x1000000000)
 
-#define UTF8_IS_INVARIANT(c) UTF8_IS_ASCII(c)
+#define UTF8_IS_ASCII(c) UTF8_IS_INVARIANT(c)

diff --git a/utfebcdic.h b/utfebcdic.h
new file mode 100644 (file)
index 0000000..efcd00d
--- /dev/null
+++ b/utfebcdic.h
@@ -0,0 +1,289 @@
+/*    utfebcdic.h
+ *
+ *    Copyright (c) 2001, Larry Wall, Nick Ing-Simmons
+ *
+ *    You may distribute under the terms of either the GNU General Public
+ *    License or the Artistic License, as specified in the README file.
+ *
+ * Macros to implement UTF-EBCDIC as perl's internal encoding
+ * Taken from version 7.1 of Unicode Techical Report #16:
+ *  http://www.unicode.org/unicode/reports/tr16
+ */
+
+START_EXTERN_C
+
+#ifdef DOINIT
+/* Indexed by encoded byte this table gives the length of the sequence.
+   Adapted from the shadow flags table in tr16.
+   The entries marked 9 are continuation bytes.
+*/
+EXTCONST unsigned char PL_utf8skip[] = {
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
+1,9,9,9,9,9,9,9,9,9,9,1,1,1,1,1,
+1,9,9,9,9,9,9,9,9,9,1,1,1,1,1,1,
+1,1,9,9,9,9,9,9,9,9,9,1,1,1,1,1,
+9,9,9,9,2,2,2,2,2,1,1,1,1,1,1,1,
+2,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,
+2,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2,
+2,1,1,1,1,1,1,1,1,1,2,2,2,1,2,2,
+2,2,2,2,2,2,2,3,3,3,3,3,3,1,3,3,
+1,1,1,1,1,1,1,1,1,1,3,3,3,3,3,3,
+1,1,1,1,1,1,1,1,1,1,3,3,4,4,4,4,
+1,4,1,1,1,1,1,1,1,1,4,4,4,5,5,5,
+1,1,1,1,1,1,1,1,1,1,5,6,6,7,7,1
+};
+
+/* Transform tables from tr16 applied after encoding to render encoding EBCDIC like */
+
+EXTCONST unsigned char PL_utf2e[] = { /* UTF-8-mod to EBCDIC (IBM-1047) */
+ 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+ 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F,
+ 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
+ 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
+ 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
+ 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xAD, 0xE0, 0xBD, 0x5F, 0x6D,
+ 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
+ 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07,
+ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B,
+ 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF,
+ 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56,
+ 0x57, 0x58, 0x59, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x70, 0x71, 0x72, 0x73,
+ 0x74, 0x75, 0x76, 0x77, 0x78, 0x80, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 0x90, 0x9A, 0x9B, 0x9C,
+ 0x9D, 0x9E, 0x9F, 0xA0, 0xAA, 0xAB, 0xAC, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6,
+ 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBE, 0xBF, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xDA, 0xDB,
+ 0xDC, 0xDD, 0xDE, 0xDF, 0xE1, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE,
+};
+
+EXTCONST unsigned char PL_e2utf[] = { /* EBCDIC to UTF-8-mod */
+ 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+ 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F,
+ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07,
+ 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A,
+ 0x20, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x2E, 0x3C, 0x28, 0x2B, 0x7C,
+ 0x26, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x5E,
+ 0x2D, 0x2F, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0x2C, 0x25, 0x5F, 0x3E, 0x3F,
+ 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22,
+ 0xC5, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB,
+ 0xCC, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2,
+ 0xD3, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xD4, 0xD5, 0xD6, 0x5B, 0xD7, 0xD8,
+ 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0x5D, 0xE6, 0xE7,
+ 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED,
+ 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3,
+ 0x5C, 0xF4, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA,
+ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 0x9F,
+};
+
+/* These tables moved from perl.h and converted to hex.
+   They map platfrom code page from/to bottom 256 codes of Unicode (i.e. iso-8859-1).
+*/
+
+#if '^' == 95   /* if defined(__MVS__) || defined(??) (VM/ESA?) 1047 */
+EXTCONST unsigned char PL_a2e[] = { /* ASCII (iso-8859-1) to EBCDIC (IBM-1047) */
+ 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+ 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F,
+ 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
+ 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
+ 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
+ 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xAD, 0xE0, 0xBD, 0x5F, 0x6D,
+ 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
+ 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07,
+ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B,
+ 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF,
+ 0x41, 0xAA, 0x4A, 0xB1, 0x9F, 0xB2, 0x6A, 0xB5, 0xBB, 0xB4, 0x9A, 0x8A, 0xB0, 0xCA, 0xAF, 0xBC,
+ 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB,
+ 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77,
+ 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, 0x80, 0xFD, 0xFE, 0xFB, 0xFC, 0xBA, 0xAE, 0x59,
+ 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57,
+ 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, 0x70, 0xDD, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF
+};
+
+EXTCONST unsigned char PL_e2a[] = { /* ASCII (iso-8859-1) to EBCDIC (IBM-1047) */
+ 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+ 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F,
+ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07,
+ 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A,
+ 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, 0xE7, 0xF1, 0xA2, 0x2E, 0x3C, 0x28, 0x2B, 0x7C,
+ 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x5E,
+ 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, 0xC7, 0xD1, 0xA6, 0x2C, 0x25, 0x5F, 0x3E, 0x3F,
+ 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, 0xCC, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22,
+ 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1,
+ 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4,
+ 0xB5, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0x5B, 0xDE, 0xAE,
+ 0xAC, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, 0xBD, 0xBE, 0xDD, 0xA8, 0xAF, 0x5D, 0xB4, 0xD7,
+ 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5,
+ 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xF9, 0xFA, 0xFF,
+ 0x5C, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5,
+ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xB3, 0xDB, 0xDC, 0xD9, 0xDA, 0x9F
+};
+#endif /* 1047 */
+
+#if '^' == 106  /* if defined(_OSD_POSIX) POSIX-BC */
+EXTCONST unsigned char PL_a2e[] = { /* ASCII (ISO8859-1) to EBCDIC (POSIX-BC) */
+ 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+ 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F,
+ 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
+ 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
+ 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
+ 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xBB, 0xBC, 0xBD, 0x6A, 0x6D,
+ 0x4A, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
+ 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xFB, 0x4F, 0xFD, 0xFF, 0x07,
+ 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B,
+ 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0x5F,
+ 0x41, 0xAA, 0xB0, 0xB1, 0x9F, 0xB2, 0xD0, 0xB5, 0x79, 0xB4, 0x9A, 0x8A, 0xBA, 0xCA, 0xAF, 0xA1,
+ 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB,
+ 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77,
+ 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, 0x80, 0xE0, 0xFE, 0xDD, 0xFC, 0xAD, 0xAE, 0x59,
+ 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57,
+ 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, 0x70, 0xC0, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF
+};
+
+EXTCONST unsigned char PL_e2a[] = { /* EBCDIC (POSIX-BC) to ASCII (ISO8859-1) */
+ 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+ 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F,
+ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07,
+ 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A,
+ 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, 0xE7, 0xF1, 0x60, 0x2E, 0x3C, 0x28, 0x2B, 0x7C,
+ 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x9F,
+ 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, 0xC7, 0xD1, 0x5E, 0x2C, 0x25, 0x5F, 0x3E, 0x3F,
+ 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, 0xCC, 0xA8, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22,
+ 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1,
+ 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4,
+ 0xB5, 0xAF, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0xDD, 0xDE, 0xAE,
+ 0xA2, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, 0xBD, 0xBE, 0xAC, 0x5B, 0x5C, 0x5D, 0xB4, 0xD7,
+ 0xF9, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5,
+ 0xA6, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xDB, 0xFA, 0xFF,
+ 0xD9, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5,
+ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xB3, 0x7B, 0xDC, 0x7D, 0xDA, 0x7E
+};
+#endif          /* POSIX-BC */
+#if '^' == 176  /* if defined(??) (OS/400?) 037 */
+EXTCONST unsigned char PL_a2e[] = { /* ASCII (ISO8859-1) to EBCDIC (IBM-037) */
+ 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x25, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+ 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F,
+ 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61,
+ 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F,
+ 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6,
+ 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D,
+ 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96,
+ 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07,
+ 0x20, 0x21, 0x22, 0x23, 0x24, 0x15, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B,
+ 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF,
+ 0x41, 0xAA, 0x4A, 0xB1, 0x9F, 0xB2, 0x6A, 0xB5, 0xBD, 0xB4, 0x9A, 0x8A, 0x5F, 0xCA, 0xAF, 0xBC,
+ 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB,
+ 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77,
+ 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, 0x80, 0xFD, 0xFE, 0xFB, 0xFC, 0xAD, 0xAE, 0x59,
+ 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57,
+ 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, 0x70, 0xDD, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF
+};
+
+EXTCONST unsigned char PL_e2a[] = { /* EBCDIC (IBM-037) to ASCII (ISO8859-1) */
+ 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
+ 0x10, 0x11, 0x12, 0x13, 0x9D, 0x85, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F,
+ 0x80, 0x81, 0x82, 0x83, 0x84, 0x0A, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07,
+ 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A,
+ 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, 0xE7, 0xF1, 0xA2, 0x2E, 0x3C, 0x28, 0x2B, 0x7C,
+ 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0xAC,
+ 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, 0xC7, 0xD1, 0xA6, 0x2C, 0x25, 0x5F, 0x3E, 0x3F,
+ 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, 0xCC, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22,
+ 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1,
+ 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4,
+ 0xB5, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0xDD, 0xDE, 0xAE,
+ 0x5E, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, 0xBD, 0xBE, 0x5B, 0x5D, 0xAF, 0xA8, 0xB4, 0xD7,
+ 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5,
+ 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xF9, 0xFA, 0xFF,
+ 0x5C, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5,
+ 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xB3, 0xDB, 0xDC, 0xD9, 0xDA, 0x9F
+};
+#endif          /* 037 */
+
+#else
+EXTCONST unsigned char PL_utf8skip[];
+EXTCONST unsigned char PL_e2utf[];
+EXTCONST unsigned char PL_utf2e[];
+EXTCONST unsigned char PL_e2a[];
+EXTCONST unsigned char PL_a2e[];
+#endif
+
+END_EXTERN_C
+
+#define UTF8SKIP(s) PL_utf8skip[*(U8*)s]
+
+/* EBCDIC-happy ways of converting native code to UTF8 */
+
+/* Native to iso-8859-1 */
+#define NATIVE_TO_ASCII(ch)      PL_e2a[(U8)(ch)]
+#define ASCII_TO_NATIVE(ch)      PL_a2e[(U8)(ch)]
+/* Transform after encoding */
+#define NATIVE_TO_UTF(ch)        PL_e2utf[(U8)(ch)]
+#define UTF_TO_NATIVE(ch)        PL_utf2e[(U8)(ch)]
+/* Transform in wide UV char space */
+#define NATIVE_TO_UNI(ch)        (((ch) > 255) ? (ch) : NATIVE_TO_ASCII(ch))
+#define UNI_TO_NATIVE(ch)        (((ch) > 255) ? (ch) : (UV) ASCII_TO_NATIVE(ch))
+/* Transform in invariant..byte space */
+#define NATIVE_TO_NEED(enc,ch)   ((enc) ? UTF_TO_NATIVE(NATIVE_TO_ASCII(ch)) : (ch))
+#define ASCII_TO_NEED(enc,ch)    ((enc) ? UTF_TO_NATIVE(ch) : ASCII_TO_NATIVE(ch))
+
+/*
+ * Note: we should try and be careful never to call the isXXX_utf8() functions
+ * unless we're pretty sure we've seen the beginning of a UTF-EBCDIC character
+ * Otherwise we risk loading in the heavy-duty SWASHINIT and SWASHGET routines
+ * unnecessarily.
+ */
+
+#define isIDFIRST_lazy_if(p,c) ((IN_BYTE || (!c || UTF8_INVARIANT(*p)) \
+                               ? isIDFIRST(*(p)) \
+                               : isIDFIRST_utf8((U8*)p))
+#define isALNUM_lazy_if(p,c)   ((IN_BYTE || (!c || UTF8_INVARIANT(*p)) \
+                               ? isALNUM(*(p)) \
+                               : isALNUM_utf8((U8*)p))
+
+/*
+  The following table is adapted from tr16, it shows UTF-8-mod encoding of Unicode code points.
+
+        Unicode                             Bit pattern 1st Byte 2nd Byte 3rd Byte 4th Byte 5th Byte 6th Byte 7th byte
+    U+0000..U+007F                     000000000xxxxxxx 0xxxxxxx
+    U+0080..U+009F                     00000000100xxxxx 100xxxxx
+    U+00A0..U+00FF                     00000000yyyxxxxx 11000yyy 101xxxxx
+
+    U+00A0..U+03FF                     000000yyyyyxxxxx 110yyyyy 101xxxxx
+    U+0400..U+3FFF                     00zzzzyyyyyxxxxx 1110zzzz 101yyyyy 101xxxxx
+    U+4000..U+3FFFF                 0wwwzzzzzyyyyyxxxxx 11110www 101zzzzz 101yyyyy 101xxxxx
+   U+40000..U+3FFFFF            0vvwwwwwzzzzzyyyyyxxxxx 111110vv 101wwwww 101zzzzz 101yyyyy 101xxxxx
+  U+400000..U+3FFFFFF       0uvvvvvwwwwwzzzzzyyyyyxxxxx 1111110u 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx
+ U+4000000..U+7FFFFFFF 0tuuuuuvvvvvwwwwwzzzzzyyyyyxxxxx 1111111t 101uuuuu 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx
+
+  Note: The UTF-8-Mod transformation is valid for UCS-4 values X'0' to
+  X'7FFFFFFF' (the full extent of ISO/IEC 10646 coding space).
+
+ */
+
+#define UNISKIP(uv) ( (uv) < 0xA0           ? 1 : \
+                     (uv) < 0x400          ? 2 : \
+                     (uv) < 0x4000         ? 3 : \
+                     (uv) < 0x40000        ? 4 : \
+                     (uv) < 0x400000       ? 5 : \
+                     (uv) < 0x4000000      ? 6 : 7 )
+
+/* UTF-EBCDIC sematic macros - transform back into UTF-8-Mod and then compare */
+#define UTF8_IS_INVARIANT(c)           (NATIVE_TO_UTF(c) <  0xA0)
+#define UTF8_IS_START(c)               (NATIVE_TO_UTF(c) >= 0xA0 && (NATIVE_TO_UTF(c) & 0xE0) != 0xA0)
+#define UTF8_IS_CONTINUATION(c)                (NATIVE_TO_UTF(c) >= 0xA0 && (NATIVE_TO_UTF(c) & 0xE0) == 0xA0)
+#define UTF8_IS_CONTINUED(c)           (NATIVE_TO_UTF(c) >= 0xA0)
+#define UTF8_IS_DOWNGRADEABLE_START(c) (NATIVE_TO_UTF(c) >= 0xA0 && (NATIVE_TO_UTF(c) & 0xF8) == 0xC0)
+
+#define UTF_START_MARK(len) ((len >  7) ? 0xFF : (0xFE << (7-len)))
+#define UTF_START_MASK(len) ((len >= 6) ? 0x01 : (0x1F >> (len-2)))
+#define UTF_CONTINUATION_MARK          0xA0
+#define UTF_CONTINUATION_MASK          ((U8)0x1f)
+#define UTF_ACCUMULATION_SHIFT         5
+
+#define UTF8_ACCUMULATE(old, new)      (((old) << UTF_ACCUMULATION_SHIFT)|(NATIVE_TO_UTF(new) & UTFE_CONTINUATION_MASK))
+
+/* UTF-EBCDIC encode a downgradeable value */
+#define UTF8_EIGHT_BIT_HI(c)   UTF_TO_NATIVE((((U8)(c))>>UTF_ACCUMULATION_SHIFT)|UTF_START_MARK(2))
+#define UTF8_EIGHT_BIT_LO(c)   UTF_TO_NATIVE(((((U8)(c)))&UTF_CONTINUATION_MASK)|UTF_CONTINUATION_MARK)
+
+