From: Nick Ing-Simmons Date: Sat, 17 Mar 2001 18:29:50 +0000 (+0000) Subject: Infrastructure to use UTF-EBCDIC rather than UTF-8 as the internal X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=1d72bdf6104ef56ab17c3abedf522be0125851c7;p=p5sagit%2Fp5-mst-13.2.git Infrastructure to use UTF-EBCDIC rather than UTF-8 as the internal encoding on EBCDIC platforms. This has property that U+0000..U+009F i.e. a superset of ASCII are invariant under the encoding. This is EBCDIC friendly as an encoded string can be looked at as being EBCDIC by lexer sprintf("%d",...) etc. in same manner that a UTF-8 string be considered ASCII on ASCII machines. - re-arrange utf8.h to get ASCII specific vs Unicode generic bits seperate. - Add some more macros to comprehend different shift amounts and possible swizzle in UTF-EBCDIC vs UTF-8. Change utf8.c to use them. - add utfebcdic.h which provides UTF-EBCDIC versions of the macros, and conditionally #include it. EBCDIC build as yet untested. ASCII still fails the one test. p4raw-id: //depot/perlio@9185 --- diff --git a/utf8.c b/utf8.c index 227dcba..71a76e3 100644 --- a/utf8.c +++ b/utf8.c @@ -46,10 +46,22 @@ is the recommended Unicode-aware way of saying U8 * Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv) { - if (uv < 0x80) { + if (UTF8_IS_INVARIANT(uv)) { *d++ = uv; return d; } +#if defined(EBCDIC) || 1 /* always for testing */ + else { + STRLEN len = UNISKIP(uv); + U8 *p = d+len-1; + while (p > d) { + *p-- = UTF_TO_NATIVE((uv & UTF_CONTINUATION_MASK) | UTF_CONTINUATION_MARK); + uv >>= UTF_ACCUMULATION_SHIFT; + } + *p = UTF_TO_NATIVE((uv & UTF_START_MASK(len)) | UTF_START_MARK(len)); + return d+len; + } +#else /* Non loop style */ if (uv < 0x800) { *d++ = (( uv >> 6) | 0xc0); *d++ = (( uv & 0x3f) | 0x80); @@ -116,6 +128,7 @@ Perl_uvuni_to_utf8(pTHX_ U8 *d, UV uv) return d; } #endif +#endif /* Loop style */ } /* @@ -148,7 +161,7 @@ Perl_uvchr_to_utf8(pTHX_ U8 *d, UV uv) =for apidoc A|STRLEN|is_utf8_char|U8 *s Tests if some arbitrary number of bytes begins in a valid UTF-8 -character. Note that an ASCII character is a valid UTF-8 character. +character. Note that an INVARIANT (i.e. ASCII) character is a valid UTF-8 character. The actual number of bytes in the UTF-8 character will be returned if it is valid, otherwise 0. @@ -160,7 +173,7 @@ Perl_is_utf8_char(pTHX_ U8 *s) STRLEN slen, len; UV uv, ouv; - if (UTF8_IS_ASCII(u)) + if (UTF8_IS_INVARIANT(u)) return 1; if (!UTF8_IS_START(u)) @@ -173,7 +186,8 @@ Perl_is_utf8_char(pTHX_ U8 *s) slen = len - 1; s++; - uv = u; + /* The initial value is dubious */ + uv = u; ouv = uv; while (slen--) { if (!UTF8_IS_CONTINUATION(*s)) @@ -276,7 +290,7 @@ Perl_utf8n_to_uvuni(pTHX_ U8* s, STRLEN curlen, STRLEN* retlen, U32 flags) goto malformed; } - if (UTF8_IS_ASCII(uv)) { + if (UTF8_IS_INVARIANT(uv)) { if (retlen) *retlen = 1; return (UV) (*s); @@ -294,20 +308,29 @@ Perl_utf8n_to_uvuni(pTHX_ U8* s, STRLEN curlen, STRLEN* retlen, U32 flags) goto malformed; } +#ifdef EBCDIC + uv = NATIVE_TO_UVF(uv); +#else if ((uv == 0xfe || uv == 0xff) && !(flags & UTF8_ALLOW_FE_FF)) { warning = UTF8_WARN_FE_FF; goto malformed; } - +#endif + if (!(uv & 0x20)) { len = 2; uv &= 0x1f; } else if (!(uv & 0x10)) { len = 3; uv &= 0x0f; } else if (!(uv & 0x08)) { len = 4; uv &= 0x07; } else if (!(uv & 0x04)) { len = 5; uv &= 0x03; } +#ifdef EBCDIC + else if (!(uv & 0x02)) { len = 6; uv &= 0x01; } + else { len = 7; uv &= 0x01; } +#else else if (!(uv & 0x02)) { len = 6; uv &= 0x01; } else if (!(uv & 0x01)) { len = 7; uv = 0; } - else { len = 13; uv = 0; } /* whoa! */ - + else { len = 13; uv = 0; } /* whoa! */ +#endif + if (retlen) *retlen = len; @@ -634,9 +657,9 @@ Perl_utf8_to_bytes(pTHX_ U8* s, STRLEN *len) for (send = s + *len; s < send; ) { U8 c = *s++; - if (c >= 0x80 && - ((s >= send) || - ((*s++ & 0xc0) != 0x80) || ((c & 0xfe) != 0xc2))) { + if (!UTF8_IS_INVARIANT(c) && + (!UTF8_IS_DOWNGRADEABLE_START(c) || (s >= send) + || !(c = *s++) || !UTF8_IS_CONTINUATION(c))) { *len = -1; return 0; } @@ -679,7 +702,7 @@ Perl_bytes_from_utf8(pTHX_ U8* s, STRLEN *len, bool *is_utf8) /* ensure valid UTF8 and chars < 256 before converting string */ for (send = s + *len; s < send;) { U8 c = *s++; - if (!UTF8_IS_ASCII(c)) { + if (!UTF8_IS_INVARIANT(c)) { if (UTF8_IS_DOWNGRADEABLE_START(c) && s < send && (c = *s++) && UTF8_IS_CONTINUATION(c)) count++; @@ -700,7 +723,7 @@ Perl_bytes_from_utf8(pTHX_ U8* s, STRLEN *len, bool *is_utf8) s = start; start = d; while (s < send) { U8 c = *s++; - if (!UTF8_IS_ASCII(c)) + if (!UTF8_IS_INVARIANT(c)) c = UTF8_ACCUMULATE(c, *s++); *d++ = ASCII_TO_NATIVE(c); } @@ -732,7 +755,7 @@ Perl_bytes_to_utf8(pTHX_ U8* s, STRLEN *len) while (s < send) { UV uv = NATIVE_TO_ASCII(*s++); - if (UTF8_IS_ASCII(uv)) + if (UTF8_IS_INVARIANT(uv)) *d++ = uv; else { *d++ = UTF8_EIGHT_BIT_HI(uv); diff --git a/utf8.h b/utf8.h index 67b3d53..a606397 100644 --- a/utf8.h +++ b/utf8.h @@ -7,6 +7,13 @@ * */ +#ifdef EBCDIC +/* The equivalent of these macros but implementing UTF-EBCDIC + are in the following header file: + */ + +#include "utfebcdic.h" +#else START_EXTERN_C #ifdef DOINIT @@ -26,47 +33,20 @@ EXTCONST unsigned char PL_utf8skip[]; #endif END_EXTERN_C - -#define UTF8_MAXLEN 13 /* how wide can a single UTF8 encoded character become */ - -/* #define IN_UTF8 (PL_curcop->op_private & HINT_UTF8) */ -#define IN_BYTE (PL_curcop->op_private & HINT_BYTE) -#ifdef USE_BYTES_DOWNGRADES -#define DO_UTF8(sv) (SvUTF8(sv) && !(IN_BYTE && sv_utf8_downgrade(sv,0))) -#else -#define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTE) -#endif - -#define UTF8_ALLOW_EMPTY 0x0001 -#define UTF8_ALLOW_CONTINUATION 0x0002 -#define UTF8_ALLOW_NON_CONTINUATION 0x0004 -#define UTF8_ALLOW_FE_FF 0x0008 -#define UTF8_ALLOW_SHORT 0x0010 -#define UTF8_ALLOW_SURROGATE 0x0020 -#define UTF8_ALLOW_BOM 0x0040 -#define UTF8_ALLOW_FFFF 0x0080 -#define UTF8_ALLOW_LONG 0x0100 -#define UTF8_ALLOW_ANYUV (UTF8_ALLOW_EMPTY|UTF8_ALLOW_FE_FF|\ - UTF8_ALLOW_SURROGATE|UTF8_ALLOW_BOM|\ - UTF8_ALLOW_FFFF|UTF8_ALLOW_LONG) -#define UTF8_ALLOW_ANY 0x00ff -#define UTF8_CHECK_ONLY 0x0100 - -#define UNICODE_SURROGATE_FIRST 0xd800 -#define UNICODE_SURROGATE_LAST 0xdfff -#define UNICODE_REPLACEMENT 0xfffd -#define UNICODE_BYTER_ORDER_MARK 0xfffe -#define UNICODE_ILLEGAL 0xffff - -#define UNICODE_IS_SURROGATE(c) ((c) >= UNICODE_SURROGATE_FIRST && \ - (c) <= UNICODE_SURROGATE_LAST) -#define UNICODE_IS_REPLACEMENT(c) ((c) == UNICODE_REPLACMENT) -#define UNICODE_IS_BYTE_ORDER_MARK(c) ((c) == UNICODE_BYTER_ORDER_MARK) -#define UNICODE_IS_ILLEGAL(c) ((c) == UNICODE_ILLEGAL) - #define UTF8SKIP(s) PL_utf8skip[*(U8*)s] -#define UTF8_QUAD_MAX UINT64_C(0x1000000000) +/* Native character to iso-8859-1 */ +#define NATIVE_TO_ASCII(ch) (ch) +#define ASCII_TO_NATIVE(ch) (ch) +/* Transform after encoding */ +#define NATIVE_TO_UTF(ch) (ch) +#define UTF_TO_NATIVE(ch) (ch) +/* Transforms in wide UV chars */ +#define UNI_TO_NATIVE(ch) (ch) +#define NATIVE_TO_UNI(ch) (ch) +/* Transforms in invariant space */ +#define NATIVE_TO_NEED(enc,ch) (ch) +#define ASCII_TO_NEED(enc,ch) (ch) /* @@ -84,18 +64,22 @@ END_EXTERN_C */ -#define UTF8_IS_ASCII(c) (((U8)c) < 0x80) +#define UTF8_IS_INVARIANT(c) (((UV)c) < 0x80) #define UTF8_IS_START(c) (((U8)c) >= 0xc0 && (((U8)c) <= 0xfd)) #define UTF8_IS_CONTINUATION(c) (((U8)c) >= 0x80 && (((U8)c) <= 0xbf)) #define UTF8_IS_CONTINUED(c) (((U8)c) & 0x80) #define UTF8_IS_DOWNGRADEABLE_START(c) (((U8)c & 0xfc) == 0xc0) -#define UTF8_CONTINUATION_MASK ((U8)0x3f) -#define UTF8_ACCUMULATION_SHIFT 6 -#define UTF8_ACCUMULATE(old, new) ((old) << UTF8_ACCUMULATION_SHIFT | (((U8)new) & UTF8_CONTINUATION_MASK)) +#define UTF_START_MARK(len) ((len > 7) ? 0xFF : (0xFE << (7-len))) +#define UTF_START_MASK(len) ((len >= 7) ? 0x00 : (0x1F >> (len-2))) + +#define UTF_CONTINUATION_MARK 0x80 +#define UTF_ACCUMULATION_SHIFT 6 +#define UTF_CONTINUATION_MASK ((U8)0x3f) +#define UTF8_ACCUMULATE(old, new) ((old) << UTF_ACCUMULATION_SHIFT | (((U8)new) & UTF_CONTINUATION_MASK)) -#define UTF8_EIGHT_BIT_HI(c) ( (((U8)(c))>>6) |0xc0) -#define UTF8_EIGHT_BIT_LO(c) (((((U8)(c)) )&0x3f)|0x80) +#define UTF8_EIGHT_BIT_HI(c) ((((U8)(c))>>UTF_ACCUMULATION_SHIFT)|UTF_START_MARK(2)) +#define UTF8_EIGHT_BIT_LO(c) (((((U8)(c)))&UTF_CONTINUATION_MASK)|UTF_CONTINUATION_MARK) #ifdef HAS_QUAD #define UNISKIP(uv) ( (uv) < 0x80 ? 1 : \ @@ -115,43 +99,60 @@ END_EXTERN_C (uv) < 0x80000000 ? 6 : 7 ) #endif - /* * Note: we try to be careful never to call the isXXX_utf8() functions * unless we're pretty sure we've seen the beginning of a UTF-8 character * (that is, the two high bits are set). Otherwise we risk loading in the * heavy-duty SWASHINIT and SWASHGET routines unnecessarily. */ -#ifdef EBCDIC -#define isIDFIRST_lazy_if(p,c) isIDFIRST(*(p)) -#define isALNUM_lazy_if(p,c) isALNUM(*(p)) -#else #define isIDFIRST_lazy_if(p,c) ((IN_BYTE || (!c || (*((U8*)p) < 0xc0))) \ ? isIDFIRST(*(p)) \ : isIDFIRST_utf8((U8*)p)) #define isALNUM_lazy_if(p,c) ((IN_BYTE || (!c || (*((U8*)p) < 0xc0))) \ ? isALNUM(*(p)) \ : isALNUM_utf8((U8*)p)) -#endif + + +#endif /* EBCDIC vs ASCII */ + +/* Rest of these are attributes of Unicode and perl's internals rather than the encoding */ + #define isIDFIRST_lazy(p) isIDFIRST_lazy_if(p,1) #define isALNUM_lazy(p) isALNUM_lazy_if(p,1) -/* EBCDIC-happy ways of converting native code to UTF8 */ +#define UTF8_MAXLEN 13 /* how wide can a single UTF8 encoded character become */ -#ifdef EBCDIC -#define NATIVE_TO_ASCII(ch) PL_e2a[(ch)&255] -#define ASCII_TO_NATIVE(ch) PL_a2e[(ch)&255] -#define NATIVE_TO_UNI(ch) (((ch) > 255) ? (ch) : (UV) PL_e2a[(ch)]) -#define UNI_TO_NATIVE(ch) (((ch) > 255) ? (ch) : (UV) PL_a2e[(ch)]) -#define NATIVE_TO_NEED(enc,ch) ((enc) ? NATIVE_TO_ASCII(ch) : (ch)) -#define ASCII_TO_NEED(enc,ch) ((enc) ? (ch) : ASCII_TO_NATIVE(ch)) -#else -#define NATIVE_TO_ASCII(ch) (ch) -#define ASCII_TO_NATIVE(ch) (ch) -#define UNI_TO_NATIVE(ch) (ch) -#define NATIVE_TO_UNI(ch) (ch) -#define NATIVE_TO_NEED(enc,ch) (ch) -#define ASCII_TO_NEED(enc,ch) (ch) -#endif +/* #define IN_UTF8 (PL_curcop->op_private & HINT_UTF8) */ +#define IN_BYTE (PL_curcop->op_private & HINT_BYTE) +#define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTE) + +#define UTF8_ALLOW_EMPTY 0x0001 +#define UTF8_ALLOW_CONTINUATION 0x0002 +#define UTF8_ALLOW_NON_CONTINUATION 0x0004 +#define UTF8_ALLOW_FE_FF 0x0008 +#define UTF8_ALLOW_SHORT 0x0010 +#define UTF8_ALLOW_SURROGATE 0x0020 +#define UTF8_ALLOW_BOM 0x0040 +#define UTF8_ALLOW_FFFF 0x0080 +#define UTF8_ALLOW_LONG 0x0100 +#define UTF8_ALLOW_ANYUV (UTF8_ALLOW_EMPTY|UTF8_ALLOW_FE_FF|\ + UTF8_ALLOW_SURROGATE|UTF8_ALLOW_BOM|\ + UTF8_ALLOW_FFFF|UTF8_ALLOW_LONG) +#define UTF8_ALLOW_ANY 0x00ff +#define UTF8_CHECK_ONLY 0x0100 + +#define UNICODE_SURROGATE_FIRST 0xd800 +#define UNICODE_SURROGATE_LAST 0xdfff +#define UNICODE_REPLACEMENT 0xfffd +#define UNICODE_BYTER_ORDER_MARK 0xfffe +#define UNICODE_ILLEGAL 0xffff + +#define UNICODE_IS_SURROGATE(c) ((c) >= UNICODE_SURROGATE_FIRST && \ + (c) <= UNICODE_SURROGATE_LAST) +#define UNICODE_IS_REPLACEMENT(c) ((c) == UNICODE_REPLACMENT) +#define UNICODE_IS_BYTE_ORDER_MARK(c) ((c) == UNICODE_BYTER_ORDER_MARK) +#define UNICODE_IS_ILLEGAL(c) ((c) == UNICODE_ILLEGAL) + +#define UTF8_QUAD_MAX UINT64_C(0x1000000000) -#define UTF8_IS_INVARIANT(c) UTF8_IS_ASCII(c) +#define UTF8_IS_ASCII(c) UTF8_IS_INVARIANT(c) diff --git a/utfebcdic.h b/utfebcdic.h new file mode 100644 index 0000000..efcd00d --- /dev/null +++ b/utfebcdic.h @@ -0,0 +1,289 @@ +/* utfebcdic.h + * + * Copyright (c) 2001, Larry Wall, Nick Ing-Simmons + * + * You may distribute under the terms of either the GNU General Public + * License or the Artistic License, as specified in the README file. + * + * Macros to implement UTF-EBCDIC as perl's internal encoding + * Taken from version 7.1 of Unicode Techical Report #16: + * http://www.unicode.org/unicode/reports/tr16 + */ + +START_EXTERN_C + +#ifdef DOINIT +/* Indexed by encoded byte this table gives the length of the sequence. + Adapted from the shadow flags table in tr16. + The entries marked 9 are continuation bytes. +*/ +EXTCONST unsigned char PL_utf8skip[] = { +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, +1,9,9,9,9,9,9,9,9,9,9,1,1,1,1,1, +1,9,9,9,9,9,9,9,9,9,1,1,1,1,1,1, +1,1,9,9,9,9,9,9,9,9,9,1,1,1,1,1, +9,9,9,9,2,2,2,2,2,1,1,1,1,1,1,1, +2,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2, +2,1,1,1,1,1,1,1,1,1,2,2,2,2,2,2, +2,1,1,1,1,1,1,1,1,1,2,2,2,1,2,2, +2,2,2,2,2,2,2,3,3,3,3,3,3,1,3,3, +1,1,1,1,1,1,1,1,1,1,3,3,3,3,3,3, +1,1,1,1,1,1,1,1,1,1,3,3,4,4,4,4, +1,4,1,1,1,1,1,1,1,1,4,4,4,5,5,5, +1,1,1,1,1,1,1,1,1,1,5,6,6,7,7,1 +}; + +/* Transform tables from tr16 applied after encoding to render encoding EBCDIC like */ + +EXTCONST unsigned char PL_utf2e[] = { /* UTF-8-mod to EBCDIC (IBM-1047) */ + 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, + 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, + 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, + 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xAD, 0xE0, 0xBD, 0x5F, 0x6D, + 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, + 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, + 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF, + 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x4A, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, + 0x57, 0x58, 0x59, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x6A, 0x70, 0x71, 0x72, 0x73, + 0x74, 0x75, 0x76, 0x77, 0x78, 0x80, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 0x90, 0x9A, 0x9B, 0x9C, + 0x9D, 0x9E, 0x9F, 0xA0, 0xAA, 0xAB, 0xAC, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, + 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBE, 0xBF, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xDA, 0xDB, + 0xDC, 0xDD, 0xDE, 0xDF, 0xE1, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, +}; + +EXTCONST unsigned char PL_e2utf[] = { /* EBCDIC to UTF-8-mod */ + 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, + 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, + 0x20, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, + 0x26, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x5E, + 0x2D, 0x2F, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, + 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, + 0xC5, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xCB, + 0xCC, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, + 0xD3, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xD4, 0xD5, 0xD6, 0x5B, 0xD7, 0xD8, + 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0x5D, 0xE6, 0xE7, + 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, + 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3, + 0x5C, 0xF4, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 0x9F, +}; + +/* These tables moved from perl.h and converted to hex. + They map platfrom code page from/to bottom 256 codes of Unicode (i.e. iso-8859-1). +*/ + +#if '^' == 95 /* if defined(__MVS__) || defined(??) (VM/ESA?) 1047 */ +EXTCONST unsigned char PL_a2e[] = { /* ASCII (iso-8859-1) to EBCDIC (IBM-1047) */ + 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, + 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, + 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, + 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xAD, 0xE0, 0xBD, 0x5F, 0x6D, + 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, + 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, + 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF, + 0x41, 0xAA, 0x4A, 0xB1, 0x9F, 0xB2, 0x6A, 0xB5, 0xBB, 0xB4, 0x9A, 0x8A, 0xB0, 0xCA, 0xAF, 0xBC, + 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB, + 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77, + 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, 0x80, 0xFD, 0xFE, 0xFB, 0xFC, 0xBA, 0xAE, 0x59, + 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57, + 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, 0x70, 0xDD, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF +}; + +EXTCONST unsigned char PL_e2a[] = { /* ASCII (iso-8859-1) to EBCDIC (IBM-1047) */ + 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, + 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, + 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, 0xE7, 0xF1, 0xA2, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, + 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x5E, + 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, 0xC7, 0xD1, 0xA6, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, + 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, 0xCC, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, + 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1, + 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4, + 0xB5, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0x5B, 0xDE, 0xAE, + 0xAC, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, 0xBD, 0xBE, 0xDD, 0xA8, 0xAF, 0x5D, 0xB4, 0xD7, + 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5, + 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xF9, 0xFA, 0xFF, + 0x5C, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xB3, 0xDB, 0xDC, 0xD9, 0xDA, 0x9F +}; +#endif /* 1047 */ + +#if '^' == 106 /* if defined(_OSD_POSIX) POSIX-BC */ +EXTCONST unsigned char PL_a2e[] = { /* ASCII (ISO8859-1) to EBCDIC (POSIX-BC) */ + 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x15, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, + 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, + 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, + 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xBB, 0xBC, 0xBD, 0x6A, 0x6D, + 0x4A, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, + 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xFB, 0x4F, 0xFD, 0xFF, 0x07, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x25, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, + 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0x5F, + 0x41, 0xAA, 0xB0, 0xB1, 0x9F, 0xB2, 0xD0, 0xB5, 0x79, 0xB4, 0x9A, 0x8A, 0xBA, 0xCA, 0xAF, 0xA1, + 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB, + 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77, + 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, 0x80, 0xE0, 0xFE, 0xDD, 0xFC, 0xAD, 0xAE, 0x59, + 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57, + 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, 0x70, 0xC0, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF +}; + +EXTCONST unsigned char PL_e2a[] = { /* EBCDIC (POSIX-BC) to ASCII (ISO8859-1) */ + 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x9D, 0x0A, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, + 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, + 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, 0xE7, 0xF1, 0x60, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, + 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0x9F, + 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, 0xC7, 0xD1, 0x5E, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, + 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, 0xCC, 0xA8, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, + 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1, + 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4, + 0xB5, 0xAF, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0xDD, 0xDE, 0xAE, + 0xA2, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, 0xBD, 0xBE, 0xAC, 0x5B, 0x5C, 0x5D, 0xB4, 0xD7, + 0xF9, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5, + 0xA6, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xDB, 0xFA, 0xFF, + 0xD9, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xB3, 0x7B, 0xDC, 0x7D, 0xDA, 0x7E +}; +#endif /* POSIX-BC */ +#if '^' == 176 /* if defined(??) (OS/400?) 037 */ +EXTCONST unsigned char PL_a2e[] = { /* ASCII (ISO8859-1) to EBCDIC (IBM-037) */ + 0x00, 0x01, 0x02, 0x03, 0x37, 0x2D, 0x2E, 0x2F, 0x16, 0x05, 0x25, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x3C, 0x3D, 0x32, 0x26, 0x18, 0x19, 0x3F, 0x27, 0x1C, 0x1D, 0x1E, 0x1F, + 0x40, 0x5A, 0x7F, 0x7B, 0x5B, 0x6C, 0x50, 0x7D, 0x4D, 0x5D, 0x5C, 0x4E, 0x6B, 0x60, 0x4B, 0x61, + 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0x7A, 0x5E, 0x4C, 0x7E, 0x6E, 0x6F, + 0x7C, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, + 0xD7, 0xD8, 0xD9, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xBA, 0xE0, 0xBB, 0xB0, 0x6D, + 0x79, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, + 0x97, 0x98, 0x99, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xC0, 0x4F, 0xD0, 0xA1, 0x07, + 0x20, 0x21, 0x22, 0x23, 0x24, 0x15, 0x06, 0x17, 0x28, 0x29, 0x2A, 0x2B, 0x2C, 0x09, 0x0A, 0x1B, + 0x30, 0x31, 0x1A, 0x33, 0x34, 0x35, 0x36, 0x08, 0x38, 0x39, 0x3A, 0x3B, 0x04, 0x14, 0x3E, 0xFF, + 0x41, 0xAA, 0x4A, 0xB1, 0x9F, 0xB2, 0x6A, 0xB5, 0xBD, 0xB4, 0x9A, 0x8A, 0x5F, 0xCA, 0xAF, 0xBC, + 0x90, 0x8F, 0xEA, 0xFA, 0xBE, 0xA0, 0xB6, 0xB3, 0x9D, 0xDA, 0x9B, 0x8B, 0xB7, 0xB8, 0xB9, 0xAB, + 0x64, 0x65, 0x62, 0x66, 0x63, 0x67, 0x9E, 0x68, 0x74, 0x71, 0x72, 0x73, 0x78, 0x75, 0x76, 0x77, + 0xAC, 0x69, 0xED, 0xEE, 0xEB, 0xEF, 0xEC, 0xBF, 0x80, 0xFD, 0xFE, 0xFB, 0xFC, 0xAD, 0xAE, 0x59, + 0x44, 0x45, 0x42, 0x46, 0x43, 0x47, 0x9C, 0x48, 0x54, 0x51, 0x52, 0x53, 0x58, 0x55, 0x56, 0x57, + 0x8C, 0x49, 0xCD, 0xCE, 0xCB, 0xCF, 0xCC, 0xE1, 0x70, 0xDD, 0xDE, 0xDB, 0xDC, 0x8D, 0x8E, 0xDF +}; + +EXTCONST unsigned char PL_e2a[] = { /* EBCDIC (IBM-037) to ASCII (ISO8859-1) */ + 0x00, 0x01, 0x02, 0x03, 0x9C, 0x09, 0x86, 0x7F, 0x97, 0x8D, 0x8E, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F, + 0x10, 0x11, 0x12, 0x13, 0x9D, 0x85, 0x08, 0x87, 0x18, 0x19, 0x92, 0x8F, 0x1C, 0x1D, 0x1E, 0x1F, + 0x80, 0x81, 0x82, 0x83, 0x84, 0x0A, 0x17, 0x1B, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x05, 0x06, 0x07, + 0x90, 0x91, 0x16, 0x93, 0x94, 0x95, 0x96, 0x04, 0x98, 0x99, 0x9A, 0x9B, 0x14, 0x15, 0x9E, 0x1A, + 0x20, 0xA0, 0xE2, 0xE4, 0xE0, 0xE1, 0xE3, 0xE5, 0xE7, 0xF1, 0xA2, 0x2E, 0x3C, 0x28, 0x2B, 0x7C, + 0x26, 0xE9, 0xEA, 0xEB, 0xE8, 0xED, 0xEE, 0xEF, 0xEC, 0xDF, 0x21, 0x24, 0x2A, 0x29, 0x3B, 0xAC, + 0x2D, 0x2F, 0xC2, 0xC4, 0xC0, 0xC1, 0xC3, 0xC5, 0xC7, 0xD1, 0xA6, 0x2C, 0x25, 0x5F, 0x3E, 0x3F, + 0xF8, 0xC9, 0xCA, 0xCB, 0xC8, 0xCD, 0xCE, 0xCF, 0xCC, 0x60, 0x3A, 0x23, 0x40, 0x27, 0x3D, 0x22, + 0xD8, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0xAB, 0xBB, 0xF0, 0xFD, 0xFE, 0xB1, + 0xB0, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 0x70, 0x71, 0x72, 0xAA, 0xBA, 0xE6, 0xB8, 0xC6, 0xA4, + 0xB5, 0x7E, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x7A, 0xA1, 0xBF, 0xD0, 0xDD, 0xDE, 0xAE, + 0x5E, 0xA3, 0xA5, 0xB7, 0xA9, 0xA7, 0xB6, 0xBC, 0xBD, 0xBE, 0x5B, 0x5D, 0xAF, 0xA8, 0xB4, 0xD7, + 0x7B, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0xAD, 0xF4, 0xF6, 0xF2, 0xF3, 0xF5, + 0x7D, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 0x50, 0x51, 0x52, 0xB9, 0xFB, 0xFC, 0xF9, 0xFA, 0xFF, + 0x5C, 0xF7, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x5A, 0xB2, 0xD4, 0xD6, 0xD2, 0xD3, 0xD5, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0xB3, 0xDB, 0xDC, 0xD9, 0xDA, 0x9F +}; +#endif /* 037 */ + +#else +EXTCONST unsigned char PL_utf8skip[]; +EXTCONST unsigned char PL_e2utf[]; +EXTCONST unsigned char PL_utf2e[]; +EXTCONST unsigned char PL_e2a[]; +EXTCONST unsigned char PL_a2e[]; +#endif + +END_EXTERN_C + +#define UTF8SKIP(s) PL_utf8skip[*(U8*)s] + +/* EBCDIC-happy ways of converting native code to UTF8 */ + +/* Native to iso-8859-1 */ +#define NATIVE_TO_ASCII(ch) PL_e2a[(U8)(ch)] +#define ASCII_TO_NATIVE(ch) PL_a2e[(U8)(ch)] +/* Transform after encoding */ +#define NATIVE_TO_UTF(ch) PL_e2utf[(U8)(ch)] +#define UTF_TO_NATIVE(ch) PL_utf2e[(U8)(ch)] +/* Transform in wide UV char space */ +#define NATIVE_TO_UNI(ch) (((ch) > 255) ? (ch) : NATIVE_TO_ASCII(ch)) +#define UNI_TO_NATIVE(ch) (((ch) > 255) ? (ch) : (UV) ASCII_TO_NATIVE(ch)) +/* Transform in invariant..byte space */ +#define NATIVE_TO_NEED(enc,ch) ((enc) ? UTF_TO_NATIVE(NATIVE_TO_ASCII(ch)) : (ch)) +#define ASCII_TO_NEED(enc,ch) ((enc) ? UTF_TO_NATIVE(ch) : ASCII_TO_NATIVE(ch)) + +/* + * Note: we should try and be careful never to call the isXXX_utf8() functions + * unless we're pretty sure we've seen the beginning of a UTF-EBCDIC character + * Otherwise we risk loading in the heavy-duty SWASHINIT and SWASHGET routines + * unnecessarily. + */ + +#define isIDFIRST_lazy_if(p,c) ((IN_BYTE || (!c || UTF8_INVARIANT(*p)) \ + ? isIDFIRST(*(p)) \ + : isIDFIRST_utf8((U8*)p)) +#define isALNUM_lazy_if(p,c) ((IN_BYTE || (!c || UTF8_INVARIANT(*p)) \ + ? isALNUM(*(p)) \ + : isALNUM_utf8((U8*)p)) + +/* + The following table is adapted from tr16, it shows UTF-8-mod encoding of Unicode code points. + + Unicode Bit pattern 1st Byte 2nd Byte 3rd Byte 4th Byte 5th Byte 6th Byte 7th byte + U+0000..U+007F 000000000xxxxxxx 0xxxxxxx + U+0080..U+009F 00000000100xxxxx 100xxxxx + U+00A0..U+00FF 00000000yyyxxxxx 11000yyy 101xxxxx + + U+00A0..U+03FF 000000yyyyyxxxxx 110yyyyy 101xxxxx + U+0400..U+3FFF 00zzzzyyyyyxxxxx 1110zzzz 101yyyyy 101xxxxx + U+4000..U+3FFFF 0wwwzzzzzyyyyyxxxxx 11110www 101zzzzz 101yyyyy 101xxxxx + U+40000..U+3FFFFF 0vvwwwwwzzzzzyyyyyxxxxx 111110vv 101wwwww 101zzzzz 101yyyyy 101xxxxx + U+400000..U+3FFFFFF 0uvvvvvwwwwwzzzzzyyyyyxxxxx 1111110u 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx + U+4000000..U+7FFFFFFF 0tuuuuuvvvvvwwwwwzzzzzyyyyyxxxxx 1111111t 101uuuuu 101vvvvv 101wwwww 101zzzzz 101yyyyy 101xxxxx + + Note: The UTF-8-Mod transformation is valid for UCS-4 values X'0' to + X'7FFFFFFF' (the full extent of ISO/IEC 10646 coding space). + + */ + +#define UNISKIP(uv) ( (uv) < 0xA0 ? 1 : \ + (uv) < 0x400 ? 2 : \ + (uv) < 0x4000 ? 3 : \ + (uv) < 0x40000 ? 4 : \ + (uv) < 0x400000 ? 5 : \ + (uv) < 0x4000000 ? 6 : 7 ) + +/* UTF-EBCDIC sematic macros - transform back into UTF-8-Mod and then compare */ +#define UTF8_IS_INVARIANT(c) (NATIVE_TO_UTF(c) < 0xA0) +#define UTF8_IS_START(c) (NATIVE_TO_UTF(c) >= 0xA0 && (NATIVE_TO_UTF(c) & 0xE0) != 0xA0) +#define UTF8_IS_CONTINUATION(c) (NATIVE_TO_UTF(c) >= 0xA0 && (NATIVE_TO_UTF(c) & 0xE0) == 0xA0) +#define UTF8_IS_CONTINUED(c) (NATIVE_TO_UTF(c) >= 0xA0) +#define UTF8_IS_DOWNGRADEABLE_START(c) (NATIVE_TO_UTF(c) >= 0xA0 && (NATIVE_TO_UTF(c) & 0xF8) == 0xC0) + +#define UTF_START_MARK(len) ((len > 7) ? 0xFF : (0xFE << (7-len))) +#define UTF_START_MASK(len) ((len >= 6) ? 0x01 : (0x1F >> (len-2))) +#define UTF_CONTINUATION_MARK 0xA0 +#define UTF_CONTINUATION_MASK ((U8)0x1f) +#define UTF_ACCUMULATION_SHIFT 5 + +#define UTF8_ACCUMULATE(old, new) (((old) << UTF_ACCUMULATION_SHIFT)|(NATIVE_TO_UTF(new) & UTFE_CONTINUATION_MASK)) + +/* UTF-EBCDIC encode a downgradeable value */ +#define UTF8_EIGHT_BIT_HI(c) UTF_TO_NATIVE((((U8)(c))>>UTF_ACCUMULATION_SHIFT)|UTF_START_MARK(2)) +#define UTF8_EIGHT_BIT_LO(c) UTF_TO_NATIVE(((((U8)(c)))&UTF_CONTINUATION_MASK)|UTF_CONTINUATION_MARK) + +