From: Peter Prymmer Date: Thu, 8 Mar 2001 16:23:25 +0000 (-0800) Subject: Re: Unicode/EBCDIC X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=3bd709b1a63d554f3d98d5394be78ed628eb46da;p=p5sagit%2Fp5-mst-13.2.git Re: Unicode/EBCDIC Message-ID: p4raw-id: //depot/perl@9082 --- diff --git a/perl.c b/perl.c index 03b2ec1..0920a41 100644 --- a/perl.c +++ b/perl.c @@ -253,9 +253,9 @@ perl_construct(pTHXx) if (PERL_REVISION > 127 || PERL_VERSION > 127 || PERL_SUBVERSION > 127) SvGROW(PL_patchlevel, UTF8_MAXLEN*3+1); s = (U8*)SvPVX(PL_patchlevel); - s = uv_to_utf8(s, (UV)PERL_REVISION); - s = uv_to_utf8(s, (UV)PERL_VERSION); - s = uv_to_utf8(s, (UV)PERL_SUBVERSION); + s = uv_to_utf8(s, (UV)(ASCII_TO_NATIVE(PERL_REVISION))); + s = uv_to_utf8(s, (UV)(ASCII_TO_NATIVE(PERL_VERSION))); + s = uv_to_utf8(s, (UV)(ASCII_TO_NATIVE(PERL_SUBVERSION))); *s = '\0'; SvCUR_set(PL_patchlevel, s - (U8*)SvPVX(PL_patchlevel)); SvPOK_on(PL_patchlevel); diff --git a/perl.h b/perl.h index 2b66473..4ee33cc 100644 --- a/perl.h +++ b/perl.h @@ -2436,10 +2436,220 @@ EXT char *PL_sig_name[]; EXT int PL_sig_num[]; #endif -/* fast case folding tables */ +/* fast conversion and case folding tables */ #ifdef DOINIT #ifdef EBCDIC +#if '^' == 106 /* if defined(_OSD_POSIX) POSIX-BC */ +EXT unsigned char PL_e2a[] = { /* ASCII (ISO8859-1) to EBCDIC (POSIX-BC) */ + 0, 1, 2, 3, 55, 45, 46, 47, + 22, 5, 21, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 60, 61, 50, 38, + 24, 25, 63, 39, 28, 29, 30, 31, + 64, 90, 127, 123, 91, 108, 80, 125, + 77, 93, 92, 78, 107, 96, 75, 97, + 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 122, 94, 76, 126, 110, 111, + 124, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 209, 210, 211, 212, 213, 214, + 215, 216, 217, 226, 227, 228, 229, 230, + 231, 232, 233, 187, 188, 189, 106, 109, + 74, 129, 130, 131, 132, 133, 134, 135, + 136, 137, 145, 146, 147, 148, 149, 150, + 151, 152, 153, 162, 163, 164, 165, 166, + 167, 168, 169, 251, 79, 253, 255, 7, + 32, 33, 34, 35, 36, 37, 6, 23, + 40, 41, 42, 43, 44, 9, 10, 27, + 48, 49, 26, 51, 52, 53, 54, 8, + 56, 57, 58, 59, 4, 20, 62, 95, + 65, 170, 176, 177, 159, 178, 208, 181, + 121, 180, 154, 138, 186, 202, 175, 161, + 144, 143, 234, 250, 190, 160, 182, 179, + 157, 218, 155, 139, 183, 184, 185, 171, + 100, 101, 98, 102, 99, 103, 158, 104, + 116, 113, 114, 115, 120, 117, 118, 119, + 172, 105, 237, 238, 235, 239, 236, 191, + 128, 224, 254, 221, 252, 173, 174, 89, + 68, 69, 66, 70, 67, 71, 156, 72, + 84, 81, 82, 83, 88, 85, 86, 87, + 140, 73, 205, 206, 203, 207, 204, 225, + 112, 192, 222, 219, 220, 141, 142, 223 +}; +EXT unsigned char PL_a2e[] = { /* EBCDIC (POSIX-BC) to ASCII (ISO8859-1) */ + 0, 1, 2, 3, 156, 9, 134, 127, + 151, 141, 142, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 157, 10, 8, 135, + 24, 25, 146, 143, 28, 29, 30, 31, + 128, 129, 130, 131, 132, 133, 23, 27, + 136, 137, 138, 139, 140, 5, 6, 7, + 144, 145, 22, 147, 148, 149, 150, 4, + 152, 153, 154, 155, 20, 21, 158, 26, + 32, 160, 226, 228, 224, 225, 227, 229, + 231, 241, 96, 46, 60, 40, 43, 124, + 38, 233, 234, 235, 232, 237, 238, 239, + 236, 223, 33, 36, 42, 41, 59, 159, + 45, 47, 194, 196, 192, 193, 195, 197, + 199, 209, 94, 44, 37, 95, 62, 63, + 248, 201, 202, 203, 200, 205, 206, 207, + 204, 168, 58, 35, 64, 39, 61, 34, + 216, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 171, 187, 240, 253, 254, 177, + 176, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 170, 186, 230, 184, 198, 164, + 181, 175, 115, 116, 117, 118, 119, 120, + 121, 122, 161, 191, 208, 221, 222, 174, + 162, 163, 165, 183, 169, 167, 182, 188, + 189, 190, 172, 91, 92, 93, 180, 215, + 249, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 173, 244, 246, 242, 243, 245, + 166, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 185, 251, 252, 219, 250, 255, + 217, 247, 83, 84, 85, 86, 87, 88, + 89, 90, 178, 212, 214, 210, 211, 213, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 179, 123, 220, 125, 218, 126 +}; +#endif /* POSIX-BC */ +#if '^' == 176 /* if defined(??) (OS/400?) 037 */ +EXT unsigned char PL_e2a[] = { /* ASCII (ISO8859-1) to EBCDIC (IBM-037) */ + 0, 1, 2, 3, 55, 45, 46, 47, + 22, 5, 37, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 60, 61, 50, 38, + 24, 25, 63, 39, 28, 29, 30, 31, + 64, 90, 127, 123, 91, 108, 80, 125, + 77, 93, 92, 78, 107, 96, 75, 97, + 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 122, 94, 76, 126, 110, 111, + 124, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 209, 210, 211, 212, 213, 214, + 215, 216, 217, 226, 227, 228, 229, 230, + 231, 232, 233, 186, 224, 187, 176, 109, + 121, 129, 130, 131, 132, 133, 134, 135, + 136, 137, 145, 146, 147, 148, 149, 150, + 151, 152, 153, 162, 163, 164, 165, 166, + 167, 168, 169, 192, 79, 208, 161, 7, + 32, 33, 34, 35, 36, 21, 6, 23, + 40, 41, 42, 43, 44, 9, 10, 27, + 48, 49, 26, 51, 52, 53, 54, 8, + 56, 57, 58, 59, 4, 20, 62, 255, + 65, 170, 74, 177, 159, 178, 106, 181, + 189, 180, 154, 138, 95, 202, 175, 188, + 144, 143, 234, 250, 190, 160, 182, 179, + 157, 218, 155, 139, 183, 184, 185, 171, + 100, 101, 98, 102, 99, 103, 158, 104, + 116, 113, 114, 115, 120, 117, 118, 119, + 172, 105, 237, 238, 235, 239, 236, 191, + 128, 253, 254, 251, 252, 173, 174, 89, + 68, 69, 66, 70, 67, 71, 156, 72, + 84, 81, 82, 83, 88, 85, 86, 87, + 140, 73, 205, 206, 203, 207, 204, 225, + 112, 221, 222, 219, 220, 141, 142, 223 +}; +EXT unsigned char PL_a2e[] = { /* EBCDIC (IBM-037) to ASCII (ISO8859-1) */ + 0, 1, 2, 3, 156, 9, 134, 127, + 151, 141, 142, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 157, 133, 8, 135, + 24, 25, 146, 143, 28, 29, 30, 31, + 128, 129, 130, 131, 132, 10, 23, 27, + 136, 137, 138, 139, 140, 5, 6, 7, + 144, 145, 22, 147, 148, 149, 150, 4, + 152, 153, 154, 155, 20, 21, 158, 26, + 32, 160, 226, 228, 224, 225, 227, 229, + 231, 241, 162, 46, 60, 40, 43, 124, + 38, 233, 234, 235, 232, 237, 238, 239, + 236, 223, 33, 36, 42, 41, 59, 172, + 45, 47, 194, 196, 192, 193, 195, 197, + 199, 209, 166, 44, 37, 95, 62, 63, + 248, 201, 202, 203, 200, 205, 206, 207, + 204, 96, 58, 35, 64, 39, 61, 34, + 216, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 171, 187, 240, 253, 254, 177, + 176, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 170, 186, 230, 184, 198, 164, + 181, 126, 115, 116, 117, 118, 119, 120, + 121, 122, 161, 191, 208, 221, 222, 174, + 94, 163, 165, 183, 169, 167, 182, 188, + 189, 190, 91, 93, 175, 168, 180, 215, + 123, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 173, 244, 246, 242, 243, 245, + 125, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 185, 251, 252, 249, 250, 255, + 92, 247, 83, 84, 85, 86, 87, 88, + 89, 90, 178, 212, 214, 210, 211, 213, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 179, 219, 220, 217, 218, 159 +}; +#endif /* 037 */ +#if '^' == 95 /* if defined(__MVS__) || defined(??) (VM/ESA?) 1047 */ +EXT unsigned char PL_e2a[] = { /* ASCII (ISO8859-1) to EBCDIC (IBM-1047) */ + 0, 1, 2, 3, 55, 45, 46, 47, + 22, 5, 21, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 60, 61, 50, 38, + 24, 25, 63, 39, 28, 29, 30, 31, + 64, 90, 127, 123, 91, 108, 80, 125, + 77, 93, 92, 78, 107, 96, 75, 97, + 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 122, 94, 76, 126, 110, 111, + 124, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 209, 210, 211, 212, 213, 214, + 215, 216, 217, 226, 227, 228, 229, 230, + 231, 232, 233, 173, 224, 189, 95, 109, + 121, 129, 130, 131, 132, 133, 134, 135, + 136, 137, 145, 146, 147, 148, 149, 150, + 151, 152, 153, 162, 163, 164, 165, 166, + 167, 168, 169, 192, 79, 208, 161, 7, + 32, 33, 34, 35, 36, 37, 6, 23, + 40, 41, 42, 43, 44, 9, 10, 27, + 48, 49, 26, 51, 52, 53, 54, 8, + 56, 57, 58, 59, 4, 20, 62, 255, + 65, 170, 74, 177, 159, 178, 106, 181, + 187, 180, 154, 138, 176, 202, 175, 188, + 144, 143, 234, 250, 190, 160, 182, 179, + 157, 218, 155, 139, 183, 184, 185, 171, + 100, 101, 98, 102, 99, 103, 158, 104, + 116, 113, 114, 115, 120, 117, 118, 119, + 172, 105, 237, 238, 235, 239, 236, 191, + 128, 253, 254, 251, 252, 186, 174, 89, + 68, 69, 66, 70, 67, 71, 156, 72, + 84, 81, 82, 83, 88, 85, 86, 87, + 140, 73, 205, 206, 203, 207, 204, 225, + 112, 221, 222, 219, 220, 141, 142, 223 +}; +EXT unsigned char PL_a2e[] = { /* EBCDIC (IBM-1047) to ASCII (ISO8859-1) */ + 0, 1, 2, 3, 156, 9, 134, 127, + 151, 141, 142, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 157, 10, 8, 135, + 24, 25, 146, 143, 28, 29, 30, 31, + 128, 129, 130, 131, 132, 133, 23, 27, + 136, 137, 138, 139, 140, 5, 6, 7, + 144, 145, 22, 147, 148, 149, 150, 4, + 152, 153, 154, 155, 20, 21, 158, 26, + 32, 160, 226, 228, 224, 225, 227, 229, + 231, 241, 162, 46, 60, 40, 43, 124, + 38, 233, 234, 235, 232, 237, 238, 239, + 236, 223, 33, 36, 42, 41, 59, 94, + 45, 47, 194, 196, 192, 193, 195, 197, + 199, 209, 166, 44, 37, 95, 62, 63, + 248, 201, 202, 203, 200, 205, 206, 207, + 204, 96, 58, 35, 64, 39, 61, 34, + 216, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 171, 187, 240, 253, 254, 177, + 176, 106, 107, 108, 109, 110, 111, 112, + 113, 114, 170, 186, 230, 184, 198, 164, + 181, 126, 115, 116, 117, 118, 119, 120, + 121, 122, 161, 191, 208, 91, 222, 174, + 172, 163, 165, 183, 169, 167, 182, 188, + 189, 190, 221, 168, 175, 93, 180, 215, + 123, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 173, 244, 246, 242, 243, 245, + 125, 74, 75, 76, 77, 78, 79, 80, + 81, 82, 185, 251, 252, 249, 250, 255, + 92, 247, 83, 84, 85, 86, 87, 88, + 89, 90, 178, 212, 214, 210, 211, 213, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 179, 219, 220, 217, 218, 159 +}; +#endif /* 1047 */ EXT unsigned char PL_fold[] = { /* fast EBCDIC case folding table */ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, @@ -2512,6 +2722,10 @@ EXTCONST unsigned char PL_fold[] = { #endif /* !EBCDIC */ #else EXTCONST unsigned char PL_fold[]; +#ifdef EBCDIC +EXTCONST unsigned char PL_e2a[]; +EXTCONST unsigned char PL_a2e[]; +#endif /* EBCDIC */ #endif #ifdef DOINIT diff --git a/sv.c b/sv.c index d47a2f5..20b4f2a 100644 --- a/sv.c +++ b/sv.c @@ -7188,6 +7188,8 @@ Perl_sv_vcatpvfn(pTHX_ SV *sv, const char *pat, STRLEN patlen, va_list *args, SV iv = *vecstr; ulen = 1; } + if (iv <256) + iv = NATIVE_TO_ASCII(iv); /* v-strings are codepoints */ vecstr += ulen; veclen -= ulen; } @@ -7268,6 +7270,8 @@ Perl_sv_vcatpvfn(pTHX_ SV *sv, const char *pat, STRLEN patlen, va_list *args, SV uv = *vecstr; ulen = 1; } + if (uv <256) + uv = NATIVE_TO_ASCII(uv); /* v-strings are codepoints */ vecstr += ulen; veclen -= ulen; } diff --git a/toke.c b/toke.c index 4e90201..2bb9282 100644 --- a/toke.c +++ b/toke.c @@ -7273,6 +7273,10 @@ vstring: "Integer overflow in decimal number"); } } + /* THIS IS EVIL */ + if (rev < 256) + rev = ASCII_TO_NATIVE(rev); + tmpend = uv_to_utf8(tmpbuf, rev); if (rev > revmax) revmax = rev; @@ -7289,11 +7293,11 @@ vstring: SvPOK_on(sv); SvREADONLY_on(sv); - if (revmax > 127) { - SvUTF8_on(sv); + /* if (revmax > 127) { */ + SvUTF8_on(sv); /* if (revmax < 256) sv_utf8_downgrade(sv, TRUE); - } + } */ } } break; diff --git a/utf8.c b/utf8.c index f00659a..55a8c7c 100644 --- a/utf8.c +++ b/utf8.c @@ -46,6 +46,8 @@ is the recommended Unicode-aware way of saying U8 * Perl_uv_to_utf8(pTHX_ U8 *d, UV uv) { + if (uv < 0x100) + uv = NATIVE_TO_ASCII(uv); if (uv < 0x80) { *d++ = uv; return d; @@ -254,7 +256,7 @@ Perl_utf8_to_uv(pTHX_ U8* s, STRLEN curlen, STRLEN* retlen, U32 flags) if (UTF8_IS_ASCII(uv)) { if (retlen) *retlen = 1; - return *s; + return ASCII_TO_NATIVE(*s); } if (UTF8_IS_CONTINUATION(uv) && diff --git a/utf8.h b/utf8.h index 8b0c8c3..4917811 100644 --- a/utf8.h +++ b/utf8.h @@ -131,3 +131,22 @@ END_EXTERN_C #endif #define isIDFIRST_lazy(p) isIDFIRST_lazy_if(p,1) #define isALNUM_lazy(p) isALNUM_lazy_if(p,1) + +/* EBCDIC-happy ways of converting native code to UTF8; the reverse + process is taken care of in utf8_to_uv */ + +#ifdef EBCDIC +#define NATIVE_TO_ASCII(ch) PL_e2a[(ch)] +#define ASCII_TO_NATIVE(ch) PL_a2e[(ch)] +#else +#define NATIVE_TO_ASCII(ch) (ch) +#define ASCII_TO_NATIVE(ch) (ch) +#endif + +#define UTF8_NEEDS_UPGRADE(ch) (NATIVE_TO_ASCII(ch) & 0x80) +#define NATIVE_TO_UTF8(ch, string) STMT_START { \ + if (!UTF8_NEEDS_UPGRADE(ch)) \ + *(string)++ = NATIVE_TO_ASCII(ch); \ + else /* uv_to_utf8 is EBCDIC-aware */ \ + string = uv_to_utf8(string, ch); \ + } STMT_END