Test case for #10433/#10424.
[p5sagit/p5-mst-13.2.git] / utf8.h
CommitLineData
a0ed51b3 1/* utf8.h
2 *
bc89e66f 3 * Copyright (c) 1998-2001, Larry Wall
a0ed51b3 4 *
5 * You may distribute under the terms of either the GNU General Public
6 * License or the Artistic License, as specified in the README file.
7 *
8 */
9
1d72bdf6 10#ifdef EBCDIC
11/* The equivalent of these macros but implementing UTF-EBCDIC
12 are in the following header file:
13 */
14
15#include "utfebcdic.h"
16#else
73c4f7a1 17START_EXTERN_C
18
a0ed51b3 19#ifdef DOINIT
6f06b55f 20EXTCONST unsigned char PL_utf8skip[] = {
a0ed51b3 211,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
221,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
231,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
241,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* ascii */
251,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus */
261,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, /* bogus */
272,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, /* scripts */
3c77ea2b 283,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,6,6, /* cjk etc. */
297,13, /* Perl extended (not UTF-8). Up to 72bit allowed (64-bit + reserved). */
a0ed51b3 30};
31#else
6f06b55f 32EXTCONST unsigned char PL_utf8skip[];
a0ed51b3 33#endif
34
73c4f7a1 35END_EXTERN_C
6f06b55f 36#define UTF8SKIP(s) PL_utf8skip[*(U8*)s]
7e2040f0 37
1d72bdf6 38/* Native character to iso-8859-1 */
39#define NATIVE_TO_ASCII(ch) (ch)
40#define ASCII_TO_NATIVE(ch) (ch)
41/* Transform after encoding */
42#define NATIVE_TO_UTF(ch) (ch)
43#define UTF_TO_NATIVE(ch) (ch)
44/* Transforms in wide UV chars */
45#define UNI_TO_NATIVE(ch) (ch)
46#define NATIVE_TO_UNI(ch) (ch)
47/* Transforms in invariant space */
48#define NATIVE_TO_NEED(enc,ch) (ch)
49#define ASCII_TO_NEED(enc,ch) (ch)
d7578b48 50
2b9d42f0 51/* As there are no translations avoid the function wrapper */
52#define Perl_utf8n_to_uvchr Perl_utf8n_to_uvuni
53#define Perl_uvchr_to_utf8 Perl_uvuni_to_utf8
54
877d9f0d 55/*
9041c2e3 56
877d9f0d 57 The following table is from Unicode 3.1.
58
59 Code Points 1st Byte 2nd Byte 3rd Byte 4th Byte
60
61 U+0000..U+007F 00..7F   
62 U+0080..U+07FF C2..DF 80..BF   
63 U+0800..U+0FFF E0 A0..BF 80..BF  
64 U+1000..U+FFFF E1..EF 80..BF 80..BF  
65 U+10000..U+3FFFF F0 90..BF 80..BF 80..BF
66 U+40000..U+FFFFF F1..F3 80..BF 80..BF 80..BF
67 U+100000..U+10FFFF F4 80..8F 80..BF 80..BF
68
69 */
70
c4d5f83a 71#define UNI_IS_INVARIANT(c) (((UV)c) < 0x80)
72#define UTF8_IS_INVARIANT(c) UNI_IS_INVARIANT(NATIVE_TO_UTF(c))
73#define NATIVE_IS_INVARIANT(c) UNI_IS_INVARIANT(NATIVE_TO_ASCII(c))
c512ce4f 74#define UTF8_IS_START(c) (((U8)c) >= 0xc0 && (((U8)c) <= 0xfd))
75#define UTF8_IS_CONTINUATION(c) (((U8)c) >= 0x80 && (((U8)c) <= 0xbf))
76#define UTF8_IS_CONTINUED(c) (((U8)c) & 0x80)
db42d148 77#define UTF8_IS_DOWNGRADEABLE_START(c) (((U8)c & 0xfc) == 0xc0)
8850bf83 78
1d72bdf6 79#define UTF_START_MARK(len) ((len > 7) ? 0xFF : (0xFE << (7-len)))
80#define UTF_START_MASK(len) ((len >= 7) ? 0x00 : (0x1F >> (len-2)))
81
82#define UTF_CONTINUATION_MARK 0x80
83#define UTF_ACCUMULATION_SHIFT 6
84#define UTF_CONTINUATION_MASK ((U8)0x3f)
cb359b41 85#define UTF8_ACCUMULATE(old, new) (((old) << UTF_ACCUMULATION_SHIFT) | (((U8)new) & UTF_CONTINUATION_MASK))
421a8bf2 86
1d72bdf6 87#define UTF8_EIGHT_BIT_HI(c) ((((U8)(c))>>UTF_ACCUMULATION_SHIFT)|UTF_START_MARK(2))
88#define UTF8_EIGHT_BIT_LO(c) (((((U8)(c)))&UTF_CONTINUATION_MASK)|UTF_CONTINUATION_MARK)
c512ce4f 89
1d68d6cd 90#ifdef HAS_QUAD
5bbb0b5a 91#define UNISKIP(uv) ( (uv) < 0x80 ? 1 : \
1d68d6cd 92 (uv) < 0x800 ? 2 : \
93 (uv) < 0x10000 ? 3 : \
94 (uv) < 0x200000 ? 4 : \
95 (uv) < 0x4000000 ? 5 : \
96 (uv) < 0x80000000 ? 6 : \
9041c2e3 97 (uv) < UTF8_QUAD_MAX ? 7 : 13 )
1d68d6cd 98#else
99/* No, I'm not even going to *TRY* putting #ifdef inside a #define */
5bbb0b5a 100#define UNISKIP(uv) ( (uv) < 0x80 ? 1 : \
1d68d6cd 101 (uv) < 0x800 ? 2 : \
102 (uv) < 0x10000 ? 3 : \
103 (uv) < 0x200000 ? 4 : \
104 (uv) < 0x4000000 ? 5 : \
105 (uv) < 0x80000000 ? 6 : 7 )
106#endif
107
7e2040f0 108/*
109 * Note: we try to be careful never to call the isXXX_utf8() functions
110 * unless we're pretty sure we've seen the beginning of a UTF-8 character
111 * (that is, the two high bits are set). Otherwise we risk loading in the
112 * heavy-duty SWASHINIT and SWASHGET routines unnecessarily.
113 */
0064a8a9 114#define isIDFIRST_lazy_if(p,c) ((IN_BYTES || (!c || (*((U8*)p) < 0xc0))) \
7e2040f0 115 ? isIDFIRST(*(p)) \
116 : isIDFIRST_utf8((U8*)p))
0064a8a9 117#define isALNUM_lazy_if(p,c) ((IN_BYTES || (!c || (*((U8*)p) < 0xc0))) \
7e2040f0 118 ? isALNUM(*(p)) \
119 : isALNUM_utf8((U8*)p))
1d72bdf6 120
121
122#endif /* EBCDIC vs ASCII */
123
124/* Rest of these are attributes of Unicode and perl's internals rather than the encoding */
125
7e2040f0 126#define isIDFIRST_lazy(p) isIDFIRST_lazy_if(p,1)
127#define isALNUM_lazy(p) isALNUM_lazy_if(p,1)
3bd709b1 128
1d72bdf6 129#define UTF8_MAXLEN 13 /* how wide can a single UTF8 encoded character become */
3bd709b1 130
1d72bdf6 131/* #define IN_UTF8 (PL_curcop->op_private & HINT_UTF8) */
0064a8a9 132#define IN_BYTES (PL_curcop->op_private & HINT_BYTES)
133#define DO_UTF8(sv) (SvUTF8(sv) && !IN_BYTES)
1d72bdf6 134
135#define UTF8_ALLOW_EMPTY 0x0001
136#define UTF8_ALLOW_CONTINUATION 0x0002
137#define UTF8_ALLOW_NON_CONTINUATION 0x0004
138#define UTF8_ALLOW_FE_FF 0x0008
139#define UTF8_ALLOW_SHORT 0x0010
140#define UTF8_ALLOW_SURROGATE 0x0020
141#define UTF8_ALLOW_BOM 0x0040
142#define UTF8_ALLOW_FFFF 0x0080
143#define UTF8_ALLOW_LONG 0x0100
144#define UTF8_ALLOW_ANYUV (UTF8_ALLOW_EMPTY|UTF8_ALLOW_FE_FF|\
145 UTF8_ALLOW_SURROGATE|UTF8_ALLOW_BOM|\
146 UTF8_ALLOW_FFFF|UTF8_ALLOW_LONG)
147#define UTF8_ALLOW_ANY 0x00ff
148#define UTF8_CHECK_ONLY 0x0100
149
150#define UNICODE_SURROGATE_FIRST 0xd800
151#define UNICODE_SURROGATE_LAST 0xdfff
152#define UNICODE_REPLACEMENT 0xfffd
153#define UNICODE_BYTER_ORDER_MARK 0xfffe
154#define UNICODE_ILLEGAL 0xffff
155
156#define UNICODE_IS_SURROGATE(c) ((c) >= UNICODE_SURROGATE_FIRST && \
157 (c) <= UNICODE_SURROGATE_LAST)
a10ec373 158#define UNICODE_IS_REPLACEMENT(c) ((c) == UNICODE_REPLACEMENT)
1d72bdf6 159#define UNICODE_IS_BYTE_ORDER_MARK(c) ((c) == UNICODE_BYTER_ORDER_MARK)
160#define UNICODE_IS_ILLEGAL(c) ((c) == UNICODE_ILLEGAL)
161
162#define UTF8_QUAD_MAX UINT64_C(0x1000000000)
3bd709b1 163
1d72bdf6 164#define UTF8_IS_ASCII(c) UTF8_IS_INVARIANT(c)