X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=blobdiff_plain;f=toke.c;h=4b65d6244b68c0eca53fdf99f2c2d328bdf0d0cc;hb=9f07fdcd4d463e2090de7661b6a313c8ecc5278b;hp=f3683677b694f2fe18498e0590defd1598197c07;hpb=a4f5cc89eed636ba46185bdb30ee0a36fa506dd8;p=p5sagit%2Fp5-mst-13.2.git diff --git a/toke.c b/toke.c index f368367..4b65d62 100644 --- a/toke.c +++ b/toke.c @@ -209,10 +209,8 @@ S_no_op(pTHX_ char *what, char *s) if (!s) s = oldbp; - else { - assert(s >= oldbp); + else PL_bufptr = s; - } yywarn(Perl_form(aTHX_ "%s found where operator expected", what)); if (is_first) Perl_warn(aTHX_ "\t(Missing semicolon on previous line?)\n"); @@ -223,8 +221,10 @@ S_no_op(pTHX_ char *what, char *s) Perl_warn(aTHX_ "\t(Do you need to predeclare %.*s?)\n", t - PL_oldoldbufptr, PL_oldoldbufptr); } - else + else { + assert(s >= oldbp); Perl_warn(aTHX_ "\t(Missing operator before %.*s?)\n", s - oldbp, oldbp); + } PL_bufptr = oldbp; } @@ -357,7 +357,6 @@ Perl_lex_start(pTHX_ SV *line) SAVEVPTR(PL_nextval[toke]); } SAVEI32(PL_nexttoke); - PL_nexttoke = 0; } SAVECOPLINE(PL_curcop); SAVEPPTR(PL_bufptr); @@ -391,6 +390,7 @@ Perl_lex_start(pTHX_ SV *line) PL_lex_stuff = Nullsv; PL_lex_repl = Nullsv; PL_lex_inpat = 0; + PL_nexttoke = 0; PL_lex_inwhat = 0; PL_sublex_info.sub_inwhat = 0; PL_linestr = line; @@ -812,7 +812,7 @@ Perl_str_to_version(pTHX_ SV *sv) I32 skip; UV n; if (utf) - n = utf8_to_uv((U8*)start, &skip); + n = utf8_to_uv_chk((U8*)start, &skip, 0); else { n = *(U8*)start; skip = 1; @@ -1219,7 +1219,7 @@ S_scan_const(pTHX_ char *start) if (min > max) { Perl_croak(aTHX_ "Invalid [] range \"%c-%c\" in transliteration operator", - min, max); + (char)min, (char)max); } #ifndef ASCIIish @@ -1323,7 +1323,7 @@ S_scan_const(pTHX_ char *start) /* (now in tr/// code again) */ if (*s & 0x80 && thisutf) { - (void)utf8_to_uv((U8*)s, &len); + (void)utf8_to_uv_chk((U8*)s, &len, 0); if (len == 1) { /* illegal UTF8, make it valid */ char *old_pvx = SvPVX(sv); @@ -1489,7 +1489,10 @@ S_scan_const(pTHX_ char *start) char *ostart = SvPVX(sv); SvCUR_set(sv, d - ostart); SvPOK_on(sv); + *d = '\0'; sv_utf8_upgrade(sv); + /* this just broke our allocation above... */ + SvGROW(sv, send - start); d = SvPVX(sv) + SvCUR(sv); has_utf = TRUE; } @@ -2482,7 +2485,8 @@ Perl_yylex(pTHX) do { bool bof; bof = PL_rsfp && (PerlIO_tell(PL_rsfp) == 0); /* *Before* read! */ - if ((s = filter_gets(PL_linestr, PL_rsfp, 0)) == Nullch) { + s = filter_gets(PL_linestr, PL_rsfp, 0); + if (s == Nullch) { fake_eof: if (PL_rsfp) { if (PL_preprocess && !PL_in_eval) @@ -2505,6 +2509,9 @@ Perl_yylex(pTHX) PL_oldoldbufptr = PL_oldbufptr = s = PL_linestart = SvPVX(PL_linestr); sv_setpv(PL_linestr,""); TOKEN(';'); /* not infinite loop because rsfp is NULL now */ + } else if (bof) { + PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr); + s = swallow_bom((U8*)s); } if (PL_doextract) { if (*s == '#' && s[1] == '!' && instr(s,"perl")) @@ -2518,14 +2525,6 @@ Perl_yylex(pTHX) PL_doextract = FALSE; } } - if (bof) - { - PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr); - /* Shouldn't this wsallow_bom() be earlier, e.g. - * immediately after where bof is set? Currently you can't - * have e.g. a UTF16 sharpbang line. --Mike Guy */ - s = swallow_bom((U8*)s); - } incline(s); } while (PL_doextract); PL_oldoldbufptr = PL_oldbufptr = PL_bufptr = PL_linestart = s; @@ -3984,11 +3983,11 @@ Perl_yylex(pTHX) /* Mark this internal pseudo-handle as clean */ IoFLAGS(GvIOp(gv)) |= IOf_UNTAINT; if (PL_preprocess) - IoTYPE(GvIOp(gv)) = '|'; + IoTYPE(GvIOp(gv)) = IoTYPE_PIPE; else if ((PerlIO*)PL_rsfp == PerlIO_stdin()) - IoTYPE(GvIOp(gv)) = '-'; + IoTYPE(GvIOp(gv)) = IoTYPE_STD; else - IoTYPE(GvIOp(gv)) = '<'; + IoTYPE(GvIOp(gv)) = IoTYPE_RDONLY; #if defined(WIN32) && !defined(PERL_TEXTMODE_SCRIPTS) /* if the script was opened in binmode, we need to revert * it to text mode for compatibility; but only iff it has CRs @@ -3997,7 +3996,7 @@ Perl_yylex(pTHX) && PL_bufend[-1] == '\n' && PL_bufend[-2] == '\r') { Off_t loc = 0; - if (IoTYPE(GvIOp(gv)) == '<') { + if (IoTYPE(GvIOp(gv)) == IoTYPE_RDONLY) { loc = PerlIO_tell(PL_rsfp); (void)PerlIO_seek(PL_rsfp, 0L, 0); } @@ -5096,12 +5095,12 @@ Perl_keyword(pTHX_ register char *d, I32 len) if (strEQ(d,"cos")) return -KEY_cos; break; case 4: - if (strEQ(d,"chop")) return KEY_chop; + if (strEQ(d,"chop")) return -KEY_chop; break; case 5: if (strEQ(d,"close")) return -KEY_close; if (strEQ(d,"chdir")) return -KEY_chdir; - if (strEQ(d,"chomp")) return KEY_chomp; + if (strEQ(d,"chomp")) return -KEY_chomp; if (strEQ(d,"chmod")) return -KEY_chmod; if (strEQ(d,"chown")) return -KEY_chown; if (strEQ(d,"crypt")) return -KEY_crypt; @@ -5146,7 +5145,6 @@ Perl_keyword(pTHX_ register char *d, I32 len) } break; case 'E': - if (strEQ(d,"EQ")) { deprecate(d); return -KEY_eq;} if (strEQ(d,"END")) return KEY_END; break; case 'e': @@ -5212,12 +5210,6 @@ Perl_keyword(pTHX_ register char *d, I32 len) break; } break; - case 'G': - if (len == 2) { - if (strEQ(d,"GT")) { deprecate(d); return -KEY_gt;} - if (strEQ(d,"GE")) { deprecate(d); return -KEY_ge;} - } - break; case 'g': if (strnEQ(d,"get",3)) { d += 3; @@ -5317,12 +5309,6 @@ Perl_keyword(pTHX_ register char *d, I32 len) if (strEQ(d,"kill")) return -KEY_kill; } break; - case 'L': - if (len == 2) { - if (strEQ(d,"LT")) { deprecate(d); return -KEY_lt;} - if (strEQ(d,"LE")) { deprecate(d); return -KEY_le;} - } - break; case 'l': switch (len) { case 2: @@ -5374,9 +5360,6 @@ Perl_keyword(pTHX_ register char *d, I32 len) break; } break; - case 'N': - if (strEQ(d,"NE")) { deprecate(d); return -KEY_ne;} - break; case 'n': if (strEQ(d,"next")) return KEY_next; if (strEQ(d,"ne")) return -KEY_ne; @@ -5762,13 +5745,12 @@ S_new_constant(pTHX_ char *s, STRLEN len, const char *key, SV *sv, SV *pv, SAVETMPS; PUSHMARK(SP) ; - EXTEND(sp, 4); + EXTEND(sp, 3); if (pv) PUSHs(pv); PUSHs(sv); if (pv) PUSHs(typesv); - PUSHs(cv); PUTBACK; call_sv(cv, G_SCALAR | ( PL_in_eval ? 0 : G_EVAL)); @@ -7375,7 +7357,7 @@ Perl_yyerror(pTHX_ char *s) qerror(msg); if (PL_error_count >= 10) { if (PL_in_eval && SvCUR(ERRSV)) - Perl_croak(aTHX_ "%_%s has too many errors.\n", + Perl_croak(aTHX_ "%"SVf"%s has too many errors.\n", ERRSV, CopFILE(PL_curcop)); else Perl_croak(aTHX_ "%s has too many errors.\n", @@ -7395,18 +7377,26 @@ S_swallow_bom(pTHX_ U8 *s) case 0xFF: if (s[1] == 0xFE) { /* UTF-16 little-endian */ -#ifndef PERL_NO_UTF16_FILTER - U8 *news; -#endif if (s[2] == 0 && s[3] == 0) /* UTF-32 little-endian */ Perl_croak(aTHX_ "Unsupported script encoding"); #ifndef PERL_NO_UTF16_FILTER + DEBUG_p(PerlIO_printf(Perl_debug_log, "UTF-LE script encoding\n")); s += 2; - filter_add(utf16rev_textfilter, NULL); - New(898, news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8); - PL_bufend = (char*)utf16_to_utf8((U16*)s, news, - PL_bufend - (char*)s); - s = news; + if (PL_bufend > (char*)s) { + U8 *news; + I32 newlen; + + filter_add(utf16rev_textfilter, NULL); + New(898, news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8); + PL_bufend = (char*)utf16_to_utf8_reversed(s, news, + PL_bufend - (char*)s - 1, + &newlen); + Copy(news, s, newlen, U8); + SvCUR_set(PL_linestr, newlen); + PL_bufend = SvPVX(PL_linestr) + newlen; + news[newlen++] = '\0'; + Safefree(news); + } #else Perl_croak(aTHX_ "Unsupported script encoding"); #endif @@ -7415,12 +7405,23 @@ S_swallow_bom(pTHX_ U8 *s) case 0xFE: if (s[1] == 0xFF) { /* UTF-16 big-endian */ #ifndef PERL_NO_UTF16_FILTER - U8 *news; - filter_add(utf16_textfilter, NULL); - New(898, news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8); - PL_bufend = (char*)utf16_to_utf8((U16*)s, news, - PL_bufend - (char*)s); - s = news; + DEBUG_p(PerlIO_printf(Perl_debug_log, "UTF-16BE script encoding\n")); + s += 2; + if (PL_bufend > (char *)s) { + U8 *news; + I32 newlen; + + filter_add(utf16_textfilter, NULL); + New(898, news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8); + PL_bufend = (char*)utf16_to_utf8(s, news, + PL_bufend - (char*)s, + &newlen); + Copy(news, s, newlen, U8); + SvCUR_set(PL_linestr, newlen); + PL_bufend = SvPVX(PL_linestr) + newlen; + news[newlen++] = '\0'; + Safefree(news); + } #else Perl_croak(aTHX_ "Unsupported script encoding"); #endif @@ -7428,6 +7429,7 @@ S_swallow_bom(pTHX_ U8 *s) break; case 0xEF: if (slen > 2 && s[1] == 0xBB && s[2] == 0xBF) { + DEBUG_p(PerlIO_printf(Perl_debug_log, "UTF-8 script encoding\n")); s += 3; /* UTF-8 */ } break; @@ -7470,8 +7472,13 @@ utf16_textfilter(pTHXo_ int idx, SV *sv, int maxlen) if (count) { U8* tmps; U8* tend; + I32 newlen; New(898, tmps, SvCUR(sv) * 3 / 2 + 1, U8); - tend = utf16_to_utf8((U16*)SvPVX(sv), tmps, SvCUR(sv)); + if (!*SvPV_nolen(sv)) + /* Game over, but don't feed an odd-length string to utf16_to_utf8 */ + return count; + + tend = utf16_to_utf8((U8*)SvPVX(sv), tmps, SvCUR(sv), &newlen); sv_usepvn(sv, (char*)tmps, tend - tmps); } return count; @@ -7484,8 +7491,13 @@ utf16rev_textfilter(pTHXo_ int idx, SV *sv, int maxlen) if (count) { U8* tmps; U8* tend; + I32 newlen; + if (!*SvPV_nolen(sv)) + /* Game over, but don't feed an odd-length string to utf16_to_utf8 */ + return count; + New(898, tmps, SvCUR(sv) * 3 / 2 + 1, U8); - tend = utf16_to_utf8_reversed((U16*)SvPVX(sv), tmps, SvCUR(sv)); + tend = utf16_to_utf8_reversed((U8*)SvPVX(sv), tmps, SvCUR(sv), &newlen); sv_usepvn(sv, (char*)tmps, tend - tmps); } return count;