Thinko in #7222.
[p5sagit/p5-mst-13.2.git] / toke.c
diff --git a/toke.c b/toke.c
index f368367..4b65d62 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -209,10 +209,8 @@ S_no_op(pTHX_ char *what, char *s)
 
     if (!s)
        s = oldbp;
-    else {
-       assert(s >= oldbp);
+    else
        PL_bufptr = s;
-    }
     yywarn(Perl_form(aTHX_ "%s found where operator expected", what));
     if (is_first)
        Perl_warn(aTHX_ "\t(Missing semicolon on previous line?)\n");
@@ -223,8 +221,10 @@ S_no_op(pTHX_ char *what, char *s)
            Perl_warn(aTHX_ "\t(Do you need to predeclare %.*s?)\n",
                t - PL_oldoldbufptr, PL_oldoldbufptr);
     }
-    else
+    else {
+       assert(s >= oldbp);
        Perl_warn(aTHX_ "\t(Missing operator before %.*s?)\n", s - oldbp, oldbp);
+    }
     PL_bufptr = oldbp;
 }
 
@@ -357,7 +357,6 @@ Perl_lex_start(pTHX_ SV *line)
            SAVEVPTR(PL_nextval[toke]);
        }
        SAVEI32(PL_nexttoke);
-       PL_nexttoke = 0;
     }
     SAVECOPLINE(PL_curcop);
     SAVEPPTR(PL_bufptr);
@@ -391,6 +390,7 @@ Perl_lex_start(pTHX_ SV *line)
     PL_lex_stuff = Nullsv;
     PL_lex_repl = Nullsv;
     PL_lex_inpat = 0;
+    PL_nexttoke = 0;
     PL_lex_inwhat = 0;
     PL_sublex_info.sub_inwhat = 0;
     PL_linestr = line;
@@ -812,7 +812,7 @@ Perl_str_to_version(pTHX_ SV *sv)
        I32 skip;
        UV n;
        if (utf)
-           n = utf8_to_uv((U8*)start, &skip);
+           n = utf8_to_uv_chk((U8*)start, &skip, 0);
        else {
            n = *(U8*)start;
            skip = 1;
@@ -1219,7 +1219,7 @@ S_scan_const(pTHX_ char *start)
                 if (min > max) {
                    Perl_croak(aTHX_
                               "Invalid [] range \"%c-%c\" in transliteration operator",
-                              min, max);
+                              (char)min, (char)max);
                 }
 
 #ifndef ASCIIish
@@ -1323,7 +1323,7 @@ S_scan_const(pTHX_ char *start)
        /* (now in tr/// code again) */
 
        if (*s & 0x80 && thisutf) {
-          (void)utf8_to_uv((U8*)s, &len);
+          (void)utf8_to_uv_chk((U8*)s, &len, 0);
           if (len == 1) {
               /* illegal UTF8, make it valid */
               char *old_pvx = SvPVX(sv);
@@ -1489,7 +1489,10 @@ S_scan_const(pTHX_ char *start)
                        char *ostart = SvPVX(sv);
                        SvCUR_set(sv, d - ostart);
                        SvPOK_on(sv);
+                       *d = '\0';
                        sv_utf8_upgrade(sv);
+                       /* this just broke our allocation above... */
+                       SvGROW(sv, send - start);
                        d = SvPVX(sv) + SvCUR(sv);
                        has_utf = TRUE;
                    }
@@ -2482,7 +2485,8 @@ Perl_yylex(pTHX)
        do {
            bool bof;
            bof = PL_rsfp && (PerlIO_tell(PL_rsfp) == 0); /* *Before* read! */
-           if ((s = filter_gets(PL_linestr, PL_rsfp, 0)) == Nullch) {
+           s = filter_gets(PL_linestr, PL_rsfp, 0);
+           if (s == Nullch) {
              fake_eof:
                if (PL_rsfp) {
                    if (PL_preprocess && !PL_in_eval)
@@ -2505,6 +2509,9 @@ Perl_yylex(pTHX)
                PL_oldoldbufptr = PL_oldbufptr = s = PL_linestart = SvPVX(PL_linestr);
                sv_setpv(PL_linestr,"");
                TOKEN(';');     /* not infinite loop because rsfp is NULL now */
+           } else if (bof) {
+               PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
+               s = swallow_bom((U8*)s);
            }
            if (PL_doextract) {
                if (*s == '#' && s[1] == '!' && instr(s,"perl"))
@@ -2518,14 +2525,6 @@ Perl_yylex(pTHX)
                    PL_doextract = FALSE;
                }
            } 
-           if (bof)
-           {
-               PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
-               /* Shouldn't this wsallow_bom() be earlier, e.g.
-                * immediately after where bof is set?  Currently you can't
-                * have e.g. a UTF16 sharpbang line. --Mike Guy */
-               s = swallow_bom((U8*)s);
-           }
            incline(s);
        } while (PL_doextract);
        PL_oldoldbufptr = PL_oldbufptr = PL_bufptr = PL_linestart = s;
@@ -3984,11 +3983,11 @@ Perl_yylex(pTHX)
                /* Mark this internal pseudo-handle as clean */
                IoFLAGS(GvIOp(gv)) |= IOf_UNTAINT;
                if (PL_preprocess)
-                   IoTYPE(GvIOp(gv)) = '|';
+                   IoTYPE(GvIOp(gv)) = IoTYPE_PIPE;
                else if ((PerlIO*)PL_rsfp == PerlIO_stdin())
-                   IoTYPE(GvIOp(gv)) = '-';
+                   IoTYPE(GvIOp(gv)) = IoTYPE_STD;
                else
-                   IoTYPE(GvIOp(gv)) = '<';
+                   IoTYPE(GvIOp(gv)) = IoTYPE_RDONLY;
 #if defined(WIN32) && !defined(PERL_TEXTMODE_SCRIPTS)
                /* if the script was opened in binmode, we need to revert
                 * it to text mode for compatibility; but only iff it has CRs
@@ -3997,7 +3996,7 @@ Perl_yylex(pTHX)
                    && PL_bufend[-1] == '\n' && PL_bufend[-2] == '\r')
                {
                    Off_t loc = 0;
-                   if (IoTYPE(GvIOp(gv)) == '<') {
+                   if (IoTYPE(GvIOp(gv)) == IoTYPE_RDONLY) {
                        loc = PerlIO_tell(PL_rsfp);
                        (void)PerlIO_seek(PL_rsfp, 0L, 0);
                    }
@@ -5096,12 +5095,12 @@ Perl_keyword(pTHX_ register char *d, I32 len)
            if (strEQ(d,"cos"))                 return -KEY_cos;
            break;
        case 4:
-           if (strEQ(d,"chop"))                return KEY_chop;
+           if (strEQ(d,"chop"))                return -KEY_chop;
            break;
        case 5:
            if (strEQ(d,"close"))               return -KEY_close;
            if (strEQ(d,"chdir"))               return -KEY_chdir;
-           if (strEQ(d,"chomp"))               return KEY_chomp;
+           if (strEQ(d,"chomp"))               return -KEY_chomp;
            if (strEQ(d,"chmod"))               return -KEY_chmod;
            if (strEQ(d,"chown"))               return -KEY_chown;
            if (strEQ(d,"crypt"))               return -KEY_crypt;
@@ -5146,7 +5145,6 @@ Perl_keyword(pTHX_ register char *d, I32 len)
        }
        break;
     case 'E':
-       if (strEQ(d,"EQ")) { deprecate(d);      return -KEY_eq;}
        if (strEQ(d,"END"))                     return KEY_END;
        break;
     case 'e':
@@ -5212,12 +5210,6 @@ Perl_keyword(pTHX_ register char *d, I32 len)
            break;
        }
        break;
-    case 'G':
-       if (len == 2) {
-           if (strEQ(d,"GT")) { deprecate(d);  return -KEY_gt;}
-           if (strEQ(d,"GE")) { deprecate(d);  return -KEY_ge;}
-       }
-       break;
     case 'g':
        if (strnEQ(d,"get",3)) {
            d += 3;
@@ -5317,12 +5309,6 @@ Perl_keyword(pTHX_ register char *d, I32 len)
            if (strEQ(d,"kill"))                return -KEY_kill;
        }
        break;
-    case 'L':
-       if (len == 2) {
-           if (strEQ(d,"LT")) { deprecate(d);  return -KEY_lt;}
-           if (strEQ(d,"LE")) { deprecate(d);  return -KEY_le;}
-       }
-       break;
     case 'l':
        switch (len) {
        case 2:
@@ -5374,9 +5360,6 @@ Perl_keyword(pTHX_ register char *d, I32 len)
            break;
        }
        break;
-    case 'N':
-       if (strEQ(d,"NE")) { deprecate(d);      return -KEY_ne;}
-       break;
     case 'n':
        if (strEQ(d,"next"))                    return KEY_next;
        if (strEQ(d,"ne"))                      return -KEY_ne;
@@ -5762,13 +5745,12 @@ S_new_constant(pTHX_ char *s, STRLEN len, const char *key, SV *sv, SV *pv,
     SAVETMPS;
     
     PUSHMARK(SP) ;
-    EXTEND(sp, 4);
+    EXTEND(sp, 3);
     if (pv)
        PUSHs(pv);
     PUSHs(sv);
     if (pv)
        PUSHs(typesv);
-    PUSHs(cv);
     PUTBACK;
     call_sv(cv, G_SCALAR | ( PL_in_eval ? 0 : G_EVAL));
     
@@ -7375,7 +7357,7 @@ Perl_yyerror(pTHX_ char *s)
        qerror(msg);
     if (PL_error_count >= 10) {
        if (PL_in_eval && SvCUR(ERRSV))
-           Perl_croak(aTHX_ "%_%s has too many errors.\n",
+           Perl_croak(aTHX_ "%"SVf"%s has too many errors.\n",
                       ERRSV, CopFILE(PL_curcop));
        else
            Perl_croak(aTHX_ "%s has too many errors.\n",
@@ -7395,18 +7377,26 @@ S_swallow_bom(pTHX_ U8 *s)
     case 0xFF:       
        if (s[1] == 0xFE) { 
            /* UTF-16 little-endian */
-#ifndef PERL_NO_UTF16_FILTER
-           U8 *news;
-#endif
            if (s[2] == 0 && s[3] == 0)  /* UTF-32 little-endian */
                Perl_croak(aTHX_ "Unsupported script encoding");
 #ifndef PERL_NO_UTF16_FILTER
+           DEBUG_p(PerlIO_printf(Perl_debug_log, "UTF-LE script encoding\n"));
            s += 2;
-           filter_add(utf16rev_textfilter, NULL);
-           New(898, news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8);
-           PL_bufend = (char*)utf16_to_utf8((U16*)s, news,
-                                            PL_bufend - (char*)s);
-           s = news;
+           if (PL_bufend > (char*)s) {
+               U8 *news;
+               I32 newlen;
+
+               filter_add(utf16rev_textfilter, NULL);
+               New(898, news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8);
+               PL_bufend = (char*)utf16_to_utf8_reversed(s, news,
+                                                PL_bufend - (char*)s - 1,
+                                                &newlen);
+               Copy(news, s, newlen, U8);
+               SvCUR_set(PL_linestr, newlen);
+               PL_bufend = SvPVX(PL_linestr) + newlen;
+               news[newlen++] = '\0';
+               Safefree(news);
+           }
 #else
            Perl_croak(aTHX_ "Unsupported script encoding");
 #endif
@@ -7415,12 +7405,23 @@ S_swallow_bom(pTHX_ U8 *s)
     case 0xFE:
        if (s[1] == 0xFF) {   /* UTF-16 big-endian */
 #ifndef PERL_NO_UTF16_FILTER
-           U8 *news;
-           filter_add(utf16_textfilter, NULL);
-           New(898, news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8);
-           PL_bufend = (char*)utf16_to_utf8((U16*)s, news,
-                                            PL_bufend - (char*)s);
-           s = news;
+           DEBUG_p(PerlIO_printf(Perl_debug_log, "UTF-16BE script encoding\n"));
+           s += 2;
+           if (PL_bufend > (char *)s) {
+               U8 *news;
+               I32 newlen;
+
+               filter_add(utf16_textfilter, NULL);
+               New(898, news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8);
+               PL_bufend = (char*)utf16_to_utf8(s, news,
+                                                PL_bufend - (char*)s,
+                                                &newlen);
+               Copy(news, s, newlen, U8);
+               SvCUR_set(PL_linestr, newlen);
+               PL_bufend = SvPVX(PL_linestr) + newlen;
+               news[newlen++] = '\0';
+               Safefree(news);
+           }
 #else
            Perl_croak(aTHX_ "Unsupported script encoding");
 #endif
@@ -7428,6 +7429,7 @@ S_swallow_bom(pTHX_ U8 *s)
        break;
     case 0xEF:
        if (slen > 2 && s[1] == 0xBB && s[2] == 0xBF) {
+           DEBUG_p(PerlIO_printf(Perl_debug_log, "UTF-8 script encoding\n"));
            s += 3;                      /* UTF-8 */
        }
        break;
@@ -7470,8 +7472,13 @@ utf16_textfilter(pTHXo_ int idx, SV *sv, int maxlen)
     if (count) {
        U8* tmps;
        U8* tend;
+       I32 newlen;
        New(898, tmps, SvCUR(sv) * 3 / 2 + 1, U8);
-       tend = utf16_to_utf8((U16*)SvPVX(sv), tmps, SvCUR(sv));
+       if (!*SvPV_nolen(sv))
+       /* Game over, but don't feed an odd-length string to utf16_to_utf8 */
+       return count;
+       
+       tend = utf16_to_utf8((U8*)SvPVX(sv), tmps, SvCUR(sv), &newlen);
        sv_usepvn(sv, (char*)tmps, tend - tmps);
     }
     return count;
@@ -7484,8 +7491,13 @@ utf16rev_textfilter(pTHXo_ int idx, SV *sv, int maxlen)
     if (count) {
        U8* tmps;
        U8* tend;
+       I32 newlen;
+       if (!*SvPV_nolen(sv))
+       /* Game over, but don't feed an odd-length string to utf16_to_utf8 */
+       return count;
+
        New(898, tmps, SvCUR(sv) * 3 / 2 + 1, U8);
-       tend = utf16_to_utf8_reversed((U16*)SvPVX(sv), tmps, SvCUR(sv));
+       tend = utf16_to_utf8_reversed((U8*)SvPVX(sv), tmps, SvCUR(sv), &newlen);
        sv_usepvn(sv, (char*)tmps, tend - tmps);
     }
     return count;