Fix doc bug #22067, noticed by Mark Knutsen
[p5sagit/p5-mst-13.2.git] / toke.c
diff --git a/toke.c b/toke.c
index 28e552b..850480c 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -1,6 +1,7 @@
 /*    toke.c
  *
- *    Copyright (c) 1991-2000, Larry Wall
+ *    Copyright (C) 1991, 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999,
+ *    2000, 2001, 2002, 2003, by Larry Wall and others
  *
  *    You may distribute under the terms of either the GNU General Public
  *    License or the Artistic License, as specified in the README file.
@@ -13,7 +14,7 @@
 
 /*
  * This file is the lexer for Perl.  It's closely linked to the
- * parser, perly.y.  
+ * parser, perly.y.
  *
  * The main routine is yylex(), which returns the next token.
  */
 #define yylval PL_yylval
 
 static char ident_too_long[] = "Identifier too long";
+static char c_without_g[] = "Use of /c modifier is meaningless without /g";
+static char c_in_subst[] = "Use of /c modifier is meaningless in s///";
 
-static void restore_rsfp(pTHXo_ void *f);
+static void restore_rsfp(pTHX_ void *f);
 #ifndef PERL_NO_UTF16_FILTER
-static I32 utf16_textfilter(pTHXo_ int idx, SV *sv, int maxlen);
-static I32 utf16rev_textfilter(pTHXo_ int idx, SV *sv, int maxlen);
+static I32 utf16_textfilter(pTHX_ int idx, SV *sv, int maxlen);
+static I32 utf16rev_textfilter(pTHX_ int idx, SV *sv, int maxlen);
 #endif
 
 #define XFAKEBRACK 128
 #define XENUMMASK 127
 
-/*#define UTF (SvUTF8(PL_linestr) && !(PL_hints & HINT_BYTE))*/
-#define UTF (PL_hints & HINT_UTF8)
+#ifdef USE_UTF8_SCRIPTS
+#   define UTF (!IN_BYTES)
+#else
+#   define UTF ((PL_linestr && DO_UTF8(PL_linestr)) || (PL_hints & HINT_UTF8))
+#endif
 
-/* In variables name $^X, these are the legal values for X.  
+/* In variables named $^X, these are the legal values for X.
  * 1999-02-27 mjd-perl-patch@plover.com */
 #define isCONTROLVAR(x) (isUPPER(x) || strchr("[\\]^_?", (x)))
 
@@ -79,13 +85,13 @@ static I32 utf16rev_textfilter(pTHXo_ int idx, SV *sv, int maxlen);
 #  endif
 YYSTYPE* yylval_pointer[YYMAXLEVEL];
 int* yychar_pointer[YYMAXLEVEL];
-int yyactlevel = 0;
+int yyactlevel = -1;
 #  undef yylval
 #  undef yychar
 #  define yylval (*yylval_pointer[yyactlevel])
 #  define yychar (*yychar_pointer[yyactlevel])
 #  define PERL_YYLEX_PARAM yylval_pointer[yyactlevel],yychar_pointer[yyactlevel]
-#  undef yylex 
+#  undef yylex
 #  define yylex()      Perl_yylex_r(aTHX_ yylval_pointer[yyactlevel],yychar_pointer[yyactlevel])
 #endif
 
@@ -126,38 +132,54 @@ int yyactlevel = 0;
  * Also see LOP and lop() below.
  */
 
-#define TOKEN(retval) return (PL_bufptr = s,(int)retval)
-#define OPERATOR(retval) return (PL_expect = XTERM,PL_bufptr = s,(int)retval)
-#define AOPERATOR(retval) return ao((PL_expect = XTERM,PL_bufptr = s,(int)retval))
-#define PREBLOCK(retval) return (PL_expect = XBLOCK,PL_bufptr = s,(int)retval)
-#define PRETERMBLOCK(retval) return (PL_expect = XTERMBLOCK,PL_bufptr = s,(int)retval)
-#define PREREF(retval) return (PL_expect = XREF,PL_bufptr = s,(int)retval)
-#define TERM(retval) return (CLINE, PL_expect = XOPERATOR,PL_bufptr = s,(int)retval)
-#define LOOPX(f) return(yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)LOOPEX)
-#define FTST(f) return(yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)UNIOP)
-#define FUN0(f) return(yylval.ival = f,PL_expect = XOPERATOR,PL_bufptr = s,(int)FUNC0)
-#define FUN1(f) return(yylval.ival = f,PL_expect = XOPERATOR,PL_bufptr = s,(int)FUNC1)
-#define BOop(f) return ao((yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)BITOROP))
-#define BAop(f) return ao((yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)BITANDOP))
-#define SHop(f) return ao((yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)SHIFTOP))
-#define PWop(f) return ao((yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)POWOP))
-#define PMop(f) return(yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)MATCHOP)
-#define Aop(f) return ao((yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)ADDOP))
-#define Mop(f) return ao((yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)MULOP))
-#define Eop(f) return(yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)EQOP)
-#define Rop(f) return(yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)RELOP)
+/* Note that REPORT() and REPORT2() will be expressions that supply
+ * their own trailing comma, not suitable for statements as such. */
+#ifdef DEBUGGING /* Serve -DT. */
+#   define REPORT(x,retval) tokereport(x,s,(int)retval),
+#   define REPORT2(x,retval) tokereport(x,s, yylval.ival),
+#else
+#   define REPORT(x,retval)
+#   define REPORT2(x,retval)
+#endif
+
+#define TOKEN(retval) return (REPORT2("token",retval) PL_bufptr = s,(int)retval)
+#define OPERATOR(retval) return (REPORT2("operator",retval) PL_expect = XTERM, PL_bufptr = s,(int)retval)
+#define AOPERATOR(retval) return ao((REPORT2("aop",retval) PL_expect = XTERM, PL_bufptr = s,(int)retval))
+#define PREBLOCK(retval) return (REPORT2("preblock",retval) PL_expect = XBLOCK,PL_bufptr = s,(int)retval)
+#define PRETERMBLOCK(retval) return (REPORT2("pretermblock",retval) PL_expect = XTERMBLOCK,PL_bufptr = s,(int)retval)
+#define PREREF(retval) return (REPORT2("preref",retval) PL_expect = XREF,PL_bufptr = s,(int)retval)
+#define TERM(retval) return (CLINE, REPORT2("term",retval) PL_expect = XOPERATOR, PL_bufptr = s,(int)retval)
+#define LOOPX(f) return(yylval.ival=f, REPORT("loopx",f) PL_expect = XTERM,PL_bufptr = s,(int)LOOPEX)
+#define FTST(f) return(yylval.ival=f, REPORT("ftst",f) PL_expect = XTERMORDORDOR,PL_bufptr = s,(int)UNIOP)
+#define FUN0(f) return(yylval.ival = f, REPORT("fun0",f) PL_expect = XOPERATOR,PL_bufptr = s,(int)FUNC0)
+#define FUN1(f) return(yylval.ival = f, REPORT("fun1",f) PL_expect = XOPERATOR,PL_bufptr = s,(int)FUNC1)
+#define BOop(f) return ao((yylval.ival=f, REPORT("bitorop",f) PL_expect = XTERM,PL_bufptr = s,(int)BITOROP))
+#define BAop(f) return ao((yylval.ival=f, REPORT("bitandop",f) PL_expect = XTERM,PL_bufptr = s,(int)BITANDOP))
+#define SHop(f) return ao((yylval.ival=f, REPORT("shiftop",f) PL_expect = XTERM,PL_bufptr = s,(int)SHIFTOP))
+#define PWop(f) return ao((yylval.ival=f, REPORT("powop",f) PL_expect = XTERM,PL_bufptr = s,(int)POWOP))
+#define PMop(f) return(yylval.ival=f, REPORT("matchop",f) PL_expect = XTERM,PL_bufptr = s,(int)MATCHOP)
+#define Aop(f) return ao((yylval.ival=f, REPORT("add",f) PL_expect = XTERM,PL_bufptr = s,(int)ADDOP))
+#define Mop(f) return ao((yylval.ival=f, REPORT("mul",f) PL_expect = XTERM,PL_bufptr = s,(int)MULOP))
+#define Eop(f) return(yylval.ival=f, REPORT("eq",f) PL_expect = XTERM,PL_bufptr = s,(int)EQOP)
+#define Rop(f) return(yylval.ival=f, REPORT("rel",f) PL_expect = XTERM,PL_bufptr = s,(int)RELOP)
 
 /* This bit of chicanery makes a unary function followed by
  * a parenthesis into a function with one argument, highest precedence.
+ * The UNIDOR macro is for unary functions that can be followed by the //
+ * operator (such as C<shift // 0>).
  */
-#define UNI(f) return(yylval.ival = f, \
-       PL_expect = XTERM, \
+#define UNI2(f,x) return(yylval.ival = f, \
+       REPORT("uni",f) \
+       PL_expect = x, \
        PL_bufptr = s, \
        PL_last_uni = PL_oldbufptr, \
        PL_last_lop_op = f, \
        (*s == '(' || (s = skipspace(s), *s == '(') ? (int)FUNC1 : (int)UNIOP) )
+#define UNI(f)    UNI2(f,XTERM)
+#define UNIDOR(f) UNI2(f,XTERMORDORDOR)
 
 #define UNIBRACK(f) return(yylval.ival = f, \
+        REPORT("uni",f) \
        PL_bufptr = s, \
        PL_last_uni = PL_oldbufptr, \
        (*s == '(' || (s = skipspace(s), *s == '(') ? (int)FUNC1 : (int)UNIOP) )
@@ -165,11 +187,33 @@ int yyactlevel = 0;
 /* grandfather return to old style */
 #define OLDLOP(f) return(yylval.ival=f,PL_expect = XTERM,PL_bufptr = s,(int)LSTOP)
 
+#ifdef DEBUGGING
+
+STATIC void
+S_tokereport(pTHX_ char *thing, char* s, I32 rv)
+{
+    DEBUG_T({
+        SV* report = newSVpv(thing, 0);
+        Perl_sv_catpvf(aTHX_ report, ":line %d:%"IVdf":", CopLINE(PL_curcop),
+               (IV)rv);
+
+        if (s - PL_bufptr > 0)
+            sv_catpvn(report, PL_bufptr, s - PL_bufptr);
+        else {
+            if (PL_oldbufptr && *PL_oldbufptr)
+                sv_catpv(report, PL_tokenbuf);
+        }
+        PerlIO_printf(Perl_debug_log, "### %s\n", SvPV_nolen(report));
+    });
+}
+
+#endif
+
 /*
  * S_ao
  *
- * This subroutine detects &&= and ||= and turns an ANDAND or OROR
- * into an OP_ANDASSIGN or OP_ORASSIGN
+ * This subroutine detects &&=, ||=, and //= and turns an ANDAND, OROR or DORDOR
+ * into an OP_ANDASSIGN, OP_ORASSIGN, or OP_DORASSIGN
  */
 
 STATIC int
@@ -181,6 +225,8 @@ S_ao(pTHX_ int toketype)
            yylval.ival = OP_ANDASSIGN;
        else if (toketype == OROR)
            yylval.ival = OP_ORASSIGN;
+       else if (toketype == DORDOR)
+           yylval.ival = OP_DORASSIGN;
        toketype = ASSIGNOP;
     }
     return toketype;
@@ -259,7 +305,7 @@ S_missingterm(pTHX_ char *s)
        s = tmpbuf;
     }
     else {
-       *tmpbuf = PL_multi_close;
+       *tmpbuf = (char)PL_multi_close;
        tmpbuf[1] = '\0';
        s = tmpbuf;
     }
@@ -274,9 +320,24 @@ S_missingterm(pTHX_ char *s)
 void
 Perl_deprecate(pTHX_ char *s)
 {
-    dTHR;
     if (ckWARN(WARN_DEPRECATED))
-       Perl_warner(aTHX_ WARN_DEPRECATED, "Use of %s is deprecated", s);
+       Perl_warner(aTHX_ packWARN(WARN_DEPRECATED), "Use of %s is deprecated", s);
+}
+
+void
+Perl_deprecate_old(pTHX_ char *s)
+{
+    /* This function should NOT be called for any new deprecated warnings */
+    /* Use Perl_deprecate instead                                         */
+    /*                                                                    */
+    /* It is here to maintain backward compatibility with the pre-5.8     */
+    /* warnings category hierarchy. The "deprecated" category used to     */
+    /* live under the "syntax" category. It is now a top-level category   */
+    /* in its own right.                                                  */
+
+    if (ckWARN2(WARN_DEPRECATED, WARN_SYNTAX))
+       Perl_warner(aTHX_ packWARN2(WARN_DEPRECATED, WARN_SYNTAX), 
+                       "Use of %s is deprecated", s);
 }
 
 /*
@@ -287,7 +348,7 @@ Perl_deprecate(pTHX_ char *s)
 STATIC void
 S_depcom(pTHX)
 {
-    deprecate("comma-less variable list");
+    deprecate_old("comma-less variable list");
 }
 
 /*
@@ -337,7 +398,6 @@ S_cr_textfilter(pTHX_ int idx, SV *sv, int maxlen)
 void
 Perl_lex_start(pTHX_ SV *line)
 {
-    dTHR;
     char *s;
     STRLEN len;
 
@@ -361,10 +421,12 @@ Perl_lex_start(pTHX_ SV *line)
     SAVEPPTR(PL_bufend);
     SAVEPPTR(PL_oldbufptr);
     SAVEPPTR(PL_oldoldbufptr);
+    SAVEPPTR(PL_last_lop);
+    SAVEPPTR(PL_last_uni);
     SAVEPPTR(PL_linestart);
     SAVESPTR(PL_linestr);
-    SAVEPPTR(PL_lex_brackstack);
-    SAVEPPTR(PL_lex_casestack);
+    SAVEGENERICPV(PL_lex_brackstack);
+    SAVEGENERICPV(PL_lex_casestack);
     SAVEDESTRUCTOR_X(restore_rsfp, PL_rsfp);
     SAVESPTR(PL_lex_stuff);
     SAVEI32(PL_lex_defer);
@@ -379,8 +441,6 @@ Perl_lex_start(pTHX_ SV *line)
     PL_lex_brackets = 0;
     New(899, PL_lex_brackstack, 120, char);
     New(899, PL_lex_casestack, 12, char);
-    SAVEFREEPV(PL_lex_brackstack);
-    SAVEFREEPV(PL_lex_casestack);
     PL_lex_casemods = 0;
     *PL_lex_casestack = '\0';
     PL_lex_dojoin = 0;
@@ -395,7 +455,7 @@ Perl_lex_start(pTHX_ SV *line)
     if (SvREADONLY(PL_linestr))
        PL_linestr = sv_2mortal(newSVsv(PL_linestr));
     s = SvPV(PL_linestr, len);
-    if (len && s[len-1] != ';') {
+    if (!len || s[len-1] != ';') {
        if (!(SvFLAGS(PL_linestr) & SVs_TEMP))
            PL_linestr = sv_2mortal(newSVsv(PL_linestr));
        sv_catpvn(PL_linestr, "\n;", 2);
@@ -403,8 +463,7 @@ Perl_lex_start(pTHX_ SV *line)
     SvTEMP_off(PL_linestr);
     PL_oldoldbufptr = PL_oldbufptr = PL_bufptr = PL_linestart = SvPVX(PL_linestr);
     PL_bufend = PL_bufptr + SvCUR(PL_linestr);
-    SvREFCNT_dec(PL_rs);
-    PL_rs = newSVpvn("\n", 1);
+    PL_last_lop = PL_last_uni = Nullch;
     PL_rsfp = 0;
 }
 
@@ -433,7 +492,6 @@ Perl_lex_end(pTHX)
 STATIC void
 S_incline(pTHX_ char *s)
 {
-    dTHR;
     char *t;
     char *n;
     char *e;
@@ -447,9 +505,9 @@ S_incline(pTHX_ char *s)
        s += 4;
     else
        return;
-    if (*s == ' ' || *s == '\t')
+    if (SPACE_OR_TAB(*s))
        s++;
-    else 
+    else
        return;
     while (SPACE_OR_TAB(*s)) s++;
     if (!isDIGIT(*s))
@@ -475,11 +533,7 @@ S_incline(pTHX_ char *s)
     ch = *t;
     *t = '\0';
     if (t - s > 0) {
-#ifdef USE_ITHREADS
-       Safefree(CopFILE(PL_curcop));
-#else
-       SvREFCNT_dec(CopFILEGV(PL_curcop));
-#endif
+       CopFILE_free(PL_curcop);
        CopFILE_set(PL_curcop, s);
     }
     *t = ch;
@@ -495,7 +549,6 @@ S_incline(pTHX_ char *s)
 STATIC char *
 S_skipspace(pTHX_ register char *s)
 {
-    dTHR;
     if (PL_lex_formbrack && PL_lex_brackets <= PL_lex_formbrack) {
        while (s < PL_bufend && SPACE_OR_TAB(*s))
            s++;
@@ -504,7 +557,7 @@ S_skipspace(pTHX_ register char *s)
     for (;;) {
        STRLEN prevlen;
        SSize_t oldprevlen, oldoldprevlen;
-       SSize_t oldloplen, oldunilen;
+       SSize_t oldloplen = 0, oldunilen = 0;
        while (s < PL_bufend && isSPACE(*s)) {
            if (*s++ == '\n' && PL_in_eval && !PL_rsfp)
                incline(s);
@@ -550,6 +603,7 @@ S_skipspace(pTHX_ register char *s)
            PL_oldoldbufptr = PL_oldbufptr = PL_bufptr = s = PL_linestart
                = SvPVX(PL_linestr);
            PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
+           PL_last_lop = PL_last_uni = Nullch;
 
            /* Close the filehandle.  Could be from -P preprocessor,
             * STDIN, or a regular file.  If we were reading code from
@@ -595,6 +649,8 @@ S_skipspace(pTHX_ register char *s)
 
            sv_upgrade(sv, SVt_PVMG);
            sv_setpvn(sv,PL_bufptr,PL_bufend-PL_bufptr);
+            (void)SvIOK_on(sv);
+            SvIVX(sv) = 0;
            av_store(CopFILEAV(PL_curcop),(I32)CopLINE(PL_curcop),sv);
        }
     }
@@ -614,7 +670,6 @@ S_check_uni(pTHX)
 {
     char *s;
     char *t;
-    dTHR;
 
     if (PL_oldoldbufptr != PL_last_uni)
        return;
@@ -626,42 +681,13 @@ S_check_uni(pTHX)
     if (ckWARN_d(WARN_AMBIGUOUS)){
         char ch = *s;
         *s = '\0';
-        Perl_warner(aTHX_ WARN_AMBIGUOUS, 
-                  "Warning: Use of \"%s\" without parens is ambiguous", 
+        Perl_warner(aTHX_ packWARN(WARN_AMBIGUOUS),
+                  "Warning: Use of \"%s\" without parens is ambiguous",
                   PL_last_uni);
         *s = ch;
     }
 }
 
-/* workaround to replace the UNI() macro with a function.  Only the
- * hints/uts.sh file mentions this.  Other comments elsewhere in the
- * source indicate Microport Unix might need it too.
- */
-
-#ifdef CRIPPLED_CC
-
-#undef UNI
-#define UNI(f) return uni(f,s)
-
-STATIC int
-S_uni(pTHX_ I32 f, char *s)
-{
-    yylval.ival = f;
-    PL_expect = XTERM;
-    PL_bufptr = s;
-    PL_last_uni = PL_oldbufptr;
-    PL_last_lop_op = f;
-    if (*s == '(')
-       return FUNC1;
-    s = skipspace(s);
-    if (*s == '(')
-       return FUNC1;
-    else
-       return UNIOP;
-}
-
-#endif /* CRIPPLED_CC */
-
 /*
  * LOP : macro to build a list operator.  Its behaviour has been replaced
  * with a subroutine, S_lop() for which LOP is just another name.
@@ -680,13 +706,13 @@ S_uni(pTHX_ I32 f, char *s)
 STATIC I32
 S_lop(pTHX_ I32 f, int x, char *s)
 {
-    dTHR;
     yylval.ival = f;
     CLINE;
+    REPORT("lop", f)
     PL_expect = x;
     PL_bufptr = s;
     PL_last_lop = PL_oldbufptr;
-    PL_last_lop_op = f;
+    PL_last_lop_op = (OPCODE)f;
     if (PL_nexttoke)
        return LSTOP;
     if (*s == '(')
@@ -707,7 +733,7 @@ S_lop(pTHX_ I32 f, int x, char *s)
  * handles the token correctly.
  */
 
-STATIC void 
+STATIC void
 S_force_next(pTHX_ I32 type)
 {
     PL_nexttype[PL_nexttoke] = type;
@@ -740,7 +766,7 @@ S_force_word(pTHX_ register char *start, int token, int check_keyword, int allow
 {
     register char *s;
     STRLEN len;
-    
+
     start = skipspace(start);
     s = start;
     if (isIDFIRST_lazy_if(s,UTF) ||
@@ -782,7 +808,6 @@ S_force_ident(pTHX_ register char *s, int kind)
        PL_nextval[PL_nexttoke].opval = o;
        force_next(WORD);
        if (kind) {
-           dTHR;               /* just for in_eval */
            o->op_private = OPpCONST_ENTERED;
            /* XXX see note in pp_entereval() for why we forgo typo
               warnings if the symbol must be introduced in an eval.
@@ -810,7 +835,7 @@ Perl_str_to_version(pTHX_ SV *sv)
        STRLEN skip;
        UV n;
        if (utf)
-           n = utf8_to_uv((U8*)start, len, &skip, 0);
+           n = utf8n_to_uvchr((U8*)start, len, &skip, 0);
        else {
            n = *(U8*)start;
            skip = 1;
@@ -822,13 +847,16 @@ Perl_str_to_version(pTHX_ SV *sv)
     return retval;
 }
 
-/* 
+/*
  * S_force_version
  * Forces the next token to be a version number.
+ * If the next token appears to be an invalid version number, (e.g. "v2b"),
+ * and if "guessing" is TRUE, then no new token is created (and the caller
+ * must use an alternative parsing method).
  */
 
 STATIC char *
-S_force_version(pTHX_ char *s)
+S_force_version(pTHX_ char *s, int guessing)
 {
     OP *version = Nullop;
     char *d;
@@ -839,7 +867,8 @@ S_force_version(pTHX_ char *s)
     if (*d == 'v')
        d++;
     if (isDIGIT(*d)) {
-        for (; isDIGIT(*d) || *d == '_' || *d == '.'; d++);
+       while (isDIGIT(*d) || *d == '_' || *d == '.')
+           d++;
         if (*d == ';' || isSPACE(*d) || *d == '}' || !*d) {
            SV *ver;
             s = scan_num(s, &yylval);
@@ -851,13 +880,15 @@ S_force_version(pTHX_ char *s)
                SvNOK_on(ver);          /* hint that it is a version */
            }
         }
+       else if (guessing)
+           return s;
     }
 
     /* NOTE: The parser sees the package name and the VERSION swapped */
     PL_nextval[PL_nexttoke].opval = version;
-    force_next(WORD); 
+    force_next(WORD);
 
-    return (s);
+    return s;
 }
 
 /*
@@ -889,8 +920,11 @@ S_tokeq(pTHX_ SV *sv)
     if (s == send)
        goto finish;
     d = s;
-    if ( PL_hints & HINT_NEW_STRING )
+    if ( PL_hints & HINT_NEW_STRING ) {
        pv = sv_2mortal(newSVpvn(SvPVX(pv), len));
+       if (SvUTF8(sv))
+           SvUTF8_on(pv);
+    }
     while (s < send) {
        if (*s == '\\') {
            if (s + 1 < send && (s[1] == '\\'))
@@ -963,9 +997,12 @@ S_sublex_start(pTHX)
                SvUTF8_on(nsv);
            SvREFCNT_dec(sv);
            sv = nsv;
-       } 
+       }
        yylval.opval = (OP*)newSVOP(op_type, 0, sv);
        PL_lex_stuff = Nullsv;
+       /* Allow <FH> // "foo" */
+       if (op_type == OP_READLINE)
+           PL_expect = XTERMORDORDOR;
        return THING;
     }
 
@@ -995,7 +1032,6 @@ S_sublex_start(pTHX)
 STATIC I32
 S_sublex_push(pTHX)
 {
-    dTHR;
     ENTER;
 
     PL_lex_state = PL_sublex_info.super_state;
@@ -1008,12 +1044,15 @@ S_sublex_push(pTHX)
     SAVEI32(PL_lex_inwhat);
     SAVECOPLINE(PL_curcop);
     SAVEPPTR(PL_bufptr);
+    SAVEPPTR(PL_bufend);
     SAVEPPTR(PL_oldbufptr);
     SAVEPPTR(PL_oldoldbufptr);
+    SAVEPPTR(PL_last_lop);
+    SAVEPPTR(PL_last_uni);
     SAVEPPTR(PL_linestart);
     SAVESPTR(PL_linestr);
-    SAVEPPTR(PL_lex_brackstack);
-    SAVEPPTR(PL_lex_casestack);
+    SAVEGENERICPV(PL_lex_brackstack);
+    SAVEGENERICPV(PL_lex_casestack);
 
     PL_linestr = PL_lex_stuff;
     PL_lex_stuff = Nullsv;
@@ -1021,19 +1060,18 @@ S_sublex_push(pTHX)
     PL_bufend = PL_bufptr = PL_oldbufptr = PL_oldoldbufptr = PL_linestart
        = SvPVX(PL_linestr);
     PL_bufend += SvCUR(PL_linestr);
+    PL_last_lop = PL_last_uni = Nullch;
     SAVEFREESV(PL_linestr);
 
     PL_lex_dojoin = FALSE;
     PL_lex_brackets = 0;
     New(899, PL_lex_brackstack, 120, char);
     New(899, PL_lex_casestack, 12, char);
-    SAVEFREEPV(PL_lex_brackstack);
-    SAVEFREEPV(PL_lex_casestack);
     PL_lex_casemods = 0;
     *PL_lex_casestack = '\0';
     PL_lex_starts = 0;
     PL_lex_state = LEX_INTERPCONCAT;
-    CopLINE_set(PL_curcop, PL_multi_start);
+    CopLINE_set(PL_curcop, (line_t)PL_multi_start);
 
     PL_lex_inwhat = PL_sublex_info.sub_inwhat;
     if (PL_lex_inwhat == OP_MATCH || PL_lex_inwhat == OP_QR || PL_lex_inwhat == OP_SUBST)
@@ -1053,8 +1091,11 @@ STATIC I32
 S_sublex_done(pTHX)
 {
     if (!PL_lex_starts++) {
+       SV *sv = newSVpvn("",0);
+       if (SvUTF8(PL_linestr))
+           SvUTF8_on(sv);
        PL_expect = XOPERATOR;
-       yylval.opval = (OP*)newSVOP(OP_CONST, 0, newSVpvn("",0));
+       yylval.opval = (OP*)newSVOP(OP_CONST, 0, sv);
        return THING;
     }
 
@@ -1069,6 +1110,7 @@ S_sublex_done(pTHX)
        PL_lex_inpat = 0;
        PL_bufend = PL_bufptr = PL_oldbufptr = PL_oldoldbufptr = PL_linestart = SvPVX(PL_linestr);
        PL_bufend += SvCUR(PL_linestr);
+       PL_last_lop = PL_last_uni = Nullch;
        SAVEFREESV(PL_linestr);
        PL_lex_dojoin = FALSE;
        PL_lex_brackets = 0;
@@ -1141,7 +1183,7 @@ S_sublex_done(pTHX)
   It stops processing as soon as it finds an embedded $ or @ variable
   and leaves it to the caller to work out what's going on.
 
-  @ in pattern could be: @foo, @{foo}, @$foo, @'foo, @:foo.
+  @ in pattern could be: @foo, @{foo}, @$foo, @'foo, @::foo.
 
   $ in pattern could be $foo or could be tail anchor.  Assumption:
   it's a tail anchor if $ is the last thing in the string, or if it's
@@ -1169,7 +1211,7 @@ S_sublex_done(pTHX)
              } (end switch)
          } (end if backslash)
     } (end while character to read)
-                 
+               
 */
 
 STATIC char *
@@ -1181,21 +1223,22 @@ S_scan_const(pTHX_ char *start)
     register char *d = SvPVX(sv);              /* destination for copies */
     bool dorange = FALSE;                      /* are we in a translit range? */
     bool didrange = FALSE;                     /* did we just finish a range? */
-    bool has_utf8 = FALSE;                     /* embedded \x{} */
+    I32  has_utf8 = FALSE;                     /* Output constant is UTF8 */
+    I32  this_utf8 = UTF;                      /* The source string is assumed to be UTF8 */
     UV uv;
 
-    I32 utf = (PL_lex_inwhat == OP_TRANS && PL_sublex_info.sub_op)
-       ? (PL_sublex_info.sub_op->op_private & (OPpTRANS_FROM_UTF|OPpTRANS_TO_UTF))
-       : UTF;
-    I32 this_utf8 = (PL_lex_inwhat == OP_TRANS && PL_sublex_info.sub_op)
-       ? (PL_sublex_info.sub_op->op_private & (PL_lex_repl ?
-                                               OPpTRANS_FROM_UTF : OPpTRANS_TO_UTF))
-       : UTF;
     const char *leaveit =      /* set of acceptably-backslashed characters */
        PL_lex_inpat
            ? "\\.^$@AGZdDwWsSbBpPXC+*?|()-nrtfeaxcz0123456789[{]} \t\n\r\f\v#"
            : "";
 
+    if (PL_lex_inwhat == OP_TRANS && PL_sublex_info.sub_op) {
+       /* If we are doing a trans and we know we want UTF8 set expectation */
+       has_utf8   = PL_sublex_info.sub_op->op_private & (OPpTRANS_FROM_UTF|OPpTRANS_TO_UTF);
+       this_utf8  = PL_sublex_info.sub_op->op_private & (PL_lex_repl ? OPpTRANS_FROM_UTF : OPpTRANS_TO_UTF);
+    }
+
+
     while (s < send || dorange) {
         /* get transliterations out of the way (they're most literal) */
        if (PL_lex_inwhat == OP_TRANS) {
@@ -1205,6 +1248,18 @@ S_scan_const(pTHX_ char *start)
                I32 min;                        /* first character in range */
                I32 max;                        /* last character in range */
 
+               if (has_utf8) {
+                   char *c = (char*)utf8_hop((U8*)d, -1);
+                   char *e = d++;
+                   while (e-- > c)
+                       *(e + 1) = *e;
+                   *c = (char)UTF_TO_NATIVE(0xff);
+                   /* mark the range as done, and continue */
+                   dorange = FALSE;
+                   didrange = TRUE;
+                   continue;
+               }
+
                i = d - SvPVX(sv);              /* remember current offset */
                SvGROW(sv, SvLEN(sv) + 256);    /* never more than 256 chars in a range */
                d = SvPVX(sv) + i;              /* refresh d after realloc */
@@ -1215,41 +1270,41 @@ S_scan_const(pTHX_ char *start)
 
                 if (min > max) {
                    Perl_croak(aTHX_
-                              "Invalid [] range \"%c-%c\" in transliteration operator",
+                              "Invalid range \"%c-%c\" in transliteration operator",
                               (char)min, (char)max);
                 }
 
-#ifndef ASCIIish
+#ifdef EBCDIC
                if ((isLOWER(min) && isLOWER(max)) ||
                    (isUPPER(min) && isUPPER(max))) {
                    if (isLOWER(min)) {
                        for (i = min; i <= max; i++)
                            if (isLOWER(i))
-                               *d++ = i;
+                               *d++ = NATIVE_TO_NEED(has_utf8,i);
                    } else {
                        for (i = min; i <= max; i++)
                            if (isUPPER(i))
-                               *d++ = i;
+                               *d++ = NATIVE_TO_NEED(has_utf8,i);
                    }
                }
                else
 #endif
                    for (i = min; i <= max; i++)
-                       *d++ = i;
+                       *d++ = (char)i;
 
                /* mark the range as done, and continue */
                dorange = FALSE;
                didrange = TRUE;
                continue;
-           } 
+           }
 
            /* range begins (ignore - as first or last char) */
            else if (*s == '-' && s+1 < send  && s != start) {
-               if (didrange) { 
+               if (didrange) {
                    Perl_croak(aTHX_ "Ambiguous range in transliteration operator");
                }
-               if (utf) {
-                   *d++ = (char)0xff;  /* use illegal utf8 byte--see pmtrans */
+               if (has_utf8) {
+                   *d++ = (char)UTF_TO_NATIVE(0xff);   /* use illegal utf8 byte--see pmtrans */
                    s++;
                    continue;
                }
@@ -1268,7 +1323,7 @@ S_scan_const(pTHX_ char *start)
        else if (*s == '(' && PL_lex_inpat && s[1] == '?') {
            if (s[2] == '#') {
                while (s < send && *s != ')')
-                   *d++ = *s++;
+                   *d++ = NATIVE_TO_NEED(has_utf8,*s++);
            }
            else if (s[2] == '{' /* This should match regcomp.c */
                     || ((s[2] == 'p' || s[2] == '?') && s[3] == '{'))
@@ -1280,9 +1335,9 @@ S_scan_const(pTHX_ char *start)
                while (count && (c = *regparse)) {
                    if (c == '\\' && regparse[1])
                        regparse++;
-                   else if (c == '{') 
+                   else if (c == '{')
                        count++;
-                   else if (c == '}') 
+                   else if (c == '}')
                        count--;
                    regparse++;
                }
@@ -1291,7 +1346,7 @@ S_scan_const(pTHX_ char *start)
                    yyerror("Sequence (?{...}) not terminated or not {}-balanced");
                }
                while (s < regparse)
-                   *d++ = *s++;
+                   *d++ = NATIVE_TO_NEED(has_utf8,*s++);
            }
        }
 
@@ -1299,11 +1354,11 @@ S_scan_const(pTHX_ char *start)
        else if (*s == '#' && PL_lex_inpat &&
          ((PMOP*)PL_lex_inpat)->op_pmflags & PMf_EXTENDED) {
            while (s+1 < send && *s != '\n')
-               *d++ = *s++;
+               *d++ = NATIVE_TO_NEED(has_utf8,*s++);
        }
 
        /* check for embedded arrays
-          (@foo, @:foo, @'foo, @{foo}, @$foo, @+, @-)
+          (@foo, @::foo, @'foo, @{foo}, @$foo, @+, @-)
           */
        else if (*s == '@' && s[1]
                 && (isALNUM_lazy_if(s+1,UTF) || strchr(":'{$+-", s[1])))
@@ -1315,31 +1370,11 @@ S_scan_const(pTHX_ char *start)
        else if (*s == '$') {
            if (!PL_lex_inpat)  /* not a regexp, so $ must be var */
                break;
-           if (s + 1 < send && !strchr("()| \n\t", s[1]))
+           if (s + 1 < send && !strchr("()| \r\n\t", s[1]))
                break;          /* in regexp, $ might be tail anchor */
        }
 
-       /* (now in tr/// code again) */
-
-       if (*s & 0x80 && this_utf8) {
-           STRLEN len;
-           UV uv;
-
-           uv = utf8_to_uv((U8*)s, send - s, &len, UTF8_CHECK_ONLY);
-           if (len == (STRLEN)-1) {
-               /* Illegal UTF8 (a high-bit byte), make it valid. */
-               char *old_pvx = SvPVX(sv);
-               /* need space for one extra char (NOTE: SvCUR() not set here) */
-               d = SvGROW(sv, SvLEN(sv) + 1) + (d - old_pvx);
-               d = (char*)uv_to_utf8((U8*)d, (U8)*s++);
-           }
-           else {
-               while (len--)
-                   *d++ = *s++;
-           }
-           has_utf8 = TRUE;
-           continue;
-       }
+       /* End of else if chain - OP_TRANS rejoin rest */
 
        /* backslashes */
        if (*s == '\\' && s+1 < send) {
@@ -1347,8 +1382,8 @@ S_scan_const(pTHX_ char *start)
 
            /* some backslashes we leave behind */
            if (*leaveit && *s && strchr(leaveit, *s)) {
-               *d++ = '\\';
-               *d++ = *s++;
+               *d++ = NATIVE_TO_NEED(has_utf8,'\\');
+               *d++ = NATIVE_TO_NEED(has_utf8,*s++);
                continue;
            }
 
@@ -1356,9 +1391,8 @@ S_scan_const(pTHX_ char *start)
            if (PL_lex_inwhat == OP_SUBST && !PL_lex_inpat &&
                isDIGIT(*s) && *s != '0' && !isDIGIT(s[1]))
            {
-               dTHR;                   /* only for ckWARN */
                if (ckWARN(WARN_SYNTAX))
-                   Perl_warner(aTHX_ WARN_SYNTAX, "\\%c better written as $%c", *s, *s);
+                   Perl_warner(aTHX_ packWARN(WARN_SYNTAX), "\\%c better written as $%c", *s, *s);
                *--s = '$';
                break;
            }
@@ -1381,22 +1415,23 @@ S_scan_const(pTHX_ char *start)
                /* FALL THROUGH */
            default:
                {
-                   dTHR;
-                   if (ckWARN(WARN_MISC) && isALNUM(*s))
-                       Perl_warner(aTHX_ WARN_MISC, 
+                   if (ckWARN(WARN_MISC) &&
+                       isALNUM(*s) && 
+                       *s != '_')
+                       Perl_warner(aTHX_ packWARN(WARN_MISC),
                               "Unrecognized escape \\%c passed through",
                               *s);
                    /* default action is to copy the quoted character */
-                   *d++ = *s++;
-                   continue;
+                   goto default_action;
                }
 
            /* \132 indicates an octal constant */
            case '0': case '1': case '2': case '3':
            case '4': case '5': case '6': case '7':
                {
-                   STRLEN len = 0;     /* disallow underscores */
-                   uv = (UV)scan_oct(s, 3, &len);
+                    I32 flags = 0;
+                    STRLEN len = 3;
+                   uv = grok_oct(s, &len, &flags, NULL);
                    s += len;
                }
                goto NUM_ESCAPE_INSERT;
@@ -1406,77 +1441,94 @@ S_scan_const(pTHX_ char *start)
                ++s;
                if (*s == '{') {
                    char* e = strchr(s, '}');
+                    I32 flags = PERL_SCAN_ALLOW_UNDERSCORES |
+                      PERL_SCAN_DISALLOW_PREFIX;
+                   STRLEN len;
+
+                    ++s;
                    if (!e) {
                        yyerror("Missing right brace on \\x{}");
-                       e = s;
-                   }
-                   else {
-                       STRLEN len = 1;         /* allow underscores */
-                       uv = (UV)scan_hex(s + 1, e - s - 1, &len);
-                       has_utf8 = TRUE;
+                       continue;
                    }
+                    len = e - s;
+                   uv = grok_hex(s, &len, &flags, NULL);
                    s = e + 1;
                }
                else {
                    {
-                       STRLEN len = 0;         /* disallow underscores */
-                       uv = (UV)scan_hex(s, 2, &len);
+                       STRLEN len = 2;
+                        I32 flags = PERL_SCAN_DISALLOW_PREFIX;
+                       uv = grok_hex(s, &len, &flags, NULL);
                        s += len;
                    }
                }
 
              NUM_ESCAPE_INSERT:
                /* Insert oct or hex escaped character.
-                * There will always enough room in sv since such escapes will
-                * be longer than any utf8 sequence they can end up as
-                */
-               if (uv > 127 || has_utf8) {
-                   if (!this_utf8 && !has_utf8 && uv > 255) {
-                       /* might need to recode whatever we have accumulated so far
-                        * if it contains any hibit chars
+                * There will always enough room in sv since such
+                * escapes will be longer than any UTF-8 sequence
+                * they can end up as. */
+               
+               /* We need to map to chars to ASCII before doing the tests
+                  to cover EBCDIC
+               */
+               if (!UNI_IS_INVARIANT(NATIVE_TO_UNI(uv))) {
+                   if (!has_utf8 && uv > 255) {
+                       /* Might need to recode whatever we have
+                        * accumulated so far if it contains any
+                        * hibit chars.
+                        *
+                        * (Can't we keep track of that and avoid
+                        *  this rescan? --jhi)
                         */
-                       int hicount = 0;
-                       char *c;
-                       for (c = SvPVX(sv); c < d; c++) {
-                           if (*c & 0x80)
+                       int hicount = 0;
+                       U8 *c;
+                       for (c = (U8 *) SvPVX(sv); c < (U8 *)d; c++) {
+                           if (!NATIVE_IS_INVARIANT(*c)) {
                                hicount++;
+                           }
                        }
                        if (hicount) {
-                           char *old_pvx = SvPVX(sv);
-                           char *src, *dst;
-                           d = SvGROW(sv, SvCUR(sv) + hicount + 1) + (d - old_pvx);
-
-                           src = d - 1;
-                           d += hicount;
-                           dst = d - 1;
-
-                           while (src < dst) {
-                               if (*src & 0x80) {
-                                   dst--;
-                                   uv_to_utf8((U8*)dst, (U8)*src--);
-                                   dst--;
+                           STRLEN offset = d - SvPVX(sv);
+                           U8 *src, *dst;
+                           d = SvGROW(sv, SvLEN(sv) + hicount + 1) + offset;
+                           src = (U8 *)d - 1;
+                           dst = src+hicount;
+                           d  += hicount;
+                           while (src >= (U8 *)SvPVX(sv)) {
+                               if (!NATIVE_IS_INVARIANT(*src)) {
+                                   U8 ch = NATIVE_TO_ASCII(*src);
+                                   *dst-- = (U8)UTF8_EIGHT_BIT_LO(ch);
+                                   *dst-- = (U8)UTF8_EIGHT_BIT_HI(ch);
                                }
                                else {
-                                   *dst-- = *src--;
+                                   *dst-- = *src;
                                }
+                               src--;
                            }
                         }
                     }
 
                     if (has_utf8 || uv > 255) {
-                       d = (char*)uv_to_utf8((U8*)d, uv);
-                       this_utf8 = TRUE;
+                       d = (char*)uvchr_to_utf8((U8*)d, uv);
+                       has_utf8 = TRUE;
+                       if (PL_lex_inwhat == OP_TRANS &&
+                           PL_sublex_info.sub_op) {
+                           PL_sublex_info.sub_op->op_private |=
+                               (PL_lex_repl ? OPpTRANS_FROM_UTF
+                                            : OPpTRANS_TO_UTF);
+                       }
                     }
                    else {
                        *d++ = (char)uv;
                    }
                }
                else {
-                   *d++ = (char)uv;
+                   *d++ = (char) uv;
                }
                continue;
 
-           /* \N{latin small letter a} is a named character */
+           /* \N{LATIN SMALL LETTER A} is a named character */
            case 'N':
                ++s;
                if (*s == '{') {
@@ -1484,16 +1536,48 @@ S_scan_const(pTHX_ char *start)
                    SV *res;
                    STRLEN len;
                    char *str;
+
                    if (!e) {
                        yyerror("Missing right brace on \\N{}");
                        e = s - 1;
                        goto cont_scan;
                    }
+                   if (e > s + 2 && s[1] == 'U' && s[2] == '+') {
+                       /* \N{U+...} */
+                       I32 flags = PERL_SCAN_ALLOW_UNDERSCORES |
+                         PERL_SCAN_DISALLOW_PREFIX;
+                       s += 3;
+                       len = e - s;
+                       uv = grok_hex(s, &len, &flags, NULL);
+                       s = e + 1;
+                       goto NUM_ESCAPE_INSERT;
+                   }
                    res = newSVpvn(s + 1, e - s - 1);
-                   res = new_constant( Nullch, 0, "charnames", 
+                   res = new_constant( Nullch, 0, "charnames",
                                        res, Nullsv, "\\N{...}" );
+                   if (has_utf8)
+                       sv_utf8_upgrade(res);
                    str = SvPV(res,len);
+#ifdef EBCDIC_NEVER_MIND
+                   /* charnames uses pack U and that has been
+                    * recently changed to do the below uni->native
+                    * mapping, so this would be redundant (and wrong,
+                    * the code point would be doubly converted).
+                    * But leave this in just in case the pack U change
+                    * gets revoked, but the semantics is still
+                    * desireable for charnames. --jhi */
+                   {
+                        UV uv = utf8_to_uvchr((U8*)str, 0);
+
+                        if (uv < 0x100) {
+                             U8 tmpbuf[UTF8_MAXLEN+1], *d;
+
+                             d = uvchr_to_utf8(tmpbuf, UNI_TO_NATIVE(uv));
+                             sv_setpvn(res, (char *)tmpbuf, d - tmpbuf);
+                             str = SvPV(res, len);
+                        }
+                   }
+#endif
                    if (!has_utf8 && SvUTF8(res)) {
                        char *ostart = SvPVX(sv);
                        SvCUR_set(sv, d - ostart);
@@ -1501,14 +1585,14 @@ S_scan_const(pTHX_ char *start)
                        *d = '\0';
                        sv_utf8_upgrade(sv);
                        /* this just broke our allocation above... */
-                       SvGROW(sv, send - start);
+                       SvGROW(sv, (STRLEN)(send - start));
                        d = SvPVX(sv) + SvCUR(sv);
                        has_utf8 = TRUE;
                    }
-                   if (len > e - s + 4) {
+                   if (len > (STRLEN)(e - s + 4)) { /* I _guess_ 4 is \N{} --jhi */
                        char *odest = SvPVX(sv);
 
-                       SvGROW(sv, (SvCUR(sv) + len - (e - s + 4)));
+                       SvGROW(sv, (SvLEN(sv) + len - (e - s + 4)));
                        d = SvPVX(sv) + (d - odest);
                    }
                    Copy(str, d, len, char);
@@ -1524,66 +1608,87 @@ S_scan_const(pTHX_ char *start)
            /* \c is a control character */
            case 'c':
                s++;
-#ifdef EBCDIC
-               *d = *s++;
-               if (isLOWER(*d))
-                  *d = toUPPER(*d);
-               *d = toCTRL(*d); 
-               d++;
-#else
-               {
+               if (s < send) {
                    U8 c = *s++;
-                   *d++ = toCTRL(c);
-               }
+#ifdef EBCDIC
+                   if (isLOWER(c))
+                       c = toUPPER(c);
 #endif
+                   *d++ = NATIVE_TO_NEED(has_utf8,toCTRL(c));
+               }
+               else {
+                   yyerror("Missing control char name in \\c");
+               }
                continue;
 
            /* printf-style backslashes, formfeeds, newlines, etc */
            case 'b':
-               *d++ = '\b';
+               *d++ = NATIVE_TO_NEED(has_utf8,'\b');
                break;
            case 'n':
-               *d++ = '\n';
+               *d++ = NATIVE_TO_NEED(has_utf8,'\n');
                break;
            case 'r':
-               *d++ = '\r';
+               *d++ = NATIVE_TO_NEED(has_utf8,'\r');
                break;
            case 'f':
-               *d++ = '\f';
+               *d++ = NATIVE_TO_NEED(has_utf8,'\f');
                break;
            case 't':
-               *d++ = '\t';
+               *d++ = NATIVE_TO_NEED(has_utf8,'\t');
                break;
-#ifdef EBCDIC
            case 'e':
-               *d++ = '\047';  /* CP 1047 */
+               *d++ = ASCII_TO_NEED(has_utf8,'\033');
                break;
            case 'a':
-               *d++ = '\057';  /* CP 1047 */
-               break;
-#else
-           case 'e':
-               *d++ = '\033';
+               *d++ = ASCII_TO_NEED(has_utf8,'\007');
                break;
-           case 'a':
-               *d++ = '\007';
-               break;
-#endif
            } /* end switch */
 
            s++;
            continue;
        } /* end if (backslash) */
 
-       *d++ = *s++;
+    default_action:
+       /* If we started with encoded form, or already know we want it
+          and then encode the next character */
+       if ((has_utf8 || this_utf8) && !NATIVE_IS_INVARIANT((U8)(*s))) {
+           STRLEN len  = 1;
+           UV uv       = (this_utf8) ? utf8n_to_uvchr((U8*)s, send - s, &len, 0) : (UV) ((U8) *s);
+           STRLEN need = UNISKIP(NATIVE_TO_UNI(uv));
+           s += len;
+           if (need > len) {
+               /* encoded value larger than old, need extra space (NOTE: SvCUR() not set here) */
+               STRLEN off = d - SvPVX(sv);
+               d = SvGROW(sv, SvLEN(sv) + (need-len)) + off;
+           }
+           d = (char*)uvchr_to_utf8((U8*)d, uv);
+           has_utf8 = TRUE;
+       }
+       else {
+           *d++ = NATIVE_TO_NEED(has_utf8,*s++);
+       }
     } /* while loop to process each character */
 
     /* terminate the string and set up the sv */
     *d = '\0';
     SvCUR_set(sv, d - SvPVX(sv));
+    if (SvCUR(sv) >= SvLEN(sv))
+       Perl_croak(aTHX_ "panic: constant overflowed allocated space");
+
     SvPOK_on(sv);
-    if (has_utf8)
+    if (PL_encoding && !has_utf8) {
+       sv_recode_to_utf8(sv, PL_encoding);
+       if (SvUTF8(sv))
+           has_utf8 = TRUE;
+    }
+    if (has_utf8) {
        SvUTF8_on(sv);
+       if (PL_lex_inwhat == OP_TRANS && PL_sublex_info.sub_op) {
+           PL_sublex_info.sub_op->op_private |=
+                   (PL_lex_repl ? OPpTRANS_FROM_UTF : OPpTRANS_TO_UTF);
+       }
+    }
 
     /* shrink the sv if we allocated more than we used */
     if (SvCUR(sv) + 5 < SvLEN(sv)) {
@@ -1594,9 +1699,9 @@ S_scan_const(pTHX_ char *start)
     /* return the substring (via yylval) only if we parsed anything */
     if (s > PL_bufptr) {
        if ( PL_hints & ( PL_lex_inpat ? HINT_NEW_RE : HINT_NEW_STRING ) )
-           sv = new_constant(start, s - start, (PL_lex_inpat ? "qr" : "q"), 
+           sv = new_constant(start, s - start, (PL_lex_inpat ? "qr" : "q"),
                              sv, Nullsv,
-                             ( PL_lex_inwhat == OP_TRANS 
+                             ( PL_lex_inwhat == OP_TRANS
                                ? "tr"
                                : ( (PL_lex_inwhat == OP_SUBST && !PL_lex_inpat)
                                    ? "s"
@@ -1775,7 +1880,7 @@ S_intuit_more(pTHX_ register char *s)
  * Method if it's "foo $bar"
  * Not a method if it's really "print foo $bar"
  * Method if it's really "foo package::" (interpreted as package->foo)
- * Not a method if bar is known to be a subroutne ("sub bar; foo bar")
+ * Not a method if bar is known to be a subroutine ("sub bar; foo bar")
  * Not a method if bar is a filehandle or package, but is quoted with
  *   =>
  */
@@ -1859,7 +1964,7 @@ S_incl_perldb(pTHX)
 
        if (pdb)
            return pdb;
-       SETERRNO(0,SS$_NORMAL);
+       SETERRNO(0,SS_NORMAL);
        return "BEGIN { require 'perl5db.pl' }";
     }
     return "";
@@ -1867,7 +1972,7 @@ S_incl_perldb(pTHX)
 
 
 /* Encoded script support. filter_add() effectively inserts a
- * 'pre-processing' function into the current source input stream. 
+ * 'pre-processing' function into the current source input stream.
  * Note that the filter function only applies to the current source file
  * (e.g., it will not affect files 'require'd or 'use'd by this one).
  *
@@ -1898,19 +2003,19 @@ Perl_filter_add(pTHX_ filter_t funcp, SV *datasv)
     IoANY(datasv) = (void *)funcp; /* stash funcp into spare field */
     IoFLAGS(datasv) |= IOf_FAKE_DIRP;
     DEBUG_P(PerlIO_printf(Perl_debug_log, "filter_add func %p (%s)\n",
-                         funcp, SvPV_nolen(datasv)));
+                         (void*)funcp, SvPV_nolen(datasv)));
     av_unshift(PL_rsfp_filters, 1);
     av_store(PL_rsfp_filters, 0, datasv) ;
     return(datasv);
 }
+
 
 /* Delete most recently added instance of this filter function.        */
 void
 Perl_filter_del(pTHX_ filter_t funcp)
 {
     SV *datasv;
-    DEBUG_P(PerlIO_printf(Perl_debug_log, "filter_del func %p", funcp));
+    DEBUG_P(PerlIO_printf(Perl_debug_log, "filter_del func %p", (void*)funcp));
     if (!PL_rsfp_filters || AvFILLp(PL_rsfp_filters)<0)
        return;
     /* if filter is on top of stack (usual case) just pop it off */
@@ -1930,8 +2035,8 @@ Perl_filter_del(pTHX_ filter_t funcp)
 /* Invoke the n'th filter function for the current rsfp.        */
 I32
 Perl_filter_read(pTHX_ int idx, SV *buf_sv, int maxlen)
-            
-               
+
+
                                /* 0 = read one text line */
 {
     filter_t funcp;
@@ -1944,13 +2049,13 @@ Perl_filter_read(pTHX_ int idx, SV *buf_sv, int maxlen)
        /* Note that we append to the line. This is handy.      */
        DEBUG_P(PerlIO_printf(Perl_debug_log,
                              "filter_read %d: from rsfp\n", idx));
-       if (maxlen) { 
+       if (maxlen) {
            /* Want a block */
            int len ;
            int old_len = SvCUR(buf_sv) ;
 
            /* ensure buf_sv is large enough */
-           SvGROW(buf_sv, old_len + maxlen) ;
+           SvGROW(buf_sv, (STRLEN)(old_len + maxlen)) ;
            if ((len = PerlIO_read(PL_rsfp, SvPVX(buf_sv) + old_len, maxlen)) <= 0){
                if (PerlIO_error(PL_rsfp))
                    return -1;          /* error */
@@ -1980,11 +2085,11 @@ Perl_filter_read(pTHX_ int idx, SV *buf_sv, int maxlen)
     funcp = (filter_t)IoANY(datasv);
     DEBUG_P(PerlIO_printf(Perl_debug_log,
                          "filter_read %d: via function %p (%s)\n",
-                         idx, funcp, SvPV_nolen(datasv)));
+                         idx, (void*)funcp, SvPV_nolen(datasv)));
     /* Call function. The function is expected to      */
     /* call "FILTER_READ(idx+1, buf_sv)" first.                */
     /* Return: <0:error, =0:eof, >0:not eof            */
-    return (*funcp)(aTHXo_ idx, buf_sv, maxlen);
+    return (*funcp)(aTHX_ idx, buf_sv, maxlen);
 }
 
 STATIC char *
@@ -2037,7 +2142,7 @@ S_find_in_my_stash(pTHX_ char *pkgname, I32 len)
 #ifdef DEBUGGING
     static char* exp_name[] =
        { "OPERATOR", "TERM", "REF", "STATE", "BLOCK", "ATTRBLOCK",
-         "ATTRTERM", "TERMBLOCK"
+         "ATTRTERM", "TERMBLOCK", "TERMORDORDOR"
        };
 #endif
 
@@ -2067,24 +2172,21 @@ S_find_in_my_stash(pTHX_ char *pkgname, I32 len)
 */
 
 #ifdef USE_PURE_BISON
-#ifdef __SC__
-#pragma segment Perl_yylex_r
-#endif
 int
 Perl_yylex_r(pTHX_ YYSTYPE *lvalp, int *lcharp)
 {
-    dTHR;
     int r;
 
+    yyactlevel++;
     yylval_pointer[yyactlevel] = lvalp;
     yychar_pointer[yyactlevel] = lcharp;
-    yyactlevel++;
     if (yyactlevel >= YYMAXLEVEL)
        Perl_croak(aTHX_ "panic: YYMAXLEVEL");
 
     r = Perl_yylex(aTHX);
 
-    yyactlevel--;
+    if (yyactlevel > 0)
+       yyactlevel--;
 
     return r;
 }
@@ -2093,149 +2195,21 @@ Perl_yylex_r(pTHX_ YYSTYPE *lvalp, int *lcharp)
 #ifdef __SC__
 #pragma segment Perl_yylex
 #endif
-
 int
-#ifdef USE_PURE_BISON
-Perl_yylex(pTHX_ YYSTYPE *lvalp, int *lcharp)
-#else
 Perl_yylex(pTHX)
-#endif
 {
-    dTHR;
     register char *s;
     register char *d;
     register I32 tmp;
     STRLEN len;
     GV *gv = Nullgv;
     GV **gvp = 0;
+    bool bof = FALSE;
+    I32 orig_keyword = 0;
 
     /* check if there's an identifier for us to look at */
-    if (PL_pending_ident) {
-        /* pit holds the identifier we read and pending_ident is reset */
-       char pit = PL_pending_ident;
-       PL_pending_ident = 0;
-
-       DEBUG_T({ PerlIO_printf(Perl_debug_log,
-              "### Tokener saw identifier '%s'\n", PL_tokenbuf); })
-
-       /* if we're in a my(), we can't allow dynamics here.
-          $foo'bar has already been turned into $foo::bar, so
-          just check for colons.
-
-          if it's a legal name, the OP is a PADANY.
-       */
-       if (PL_in_my) {
-           if (PL_in_my == KEY_our) {  /* "our" is merely analogous to "my" */
-               if (strchr(PL_tokenbuf,':'))
-                   yyerror(Perl_form(aTHX_ "No package name allowed for "
-                                     "variable %s in \"our\"",
-                                     PL_tokenbuf));
-               tmp = pad_allocmy(PL_tokenbuf);
-           }
-           else {
-               if (strchr(PL_tokenbuf,':'))
-                   yyerror(Perl_form(aTHX_ PL_no_myglob,PL_tokenbuf));
-
-               yylval.opval = newOP(OP_PADANY, 0);
-               yylval.opval->op_targ = pad_allocmy(PL_tokenbuf);
-               return PRIVATEREF;
-           }
-       }
-
-       /* 
-          build the ops for accesses to a my() variable.
-
-          Deny my($a) or my($b) in a sort block, *if* $a or $b is
-          then used in a comparison.  This catches most, but not
-          all cases.  For instance, it catches
-              sort { my($a); $a <=> $b }
-          but not
-              sort { my($a); $a < $b ? -1 : $a == $b ? 0 : 1; }
-          (although why you'd do that is anyone's guess).
-       */
-
-       if (!strchr(PL_tokenbuf,':')) {
-#ifdef USE_THREADS
-           /* Check for single character per-thread SVs */
-           if (PL_tokenbuf[0] == '$' && PL_tokenbuf[2] == '\0'
-               && !isALPHA(PL_tokenbuf[1]) /* Rule out obvious non-threadsvs */
-               && (tmp = find_threadsv(&PL_tokenbuf[1])) != NOT_IN_PAD)
-           {
-               yylval.opval = newOP(OP_THREADSV, 0);
-               yylval.opval->op_targ = tmp;
-               return PRIVATEREF;
-           }
-#endif /* USE_THREADS */
-           if ((tmp = pad_findmy(PL_tokenbuf)) != NOT_IN_PAD) {
-               SV *namesv = AvARRAY(PL_comppad_name)[tmp];
-               /* might be an "our" variable" */
-               if (SvFLAGS(namesv) & SVpad_OUR) {
-                   /* build ops for a bareword */
-                   SV *sym = newSVpv(HvNAME(GvSTASH(namesv)),0);
-                   sv_catpvn(sym, "::", 2);
-                   sv_catpv(sym, PL_tokenbuf+1);
-                   yylval.opval = (OP*)newSVOP(OP_CONST, 0, sym);
-                   yylval.opval->op_private = OPpCONST_ENTERED;
-                   gv_fetchpv(SvPVX(sym),
-                       (PL_in_eval
-                           ? (GV_ADDMULTI | GV_ADDINEVAL)
-                           : TRUE
-                       ),
-                       ((PL_tokenbuf[0] == '$') ? SVt_PV
-                        : (PL_tokenbuf[0] == '@') ? SVt_PVAV
-                        : SVt_PVHV));
-                   return WORD;
-               }
-
-               /* if it's a sort block and they're naming $a or $b */
-               if (PL_last_lop_op == OP_SORT &&
-                   PL_tokenbuf[0] == '$' &&
-                   (PL_tokenbuf[1] == 'a' || PL_tokenbuf[1] == 'b')
-                   && !PL_tokenbuf[2])
-               {
-                   for (d = PL_in_eval ? PL_oldoldbufptr : PL_linestart;
-                        d < PL_bufend && *d != '\n';
-                        d++)
-                   {
-                       if (strnEQ(d,"<=>",3) || strnEQ(d,"cmp",3)) {
-                           Perl_croak(aTHX_ "Can't use \"my %s\" in sort comparison",
-                                 PL_tokenbuf);
-                       }
-                   }
-               }
-
-               yylval.opval = newOP(OP_PADANY, 0);
-               yylval.opval->op_targ = tmp;
-               return PRIVATEREF;
-           }
-       }
-
-       /*
-          Whine if they've said @foo in a doublequoted string,
-          and @foo isn't a variable we can find in the symbol
-          table.
-       */
-       if (pit == '@' && PL_lex_state != LEX_NORMAL && !PL_lex_brackets) {
-           GV *gv = gv_fetchpv(PL_tokenbuf+1, FALSE, SVt_PVAV);
-           if ((!gv || ((PL_tokenbuf[0] == '@') ? !GvAV(gv) : !GvHV(gv)))
-                && ckWARN(WARN_AMBIGUOUS))
-           {
-                /* Downgraded from fatal to warning 20000522 mjd */
-               Perl_warner(aTHX_ WARN_AMBIGUOUS,
-                           "Possible unintended interpolation of %s in string",
-                            PL_tokenbuf);
-           }
-       }
-
-       /* build ops for a bareword */
-       yylval.opval = (OP*)newSVOP(OP_CONST, 0, newSVpv(PL_tokenbuf+1, 0));
-       yylval.opval->op_private = OPpCONST_ENTERED;
-       gv_fetchpv(PL_tokenbuf+1, PL_in_eval ? (GV_ADDMULTI | GV_ADDINEVAL) : TRUE,
-                  ((PL_tokenbuf[0] == '$') ? SVt_PV
-                   : (PL_tokenbuf[0] == '@') ? SVt_PVAV
-                   : SVt_PVHV));
-       return WORD;
-    }
+    if (PL_pending_ident)
+        return S_pending_ident(aTHX);
 
     /* no identifier pending identification */
 
@@ -2257,7 +2231,7 @@ Perl_yylex(pTHX)
        }
        DEBUG_T({ PerlIO_printf(Perl_debug_log,
               "### Next token after '%s' was known, type %"IVdf"\n", PL_bufptr,
-              (IV)PL_nexttype[PL_nexttoke]); })
+              (IV)PL_nexttype[PL_nexttoke]); });
 
        return(PL_nexttype[PL_nexttoke]);
 
@@ -2291,41 +2265,42 @@ Perl_yylex(pTHX)
        }
        else {
            DEBUG_T({ PerlIO_printf(Perl_debug_log,
-              "### Saw case modifier at '%s'\n", PL_bufptr); })
+              "### Saw case modifier at '%s'\n", PL_bufptr); });
            s = PL_bufptr + 1;
-           if (strnEQ(s, "L\\u", 3) || strnEQ(s, "U\\l", 3))
-               tmp = *s, *s = s[2], s[2] = tmp;        /* misordered... */
-           if (strchr("LU", *s) &&
-               (strchr(PL_lex_casestack, 'L') || strchr(PL_lex_casestack, 'U')))
-           {
-               PL_lex_casestack[--PL_lex_casemods] = '\0';
-               return ')';
+           if (s[1] == '\\' && s[2] == 'E') {
+               PL_bufptr = s + 3;
+               PL_lex_state = LEX_INTERPCONCAT;
+               return yylex();
            }
-           if (PL_lex_casemods > 10) {
-               char* newlb = Renew(PL_lex_casestack, PL_lex_casemods + 2, char);
-               if (newlb != PL_lex_casestack) {
-                   SAVEFREEPV(newlb);
-                   PL_lex_casestack = newlb;
+           else {
+               if (strnEQ(s, "L\\u", 3) || strnEQ(s, "U\\l", 3))
+                   tmp = *s, *s = s[2], s[2] = (char)tmp;      /* misordered... */
+               if (strchr("LU", *s) &&
+                   (strchr(PL_lex_casestack, 'L') || strchr(PL_lex_casestack, 'U'))) {
+                   PL_lex_casestack[--PL_lex_casemods] = '\0';
+                   return ')';
                }
+               if (PL_lex_casemods > 10)
+                   Renew(PL_lex_casestack, PL_lex_casemods + 2, char);
+               PL_lex_casestack[PL_lex_casemods++] = *s;
+               PL_lex_casestack[PL_lex_casemods] = '\0';
+               PL_lex_state = LEX_INTERPCONCAT;
+               PL_nextval[PL_nexttoke].ival = 0;
+               force_next('(');
+               if (*s == 'l')
+                   PL_nextval[PL_nexttoke].ival = OP_LCFIRST;
+               else if (*s == 'u')
+                   PL_nextval[PL_nexttoke].ival = OP_UCFIRST;
+               else if (*s == 'L')
+                   PL_nextval[PL_nexttoke].ival = OP_LC;
+               else if (*s == 'U')
+                   PL_nextval[PL_nexttoke].ival = OP_UC;
+               else if (*s == 'Q')
+                   PL_nextval[PL_nexttoke].ival = OP_QUOTEMETA;
+               else
+                   Perl_croak(aTHX_ "panic: yylex");
+               PL_bufptr = s + 1;
            }
-           PL_lex_casestack[PL_lex_casemods++] = *s;
-           PL_lex_casestack[PL_lex_casemods] = '\0';
-           PL_lex_state = LEX_INTERPCONCAT;
-           PL_nextval[PL_nexttoke].ival = 0;
-           force_next('(');
-           if (*s == 'l')
-               PL_nextval[PL_nexttoke].ival = OP_LCFIRST;
-           else if (*s == 'u')
-               PL_nextval[PL_nexttoke].ival = OP_UCFIRST;
-           else if (*s == 'L')
-               PL_nextval[PL_nexttoke].ival = OP_LC;
-           else if (*s == 'U')
-               PL_nextval[PL_nexttoke].ival = OP_UC;
-           else if (*s == 'Q')
-               PL_nextval[PL_nexttoke].ival = OP_QUOTEMETA;
-           else
-               Perl_croak(aTHX_ "panic: yylex");
-           PL_bufptr = s + 1;
            force_next(FUNC);
            if (PL_lex_starts) {
                s = PL_bufptr;
@@ -2343,20 +2318,14 @@ Perl_yylex(pTHX)
        if (PL_bufptr == PL_bufend)
            return sublex_done();
        DEBUG_T({ PerlIO_printf(Perl_debug_log,
-              "### Interpolated variable at '%s'\n", PL_bufptr); })
+              "### Interpolated variable at '%s'\n", PL_bufptr); });
        PL_expect = XTERM;
        PL_lex_dojoin = (*PL_bufptr == '@');
        PL_lex_state = LEX_INTERPNORMAL;
        if (PL_lex_dojoin) {
            PL_nextval[PL_nexttoke].ival = 0;
            force_next(',');
-#ifdef USE_THREADS
-           PL_nextval[PL_nexttoke].opval = newOP(OP_THREADSV, 0);
-           PL_nextval[PL_nexttoke].opval->op_targ = find_threadsv("\"");
-           force_next(PRIVATEREF);
-#else
            force_ident("\"", '$');
-#endif /* USE_THREADS */
            PL_nextval[PL_nexttoke].ival = 0;
            force_next('$');
            PL_nextval[PL_nexttoke].ival = 0;
@@ -2443,7 +2412,7 @@ Perl_yylex(pTHX)
     DEBUG_T( {
        PerlIO_printf(Perl_debug_log, "### Tokener expecting %s at %s\n",
                      exp_name[PL_expect], s);
-    } )
+    } );
 
   retry:
     switch (*s) {
@@ -2460,9 +2429,9 @@ Perl_yylex(pTHX)
            PL_last_lop = 0;
            if (PL_lex_brackets)
                yyerror("Missing right curly or square bracket");
-            DEBUG_T( { PerlIO_printf(Perl_debug_log, 
+            DEBUG_T( { PerlIO_printf(Perl_debug_log,
                         "### Tokener got EOF\n");
-            } )
+            } );
            TOKEN(0);
        }
        if (s++ < PL_bufend)
@@ -2489,19 +2458,16 @@ Perl_yylex(pTHX)
                if (PL_minus_l)
                    sv_catpv(PL_linestr,"chomp;");
                if (PL_minus_a) {
-                   GV* gv = gv_fetchpv("::F", TRUE, SVt_PVAV);
-                   if (gv)
-                       GvIMPORTED_AV_on(gv);
                    if (PL_minus_F) {
                        if (strchr("/'\"", *PL_splitstr)
                              && strchr(PL_splitstr + 1, *PL_splitstr))
-                           Perl_sv_catpvf(aTHX_ PL_linestr, "@F=split(%s);", PL_splitstr);
+                           Perl_sv_catpvf(aTHX_ PL_linestr, "our @F=split(%s);", PL_splitstr);
                        else {
                            char delim;
                            s = "'~#\200\1'"; /* surely one char is unused...*/
                            while (s[1] && strchr(PL_splitstr, *s))  s++;
                            delim = *s;
-                           Perl_sv_catpvf(aTHX_ PL_linestr, "@F=split(%s%c",
+                           Perl_sv_catpvf(aTHX_ PL_linestr, "our @F=split(%s%c",
                                      "q" + (delim == '\''), delim);
                            for (s = PL_splitstr; *s; s++) {
                                if (*s == '\\')
@@ -2512,24 +2478,54 @@ Perl_yylex(pTHX)
                        }
                    }
                    else
-                       sv_catpv(PL_linestr,"@F=split(' ');");
+                       sv_catpv(PL_linestr,"our @F=split(' ');");
                }
            }
            sv_catpv(PL_linestr, "\n");
            PL_oldoldbufptr = PL_oldbufptr = s = PL_linestart = SvPVX(PL_linestr);
            PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
+           PL_last_lop = PL_last_uni = Nullch;
            if (PERLDB_LINE && PL_curstash != PL_debstash) {
                SV *sv = NEWSV(85,0);
 
                sv_upgrade(sv, SVt_PVMG);
                sv_setsv(sv,PL_linestr);
+                (void)SvIOK_on(sv);
+                SvIVX(sv) = 0;
                av_store(CopFILEAV(PL_curcop),(I32)CopLINE(PL_curcop),sv);
            }
            goto retry;
        }
        do {
-           bool bof = PL_rsfp ? TRUE : FALSE;
-           if (bof) {
+           bof = PL_rsfp ? TRUE : FALSE;
+           if ((s = filter_gets(PL_linestr, PL_rsfp, 0)) == Nullch) {
+             fake_eof:
+               if (PL_rsfp) {
+                   if (PL_preprocess && !PL_in_eval)
+                       (void)PerlProc_pclose(PL_rsfp);
+                   else if ((PerlIO *)PL_rsfp == PerlIO_stdin())
+                       PerlIO_clearerr(PL_rsfp);
+                   else
+                       (void)PerlIO_close(PL_rsfp);
+                   PL_rsfp = Nullfp;
+                   PL_doextract = FALSE;
+               }
+               if (!PL_in_eval && (PL_minus_n || PL_minus_p)) {
+                   sv_setpv(PL_linestr,PL_minus_p ? ";}continue{print" : "");
+                   sv_catpv(PL_linestr,";}");
+                   PL_oldoldbufptr = PL_oldbufptr = s = PL_linestart = SvPVX(PL_linestr);
+                   PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
+                   PL_last_lop = PL_last_uni = Nullch;
+                   PL_minus_n = PL_minus_p = 0;
+                   goto retry;
+               }
+               PL_oldoldbufptr = PL_oldbufptr = s = PL_linestart = SvPVX(PL_linestr);
+               PL_last_lop = PL_last_uni = Nullch;
+               sv_setpv(PL_linestr,"");
+               TOKEN(';');     /* not infinite loop because rsfp is NULL now */
+           }
+           /* if it looks like the start of a BOM, check if it in fact is */
+           else if (bof && (!*s || *(U8*)s == 0xEF || *(U8*)s >= 0xFE)) {
 #ifdef PERLIO_IS_STDIO
 #  ifdef __GNU_LIBRARY__
 #    if __GNU_LIBRARY__ == 1 /* Linux glibc5 */
@@ -2549,51 +2545,25 @@ Perl_yylex(pTHX)
                 * Workaround?  Maybe attach some extra state to PL_rsfp?
                 */
                if (!PL_preprocess)
-                   bof = PerlIO_tell(PL_rsfp) == 0;
+                   bof = PerlIO_tell(PL_rsfp) == SvCUR(PL_linestr);
 #else
-               bof = PerlIO_tell(PL_rsfp) == 0;
+               bof = PerlIO_tell(PL_rsfp) == (Off_t)SvCUR(PL_linestr);
 #endif
-           }
-           s = filter_gets(PL_linestr, PL_rsfp, 0);
-           if (s == Nullch) {
-             fake_eof:
-               if (PL_rsfp) {
-                   if (PL_preprocess && !PL_in_eval)
-                       (void)PerlProc_pclose(PL_rsfp);
-                   else if ((PerlIO *)PL_rsfp == PerlIO_stdin())
-                       PerlIO_clearerr(PL_rsfp);
-                   else
-                       (void)PerlIO_close(PL_rsfp);
-                   PL_rsfp = Nullfp;
-                   PL_doextract = FALSE;
-               }
-               if (!PL_in_eval && (PL_minus_n || PL_minus_p)) {
-                   sv_setpv(PL_linestr,PL_minus_p ? ";}continue{print" : "");
-                   sv_catpv(PL_linestr,";}");
-                   PL_oldoldbufptr = PL_oldbufptr = s = PL_linestart = SvPVX(PL_linestr);
+               if (bof) {
                    PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
-                   PL_minus_n = PL_minus_p = 0;
-                   goto retry;
+                   s = swallow_bom((U8*)s);
                }
-               PL_oldoldbufptr = PL_oldbufptr = s = PL_linestart = SvPVX(PL_linestr);
-               sv_setpv(PL_linestr,"");
-               TOKEN(';');     /* not infinite loop because rsfp is NULL now */
-           } else if (bof) {
-               PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
-               s = swallow_bom((U8*)s);
            }
            if (PL_doextract) {
-               if (*s == '#' && s[1] == '!' && instr(s,"perl"))
-                   PL_doextract = FALSE;
-
                /* Incest with pod. */
                if (*s == '=' && strnEQ(s, "=cut", 4)) {
                    sv_setpv(PL_linestr, "");
                    PL_oldoldbufptr = PL_oldbufptr = s = PL_linestart = SvPVX(PL_linestr);
                    PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
+                   PL_last_lop = PL_last_uni = Nullch;
                    PL_doextract = FALSE;
                }
-           } 
+           }
            incline(s);
        } while (PL_doextract);
        PL_oldoldbufptr = PL_oldbufptr = PL_bufptr = PL_linestart = s;
@@ -2602,9 +2572,12 @@ Perl_yylex(pTHX)
 
            sv_upgrade(sv, SVt_PVMG);
            sv_setsv(sv,PL_linestr);
+            (void)SvIOK_on(sv);
+            SvIVX(sv) = 0;
            av_store(CopFILEAV(PL_curcop),(I32)CopLINE(PL_curcop),sv);
        }
        PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
+       PL_last_lop = PL_last_uni = Nullch;
        if (CopLINE(PL_curcop) == 1) {
            while (s < PL_bufend && isSPACE(*s))
                s++;
@@ -2641,12 +2614,25 @@ Perl_yylex(pTHX)
                     * at least, set argv[0] to the basename of the Perl
                     * interpreter. So, having found "#!", we'll set it right.
                     */
-                   SV *x = GvSV(gv_fetchpv("\030", TRUE, SVt_PV));
+                   SV *x = GvSV(gv_fetchpv("\030", TRUE, SVt_PV)); /* $^X */
                    assert(SvPOK(x) || SvGMAGICAL(x));
                    if (sv_eq(x, CopFILESV(PL_curcop))) {
                        sv_setpvn(x, ipath, ipathend - ipath);
                        SvSETMAGIC(x);
                    }
+                   else {
+                       STRLEN blen;
+                       STRLEN llen;
+                       char *bstart = SvPV(CopFILESV(PL_curcop),blen);
+                       char *lstart = SvPV(x,llen);
+                       if (llen < blen) {
+                           bstart += blen - llen;
+                           if (strnEQ(bstart, lstart, llen) && bstart[-1] == '/') {
+                               sv_setpvn(x, ipath, ipathend - ipath);
+                               SvSETMAGIC(x);
+                           }
+                       }
+                   }
                    TAINT_NOT;  /* $^X is always tainted, but that's OK */
                }
 #endif /* ARG_ZERO_IS_SCRIPT */
@@ -2718,7 +2704,7 @@ Perl_yylex(pTHX)
                    else
                        newargv = PL_origargv;
                    newargv[0] = ipath;
-                   PerlProc_execv(ipath, newargv);
+                   PerlProc_execv(ipath, EXEC_ARGV_CAST(newargv));
                    Perl_croak(aTHX_ "Can't exec %s", ipath);
                }
 #endif
@@ -2731,6 +2717,7 @@ Perl_yylex(pTHX)
                    while (SPACE_OR_TAB(*d)) d++;
 
                    if (*d++ == '-') {
+                       bool switches_done = PL_doswitches;
                        do {
                            if (*d == 'M' || *d == 'm') {
                                char *m = d;
@@ -2748,11 +2735,20 @@ Perl_yylex(pTHX)
                            sv_setpv(PL_linestr, "");
                            PL_oldoldbufptr = PL_oldbufptr = s = PL_linestart = SvPVX(PL_linestr);
                            PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
+                           PL_last_lop = PL_last_uni = Nullch;
                            PL_preambled = FALSE;
                            if (PERLDB_LINE)
                                (void)gv_fetchfile(PL_origfilename);
                            goto retry;
                        }
+                       if (PL_doswitches && !switches_done) {
+                           int argc = PL_origargc;
+                           char **argv = PL_origargv;
+                           do {
+                               argc--,argv++;
+                           } while (argc && argv[0][0] == '-' && argv[0][1]);
+                           init_argv_symbols(argc,argv);
+                       }
                    }
                }
            }
@@ -2766,7 +2762,7 @@ Perl_yylex(pTHX)
     case '\r':
 #ifdef PERL_STRICT_CR
        Perl_warn(aTHX_ "Illegal character \\%03o (carriage return)", '\r');
-       Perl_croak(aTHX_ 
+       Perl_croak(aTHX_
       "\t(Maybe you didn't strip carriage returns after a network transfer?)\n");
 #endif
     case ' ': case '\t': case '\f': case 013:
@@ -2788,6 +2784,8 @@ Perl_yylex(pTHX)
                s++;
            if (s < d)
                s++;
+           else if (s > d) /* Found by Ilya: feed random input to Perl. */
+             Perl_croak(aTHX_ "panic: input overflow");
            incline(s);
            if (PL_lex_formbrack && PL_lex_brackets <= PL_lex_formbrack) {
                PL_bufptr = s;
@@ -2813,9 +2811,9 @@ Perl_yylex(pTHX)
 
            if (strnEQ(s,"=>",2)) {
                s = force_word(PL_bufptr,WORD,FALSE,FALSE,FALSE);
-                DEBUG_T( { PerlIO_printf(Perl_debug_log, 
+                DEBUG_T( { PerlIO_printf(Perl_debug_log,
                             "### Saw unary minus before =>, forcing word '%s'\n", s);
-                } )
+                } );
                OPERATOR('-');          /* unary minus */
            }
            PL_last_uni = PL_oldbufptr;
@@ -2857,19 +2855,19 @@ Perl_yylex(pTHX)
                break;
            }
            if (ftst) {
-               PL_last_lop_op = ftst;
-               DEBUG_T( { PerlIO_printf(Perl_debug_log, 
-                        "### Saw file test %c\n", ftst);
-               } )
-               if (*s == '(' && ckWARN(WARN_AMBIGUOUS))
-                   Perl_warner(aTHX_ WARN_AMBIGUOUS,
-                               "Ambiguous -%c() resolved as a file test",
-                               tmp, tmp);
+               PL_last_lop_op = (OPCODE)ftst;
+               DEBUG_T( { PerlIO_printf(Perl_debug_log,
+                        "### Saw file test %c\n", (int)ftst);
+               } );
                FTST(ftst);
            }
            else {
                /* Assume it was a minus followed by a one-letter named
                 * subroutine call (or a -bareword), then. */
+               DEBUG_T( { PerlIO_printf(Perl_debug_log,
+                       "### %c looked like a file test but was not\n",
+                       (int)ftst);
+               } );
                s -= 2;
            }
        }
@@ -2987,6 +2985,7 @@ Perl_yylex(pTHX)
                    switch (tmp) {
                    case KEY_or:
                    case KEY_and:
+                   case KEY_err:
                    case KEY_for:
                    case KEY_unless:
                    case KEY_if:
@@ -3000,10 +2999,6 @@ Perl_yylex(pTHX)
                if (*d == '(') {
                    d = scan_str(d,TRUE,TRUE);
                    if (!d) {
-                       if (PL_lex_stuff) {
-                           SvREFCNT_dec(PL_lex_stuff);
-                           PL_lex_stuff = Nullsv;
-                       }
                        /* MUST advance bufptr here to avoid bogus
                           "at end of line" context messages from yyerror().
                         */
@@ -3023,9 +3018,35 @@ Perl_yylex(pTHX)
                    PL_lex_stuff = Nullsv;
                }
                else {
-                   attrs = append_elem(OP_LIST, attrs,
-                                       newSVOP(OP_CONST, 0,
-                                               newSVpvn(s, len)));
+                   /* NOTE: any CV attrs applied here need to be part of
+                      the CVf_BUILTIN_ATTRS define in cv.h! */
+                   if (!PL_in_my && len == 6 && strnEQ(s, "lvalue", len))
+                       CvLVALUE_on(PL_compcv);
+                   else if (!PL_in_my && len == 6 && strnEQ(s, "locked", len))
+                       CvLOCKED_on(PL_compcv);
+                   else if (!PL_in_my && len == 6 && strnEQ(s, "method", len))
+                       CvMETHOD_on(PL_compcv);
+                   else if (!PL_in_my && len == 9 && strnEQ(s, "assertion", len))
+                       CvASSERTION_on(PL_compcv);
+#ifdef USE_ITHREADS
+                   else if (PL_in_my == KEY_our && len == 6 &&
+                            strnEQ(s, "unique", len))
+                       GvUNIQUE_on(cGVOPx_gv(yylval.opval));
+#endif
+                   /* After we've set the flags, it could be argued that
+                      we don't need to do the attributes.pm-based setting
+                      process, and shouldn't bother appending recognized
+                      flags.  To experiment with that, uncomment the
+                      following "else".  (Note that's already been
+                      uncommented.  That keeps the above-applied built-in
+                      attributes from being intercepted (and possibly
+                      rejected) by a package's attribute routines, but is
+                      justified by the performance win for the common case
+                      of applying only built-in attributes.) */
+                   else
+                       attrs = append_elem(OP_LIST, attrs,
+                                           newSVOP(OP_CONST, 0,
+                                                   newSVpvn(s, len)));
                }
                s = skipspace(d);
                if (*s == ':' && s[1] != ':')
@@ -3034,7 +3055,7 @@ Perl_yylex(pTHX)
                    break;      /* require real whitespace or :'s */
            }
            tmp = (PL_expect == XOPERATOR ? '=' : '{'); /*'}(' for vi */
-           if (*s != ';' && *s != tmp && (tmp != '=' || *s != ')')) {
+           if (*s != ';' && *s != '}' && *s != tmp && (tmp != '=' || *s != ')')) {
                char q = ((*s == '\'') ? '"' : '\'');
                /* If here for an expression, and parsed no attrs, back off. */
                if (tmp == '=' && !attrs) {
@@ -3068,6 +3089,7 @@ Perl_yylex(pTHX)
            PL_oldbufptr = PL_oldoldbufptr;             /* allow print(STDOUT 123) */
        else
            PL_expect = XTERM;
+       s = skipspace(s);
        TOKEN('(');
     case ';':
        CLINE;
@@ -3096,11 +3118,7 @@ Perl_yylex(pTHX)
       leftbracket:
        s++;
        if (PL_lex_brackets > 100) {
-           char* newlb = Renew(PL_lex_brackstack, PL_lex_brackets + 1, char);
-           if (newlb != PL_lex_brackstack) {
-               SAVEFREEPV(newlb);
-               PL_lex_brackstack = newlb;
-           }
+           Renew(PL_lex_brackstack, PL_lex_brackets + 10, char);
        }
        switch (PL_expect) {
        case XTERM:
@@ -3154,8 +3172,16 @@ Perl_yylex(pTHX)
                else
                    PL_lex_brackstack[PL_lex_brackets++] = XOPERATOR;
                s = skipspace(s);
-               if (*s == '}')
+               if (*s == '}') {
+                   if (PL_expect == XREF && PL_lex_state == LEX_INTERPNORMAL) {
+                       PL_expect = XTERM;
+                       /* This hack is to get the ${} in the message. */
+                       PL_bufptr = s+1;
+                       yyerror("syntax error");
+                       break;
+                   }
                    OPERATOR(HASHBRACK);
+               }
                /* This hack serves to disambiguate a pair of curlies
                 * as being a block or an anon hash.  Normally, expectation
                 * determines that, but in cases where we're not in a
@@ -3185,12 +3211,17 @@ Perl_yylex(pTHX)
                            || ((*t == 'q' || *t == 'x') && ++t < PL_bufend
                                && !isALNUM(*t))))
                    {
+                       /* skip q//-like construct */
                        char *tmps;
                        char open, close, term;
                        I32 brackets = 1;
 
                        while (t < PL_bufend && isSPACE(*t))
                            t++;
+                       /* check for q => */
+                       if (t+1 < PL_bufend && t[0] == '=' && t[1] == '>') {
+                           OPERATOR(HASHBRACK);
+                       }
                        term = *t;
                        open = term;
                        if (term && (tmps = strchr("([{< )]}> )]}>",term)))
@@ -3203,7 +3234,7 @@ Perl_yylex(pTHX)
                                else if (*t == open)
                                    break;
                            }
-                       else
+                       else {
                            for (t++; t < PL_bufend; t++) {
                                if (*t == '\\' && t+1 < PL_bufend)
                                    t++;
@@ -3212,8 +3243,13 @@ Perl_yylex(pTHX)
                                else if (*t == open)
                                    brackets++;
                            }
+                       }
+                       t++;
                    }
-                   t++;
+                   else
+                       /* skip plain q word */
+                       while (t < PL_bufend && isALNUM_lazy_if(t,UTF))
+                            t += UTF8SKIP(t);
                }
                else if (isALNUM_lazy_if(t,UTF)) {
                    t += UTF8SKIP(t);
@@ -3281,7 +3317,7 @@ Perl_yylex(pTHX)
                && isIDFIRST_lazy_if(s,UTF) && PL_bufptr == PL_linestart)
            {
                CopLINE_dec(PL_curcop);
-               Perl_warner(aTHX_ WARN_SEMICOLON, PL_warn_nosemi);
+               Perl_warner(aTHX_ packWARN(WARN_SEMICOLON), PL_warn_nosemi);
                CopLINE_inc(PL_curcop);
            }
            BAop(OP_BIT_AND);
@@ -3314,7 +3350,7 @@ Perl_yylex(pTHX)
        if (tmp == '~')
            PMop(OP_MATCH);
        if (ckWARN(WARN_SYNTAX) && tmp && isSPACE(*s) && strchr("+-*/%.^&|<",tmp))
-           Perl_warner(aTHX_ WARN_SYNTAX, "Reversed %c= operator",(int)tmp);
+           Perl_warner(aTHX_ packWARN(WARN_SYNTAX), "Reversed %c= operator",(int)tmp);
        s--;
        if (PL_expect == XSTATE && isALPHA(tmp) &&
                (s == PL_linestart+1 || s[-2] == '\n') )
@@ -3458,7 +3494,7 @@ Perl_yylex(pTHX)
                        PL_bufptr = skipspace(PL_bufptr);
                        while (t < PL_bufend && *t != ']')
                            t++;
-                       Perl_warner(aTHX_ WARN_SYNTAX,
+                       Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
                                "Multidimensional syntax %.*s not supported",
                                (t - PL_bufptr) + 1, PL_bufptr);
                    }
@@ -3476,7 +3512,7 @@ Perl_yylex(pTHX)
                        t = scan_word(t, tmpbuf, sizeof tmpbuf, TRUE, &len);
                        for (; isSPACE(*t); t++) ;
                        if (*t == ';' && get_cv(tmpbuf, FALSE))
-                           Perl_warner(aTHX_ WARN_SYNTAX,
+                           Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
                                "You need to quote \"%s\"", tmpbuf);
                    }
                }
@@ -3522,8 +3558,11 @@ Perl_yylex(pTHX)
                PL_expect = XTERM;              /* e.g. print $fh 3 */
            else if (*s == '.' && isDIGIT(s[1]))
                PL_expect = XTERM;              /* e.g. print $fh .3 */
-           else if (strchr("/?-+", *s) && !isSPACE(s[1]) && s[1] != '=')
-               PL_expect = XTERM;              /* e.g. print $fh -1 */
+           else if (strchr("?-+", *s) && !isSPACE(s[1]) && s[1] != '=')
+               PL_expect = XTERM;              /* e.g. print $fh -1 */
+           else if (*s == '/' && !isSPACE(s[1]) && s[1] != '=' && s[1] != '/')
+               PL_expect = XTERM;              /* e.g. print $fh /.../
+                                                XXX except DORDOR operator */
            else if (*s == '<' && s[1] == '<' && !isSPACE(s[2]) && s[2] != '=')
                PL_expect = XTERM;              /* print $fh <<"EOF" */
        }
@@ -3555,7 +3594,7 @@ Perl_yylex(pTHX)
                    if (*t == '}' || *t == ']') {
                        t++;
                        PL_bufptr = skipspace(PL_bufptr);
-                       Perl_warner(aTHX_ WARN_SYNTAX,
+                       Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
                            "Scalar value %.*s better written as $%.*s",
                            t-PL_bufptr, PL_bufptr, t-PL_bufptr-1, PL_bufptr+1);
                    }
@@ -3565,22 +3604,40 @@ Perl_yylex(pTHX)
        PL_pending_ident = '@';
        TERM('@');
 
-    case '/':                  /* may either be division or pattern */
-    case '?':                  /* may either be conditional or pattern */
-       if (PL_expect != XOPERATOR) {
-           /* Disable warning on "study /blah/" */
-           if (PL_oldoldbufptr == PL_last_uni 
-               && (*PL_last_uni != 's' || s - PL_last_uni < 5 
-                   || memNE(PL_last_uni, "study", 5)
-                   || isALNUM_lazy_if(PL_last_uni+5,UTF)))
-               check_uni();
-           s = scan_pat(s,OP_MATCH);
-           TERM(sublex_start());
-       }
-       tmp = *s++;
-       if (tmp == '/')
-           Mop(OP_DIVIDE);
-       OPERATOR(tmp);
+     case '/':                 /* may be division, defined-or, or pattern */
+       if (PL_expect == XTERMORDORDOR && s[1] == '/') {
+           s += 2;
+           AOPERATOR(DORDOR);
+       }
+     case '?':                 /* may either be conditional or pattern */
+        if(PL_expect == XOPERATOR) {
+            tmp = *s++;
+            if(tmp == '?') {
+                 OPERATOR('?');
+            }
+             else {
+                tmp = *s++;
+                if(tmp == '/') {
+                    /* A // operator. */
+                   AOPERATOR(DORDOR);
+                }
+                else {
+                    s--;
+                    Mop(OP_DIVIDE);
+                }
+            }
+        }
+        else {
+            /* Disable warning on "study /blah/" */
+            if (PL_oldoldbufptr == PL_last_uni
+             && (*PL_last_uni != 's' || s - PL_last_uni < 5
+                 || memNE(PL_last_uni, "study", 5)
+                 || isALNUM_lazy_if(PL_last_uni+5,UTF)
+             ))
+                check_uni();
+            s = scan_pat(s,OP_MATCH);
+            TERM(sublex_start());
+        }
 
     case '.':
        if (PL_lex_formbrack && PL_lex_brackets == PL_lex_formbrack
@@ -3615,18 +3672,18 @@ Perl_yylex(pTHX)
     case '0': case '1': case '2': case '3': case '4':
     case '5': case '6': case '7': case '8': case '9':
        s = scan_num(s, &yylval);
-        DEBUG_T( { PerlIO_printf(Perl_debug_log, 
+        DEBUG_T( { PerlIO_printf(Perl_debug_log,
                     "### Saw number in '%s'\n", s);
-        } )
+        } );
        if (PL_expect == XOPERATOR)
            no_op("Number",s);
        TERM(THING);
 
     case '\'':
        s = scan_str(s,FALSE,FALSE);
-        DEBUG_T( { PerlIO_printf(Perl_debug_log, 
-                    "### Saw string in '%s'\n", s);
-        } )
+        DEBUG_T( { PerlIO_printf(Perl_debug_log,
+                    "### Saw string before '%s'\n", s);
+        } );
        if (PL_expect == XOPERATOR) {
            if (PL_lex_formbrack && PL_lex_brackets == PL_lex_formbrack) {
                PL_expect = XTERM;
@@ -3643,9 +3700,9 @@ Perl_yylex(pTHX)
 
     case '"':
        s = scan_str(s,FALSE,FALSE);
-        DEBUG_T( { PerlIO_printf(Perl_debug_log, 
-                    "### Saw string in '%s'\n", s);
-        } )
+        DEBUG_T( { PerlIO_printf(Perl_debug_log,
+                    "### Saw string before '%s'\n", s);
+        } );
        if (PL_expect == XOPERATOR) {
            if (PL_lex_formbrack && PL_lex_brackets == PL_lex_formbrack) {
                PL_expect = XTERM;
@@ -3659,7 +3716,7 @@ Perl_yylex(pTHX)
            missingterm((char*)0);
        yylval.ival = OP_CONST;
        for (d = SvPV(PL_lex_stuff, len); len; len--, d++) {
-           if (*d == '$' || *d == '@' || *d == '\\' || *d & 0x80) {
+           if (*d == '$' || *d == '@' || *d == '\\' || !UTF8_IS_INVARIANT((U8)*d)) {
                yylval.ival = OP_STRINGIFY;
                break;
            }
@@ -3668,9 +3725,9 @@ Perl_yylex(pTHX)
 
     case '`':
        s = scan_str(s,FALSE,FALSE);
-        DEBUG_T( { PerlIO_printf(Perl_debug_log, 
-                    "### Saw backtick string in '%s'\n", s);
-        } )
+        DEBUG_T( { PerlIO_printf(Perl_debug_log,
+                    "### Saw backtick string before '%s'\n", s);
+        } );
        if (PL_expect == XOPERATOR)
            no_op("Backticks",s);
        if (!s)
@@ -3682,7 +3739,7 @@ Perl_yylex(pTHX)
     case '\\':
        s++;
        if (ckWARN(WARN_SYNTAX) && PL_lex_inwhat && isDIGIT(*s))
-           Perl_warner(aTHX_ WARN_SYNTAX,"Can't use \\%c to mean $%c in expression",
+           Perl_warner(aTHX_ packWARN(WARN_SYNTAX),"Can't use \\%c to mean $%c in expression",
                        *s, *s);
        if (PL_expect == XOPERATOR)
            no_op("Backslash",s);
@@ -3700,7 +3757,9 @@ Perl_yylex(pTHX)
                TERM(THING);
            }
            /* avoid v123abc() or $h{v1}, allow C<print v10;> */
-           else if (!isALPHA(*start) && (PL_expect == XTERM || PL_expect == XREF)) {
+           else if (!isALPHA(*start) && (PL_expect == XTERM
+                       || PL_expect == XREF || PL_expect == XSTATE
+                       || PL_expect == XTERMORDORDOR)) {
                char c = *start;
                GV *gv;
                *start = '\0';
@@ -3749,6 +3808,7 @@ Perl_yylex(pTHX)
     case 'z': case 'Z':
 
       keylookup: {
+       orig_keyword = 0;
        gv = Nullgv;
        gvp = 0;
 
@@ -3786,6 +3846,8 @@ Perl_yylex(pTHX)
            CLINE;
            yylval.opval = (OP*)newSVOP(OP_CONST, 0, newSVpv(PL_tokenbuf,0));
            yylval.opval->op_private = OPpCONST_BARE;
+           if (UTF && !IN_BYTES && is_utf8_string((U8*)PL_tokenbuf, len))
+             SvUTF8_on(((SVOP*)yylval.opval)->op_sv);
            TERM(WORD);
        }
 
@@ -3811,6 +3873,7 @@ Perl_yylex(pTHX)
                }
            }
            if (ogv) {
+               orig_keyword = tmp;
                tmp = 0;                /* overridden by import or by GLOBAL */
            }
            else if (gv && !gvp
@@ -3822,11 +3885,15 @@ Perl_yylex(pTHX)
            }
            else {                      /* no override */
                tmp = -tmp;
+               if (tmp == KEY_dump && ckWARN(WARN_MISC)) {
+                   Perl_warner(aTHX_ packWARN(WARN_MISC),
+                           "dump() better written as CORE::dump()");
+               }
                gv = Nullgv;
                gvp = 0;
                if (ckWARN(WARN_AMBIGUOUS) && hgv
                    && tmp != KEY_x && tmp != KEY_CORE) /* never ambiguous */
-                   Perl_warner(aTHX_ WARN_AMBIGUOUS,
+                   Perl_warner(aTHX_ packWARN(WARN_AMBIGUOUS),
                        "Ambiguous call resolved as CORE::%s(), %s",
                         GvENAME(hgv), "qualify as such or use &");
            }
@@ -3838,6 +3905,7 @@ Perl_yylex(pTHX)
        default:                        /* not a keyword */
          just_a_word: {
                SV *sv;
+               int pkgname = 0;
                char lastchar = (PL_bufptr == PL_oldoldbufptr ? 0 : PL_bufptr[-1]);
 
                /* Get the rest if it looks like a package qualifier */
@@ -3850,12 +3918,13 @@ Perl_yylex(pTHX)
                        Perl_croak(aTHX_ "Bad name after %s%s", PL_tokenbuf,
                                *s == '\'' ? "'" : "::");
                    len += morelen;
+                   pkgname = 1;
                }
 
                if (PL_expect == XOPERATOR) {
                    if (PL_bufptr == PL_linestart) {
                        CopLINE_dec(PL_curcop);
-                       Perl_warner(aTHX_ WARN_SEMICOLON, PL_warn_nosemi);
+                       Perl_warner(aTHX_ packWARN(WARN_SEMICOLON), PL_warn_nosemi);
                        CopLINE_inc(PL_curcop);
                    }
                    else
@@ -3870,7 +3939,7 @@ Perl_yylex(pTHX)
                    PL_tokenbuf[len - 2] == ':' && PL_tokenbuf[len - 1] == ':')
                {
                    if (ckWARN(WARN_BAREWORD) && ! gv_fetchpv(PL_tokenbuf, FALSE, SVt_PVHV))
-                       Perl_warner(aTHX_ WARN_BAREWORD, 
+                       Perl_warner(aTHX_ packWARN(WARN_BAREWORD),
                            "Bareword \"%s\" refers to nonexistent package",
                             PL_tokenbuf);
                    len -= 2;
@@ -3898,6 +3967,10 @@ Perl_yylex(pTHX)
                CLINE;
                yylval.opval = (OP*)newSVOP(OP_CONST, 0, sv);
                yylval.opval->op_private = OPpCONST_BARE;
+               /* UTF-8 package name? */
+               if (UTF && !IN_BYTES &&
+                   is_utf8_string((U8*)SvPVX(sv), SvCUR(sv)))
+                   SvUTF8_on(sv);
 
                /* And if "Foo::", then that's what it certainly is. */
 
@@ -3927,24 +4000,25 @@ Perl_yylex(pTHX)
                    /* If not a declared subroutine, it's an indirect object. */
                    /* (But it's an indir obj regardless for sort.) */
 
-                   if ((PL_last_lop_op == OP_SORT ||
-                         (!immediate_paren && (!gv || !GvCVu(gv)))) &&
+                   if ( !immediate_paren && (PL_last_lop_op == OP_SORT ||
+                         ((!gv || !GvCVu(gv)) &&
                         (PL_last_lop_op != OP_MAPSTART &&
-                        PL_last_lop_op != OP_GREPSTART))
+                        PL_last_lop_op != OP_GREPSTART))))
                    {
                        PL_expect = (PL_last_lop == PL_oldoldbufptr) ? XTERM : XOPERATOR;
                        goto bareword;
                    }
                }
 
-
                PL_expect = XOPERATOR;
                s = skipspace(s);
 
                /* Is this a word before a => operator? */
-               if (*s == '=' && s[1] == '>') {
+               if (*s == '=' && s[1] == '>' && !pkgname) {
                    CLINE;
                    sv_setpv(((SVOP*)yylval.opval)->op_sv, PL_tokenbuf);
+                   if (UTF && !IN_BYTES && is_utf8_string((U8*)PL_tokenbuf, len))
+                     SvUTF8_on(((SVOP*)yylval.opval)->op_sv);
                    TERM(WORD);
                }
 
@@ -3975,7 +4049,9 @@ Perl_yylex(pTHX)
 
                /* If followed by a bareword, see if it looks like indir obj. */
 
-               if ((isIDFIRST_lazy_if(s,UTF) || *s == '$') && (tmp = intuit_method(s,gv)))
+               if (!orig_keyword
+                       && (isIDFIRST_lazy_if(s,UTF) || *s == '$')
+                       && (tmp = intuit_method(s,gv)))
                    return tmp;
 
                /* Not a method, so call it a subroutine (if defined) */
@@ -3983,7 +4059,7 @@ Perl_yylex(pTHX)
                if (gv && GvCVu(gv)) {
                    CV* cv;
                    if (lastchar == '-' && ckWARN_d(WARN_AMBIGUOUS))
-                       Perl_warner(aTHX_ WARN_AMBIGUOUS,
+                       Perl_warner(aTHX_ packWARN(WARN_AMBIGUOUS),
                                "Ambiguous use of -%s resolved as -&%s()",
                                PL_tokenbuf, PL_tokenbuf);
                    /* Check for a constant sub */
@@ -4010,8 +4086,11 @@ Perl_yylex(pTHX)
                            TERM(FUNC0SUB);
                        if (strEQ(proto, "$"))
                            OPERATOR(UNIOPSUB);
+                       while (*proto == ';')
+                           proto++;
                        if (*proto == '&' && *s == '{') {
-                           sv_setpv(PL_subname,"__ANON__");
+                           sv_setpv(PL_subname, PL_curstash ? 
+                                       "__ANON__" : "__ANON__::__ANON__");
                            PREBLOCK(LSTOPSUB);
                        }
                    }
@@ -4030,8 +4109,8 @@ Perl_yylex(pTHX)
                    if (ckWARN(WARN_RESERVED)) {
                        if (lastchar != '-') {
                            for (d = PL_tokenbuf; *d && isLOWER(*d); d++) ;
-                           if (!*d)
-                               Perl_warner(aTHX_ WARN_RESERVED, PL_warn_reserved,
+                           if (!*d && !gv_stashpv(PL_tokenbuf,FALSE))
+                               Perl_warner(aTHX_ packWARN(WARN_RESERVED), PL_warn_reserved,
                                       PL_tokenbuf);
                        }
                    }
@@ -4039,10 +4118,10 @@ Perl_yylex(pTHX)
 
            safe_bareword:
                if (lastchar && strchr("*%&", lastchar) && ckWARN_d(WARN_AMBIGUOUS)) {
-                   Perl_warner(aTHX_ WARN_AMBIGUOUS,
+                   Perl_warner(aTHX_ packWARN(WARN_AMBIGUOUS),
                        "Operator or semicolon missing before %c%s",
                        lastchar, PL_tokenbuf);
-                   Perl_warner(aTHX_ WARN_AMBIGUOUS,
+                   Perl_warner(aTHX_ packWARN(WARN_AMBIGUOUS),
                        "Ambiguous use of %c resolved as operator %c",
                        lastchar, lastchar);
                }
@@ -4106,16 +4185,47 @@ Perl_yylex(pTHX)
                        loc = PerlIO_tell(PL_rsfp);
                        (void)PerlIO_seek(PL_rsfp, 0L, 0);
                    }
+#ifdef NETWARE
+                       if (PerlLIO_setmode(PL_rsfp, O_TEXT) != -1) {
+#else
                    if (PerlLIO_setmode(PerlIO_fileno(PL_rsfp), O_TEXT) != -1) {
-#if defined(__BORLANDC__)
+#endif /* NETWARE */
+#ifdef PERLIO_IS_STDIO /* really? */
+#  if defined(__BORLANDC__)
                        /* XXX see note in do_binmode() */
-                       ((FILE*)PL_rsfp)->flags |= _F_BIN;
+                       ((FILE*)PL_rsfp)->flags &= ~_F_BIN;
+#  endif
 #endif
                        if (loc > 0)
                            PerlIO_seek(PL_rsfp, loc, 0);
                    }
                }
 #endif
+#ifdef PERLIO_LAYERS
+               if (!IN_BYTES) {
+                   if (UTF)
+                       PerlIO_apply_layers(aTHX_ PL_rsfp, NULL, ":utf8");
+                   else if (PL_encoding) {
+                       SV *name;
+                       dSP;
+                       ENTER;
+                       SAVETMPS;
+                       PUSHMARK(sp);
+                       EXTEND(SP, 1);
+                       XPUSHs(PL_encoding);
+                       PUTBACK;
+                       call_method("name", G_SCALAR);
+                       SPAGAIN;
+                       name = POPs;
+                       PUTBACK;
+                       PerlIO_apply_layers(aTHX_ PL_rsfp, NULL, 
+                                           Perl_form(aTHX_ ":encoding(%"SVf")",
+                                                     name));
+                       FREETMPS;
+                       LEAVE;
+                   }
+               }
+#endif
                PL_rsfp = Nullfp;
            }
            goto fake_eof;
@@ -4202,12 +4312,6 @@ Perl_yylex(pTHX)
            LOP(OP_CRYPT,XTERM);
 
        case KEY_chmod:
-           if (ckWARN(WARN_CHMOD)) {
-               for (d = s; d < PL_bufend && (isSPACE(*d) || *d == '('); d++) ;
-               if (*d != '0' && isDIGIT(*d))
-                   Perl_warner(aTHX_ WARN_CHMOD,
-                               "chmod() mode argument is missing initial 0");
-           }
            LOP(OP_CHMOD,XTERM);
 
        case KEY_chown:
@@ -4230,7 +4334,7 @@ Perl_yylex(pTHX)
            if (*s == '{')
                PRETERMBLOCK(DO);
            if (*s != '\'')
-               s = force_word(s,WORD,FALSE,TRUE,FALSE);
+               s = force_word(s,WORD,TRUE,TRUE,FALSE);
            OPERATOR(DO);
 
        case KEY_die:
@@ -4266,7 +4370,7 @@ Perl_yylex(pTHX)
 
        case KEY_exists:
            UNI(OP_EXISTS);
-           
+       
        case KEY_exit:
            UNI(OP_EXIT);
 
@@ -4278,6 +4382,9 @@ Perl_yylex(pTHX)
        case KEY_eof:
            UNI(OP_EOF);
 
+       case KEY_err:
+           OPERATOR(DOROP);
+
        case KEY_exp:
            UNI(OP_EXP);
 
@@ -4361,7 +4468,7 @@ Perl_yylex(pTHX)
            UNI(OP_GMTIME);
 
        case KEY_getc:
-           UNI(OP_GETC);
+           UNIDOR(OP_GETC);
 
        case KEY_getppid:
            FUN0(OP_GETPPID);
@@ -4470,7 +4577,7 @@ Perl_yylex(pTHX)
        case KEY_last:
            s = force_word(s,WORD,TRUE,FALSE,FALSE);
            LOOPX(OP_LAST);
-           
+       
        case KEY_lc:
            UNI(OP_LC);
 
@@ -4560,7 +4667,7 @@ Perl_yylex(pTHX)
            if (PL_expect != XSTATE)
                yyerror("\"no\" not allowed in expression");
            s = force_word(s,WORD,FALSE,TRUE,FALSE);
-           s = force_version(s);
+           s = force_version(s, FALSE);
            yylval.ival = 0;
            OPERATOR(USE);
 
@@ -4576,10 +4683,14 @@ Perl_yylex(pTHX)
                char *t;
                for (d = s; isALNUM_lazy_if(d,UTF); d++) ;
                t = skipspace(d);
-               if (strchr("|&*+-=!?:.", *t) && ckWARN_d(WARN_PRECEDENCE))
-                   Perl_warner(aTHX_ WARN_PRECEDENCE,
+               if (strchr("|&*+-=!?:.", *t) && ckWARN_d(WARN_PRECEDENCE)
+                   /* [perl #16184] */
+                   && !(t[0] == '=' && t[1] == '>')
+               ) {
+                   Perl_warner(aTHX_ packWARN(WARN_PRECEDENCE),
                           "Precedence problem: open %.*s should be open(%.*s)",
-                           d-s,s, d-s,s);
+                           d - s, s, d - s, s);
+               }
            }
            LOP(OP_OPEN,XTERM);
 
@@ -4611,11 +4722,11 @@ Perl_yylex(pTHX)
            LOP(OP_PUSH,XTERM);
 
        case KEY_pop:
-           UNI(OP_POP);
+           UNIDOR(OP_POP);
 
        case KEY_pos:
-           UNI(OP_POS);
-           
+           UNIDOR(OP_POS);
+       
        case KEY_pack:
            LOP(OP_PACK,XTERM);
 
@@ -4646,18 +4757,19 @@ Perl_yylex(pTHX)
                int warned = 0;
                d = SvPV_force(PL_lex_stuff, len);
                while (len) {
+                   SV *sv;
                    for (; isSPACE(*d) && len; --len, ++d) ;
                    if (len) {
                        char *b = d;
                        if (!warned && ckWARN(WARN_QW)) {
                            for (; !isSPACE(*d) && len; --len, ++d) {
                                if (*d == ',') {
-                                   Perl_warner(aTHX_ WARN_QW,
+                                   Perl_warner(aTHX_ packWARN(WARN_QW),
                                        "Possible attempt to separate words with commas");
                                    ++warned;
                                }
                                else if (*d == '#') {
-                                   Perl_warner(aTHX_ WARN_QW,
+                                   Perl_warner(aTHX_ packWARN(WARN_QW),
                                        "Possible attempt to put comments in qw() list");
                                    ++warned;
                                }
@@ -4666,8 +4778,11 @@ Perl_yylex(pTHX)
                        else {
                            for (; !isSPACE(*d) && len; --len, ++d) ;
                        }
+                       sv = newSVpvn(b, d-b);
+                       if (DO_UTF8(PL_lex_stuff))
+                           SvUTF8_on(sv);
                        words = append_elem(OP_LIST, words,
-                                           newSVOP(OP_CONST, 0, tokeq(newSVpvn(b, d-b))));
+                                           newSVOP(OP_CONST, 0, tokeq(sv)));
                    }
                }
                if (words) {
@@ -4675,9 +4790,10 @@ Perl_yylex(pTHX)
                    force_next(THING);
                }
            }
-           if (PL_lex_stuff)
+           if (PL_lex_stuff) {
                SvREFCNT_dec(PL_lex_stuff);
-           PL_lex_stuff = Nullsv;
+               PL_lex_stuff = Nullsv;
+           }
            PL_expect = XTERM;
            TOKEN('(');
 
@@ -4707,10 +4823,12 @@ Perl_yylex(pTHX)
 
        case KEY_require:
            s = skipspace(s);
-           if (isDIGIT(*s) || (*s == 'v' && isDIGIT(s[1]))) {
-               s = force_version(s);
+           if (isDIGIT(*s)) {
+               s = force_version(s, FALSE);
            }
-           else {
+           else if (*s != 'v' || !isDIGIT(s[1])
+                   || (s = force_version(s, TRUE), *s == 'v'))
+           {
                *PL_tokenbuf = '\0';
                s = force_word(s,WORD,TRUE,TRUE,FALSE);
                if (isIDFIRST_lazy_if(PL_tokenbuf,UTF))
@@ -4747,7 +4865,7 @@ Perl_yylex(pTHX)
 
        case KEY_readline:
            set_csh();
-           UNI(OP_READLINE);
+           UNIDOR(OP_READLINE);
 
        case KEY_readpipe:
            set_csh();
@@ -4763,7 +4881,7 @@ Perl_yylex(pTHX)
            LOP(OP_REVERSE,XTERM);
 
        case KEY_readlink:
-           UNI(OP_READLINK);
+           UNIDOR(OP_READLINK);
 
        case KEY_ref:
            UNI(OP_REF);
@@ -4777,7 +4895,7 @@ Perl_yylex(pTHX)
 
        case KEY_chomp:
            UNI(OP_CHOMP);
-           
+       
        case KEY_scalar:
            UNI(OP_SCALAR);
 
@@ -4830,7 +4948,7 @@ Perl_yylex(pTHX)
            LOP(OP_SSOCKOPT,XTERM);
 
        case KEY_shift:
-           UNI(OP_SHIFT);
+           UNIDOR(OP_SHIFT);
 
        case KEY_shmctl:
            LOP(OP_SHMCTL,XTERM);
@@ -4897,9 +5015,9 @@ Perl_yylex(pTHX)
          really_sub:
            {
                char tmpbuf[sizeof PL_tokenbuf];
-               SSize_t tboffset;
+               SSize_t tboffset = 0;
                expectation attrful;
-               bool have_name, have_proto;
+               bool have_name, have_proto, bad_proto;
                int key = tmp;
 
                s = skipspace(s);
@@ -4945,20 +5063,24 @@ Perl_yylex(pTHX)
                    char *p;
 
                    s = scan_str(s,FALSE,FALSE);
-                   if (!s) {
-                       if (PL_lex_stuff)
-                           SvREFCNT_dec(PL_lex_stuff);
-                       PL_lex_stuff = Nullsv;
+                   if (!s)
                        Perl_croak(aTHX_ "Prototype not terminated");
-                   }
-                   /* strip spaces */
+                   /* strip spaces and check for bad characters */
                    d = SvPVX(PL_lex_stuff);
                    tmp = 0;
+                   bad_proto = FALSE;
                    for (p = d; *p; ++p) {
-                       if (!isSPACE(*p))
+                       if (!isSPACE(*p)) {
                            d[tmp++] = *p;
+                           if (!strchr("$@%*;[]&\\", *p))
+                               bad_proto = TRUE;
+                       }
                    }
                    d[tmp] = '\0';
+                   if (bad_proto && ckWARN(WARN_SYNTAX))
+                       Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
+                                   "Illegal character in prototype for %"SVf" : %s",
+                                   PL_subname, d);
                    SvCUR(PL_lex_stuff) = tmp;
                    have_proto = TRUE;
 
@@ -4969,6 +5091,8 @@ Perl_yylex(pTHX)
 
                if (*s == ':' && s[1] != ':')
                    PL_expect = attrful;
+               else if (!have_name && *s != '{' && key == KEY_sub)
+                   Perl_croak(aTHX_ "Illegal declaration of anonymous subroutine");
 
                if (have_proto) {
                    PL_nextval[PL_nexttoke].opval =
@@ -4977,7 +5101,8 @@ Perl_yylex(pTHX)
                    force_next(THING);
                }
                if (!have_name) {
-                   sv_setpv(PL_subname,"__ANON__");
+                   sv_setpv(PL_subname,
+                       PL_curstash ? "__ANON__" : "__ANON__::__ANON__");
                    TOKEN(ANONSUB);
                }
                (void) force_word(PL_oldbufptr + tboffset, WORD,
@@ -5055,7 +5180,7 @@ Perl_yylex(pTHX)
            LOP(OP_UNLINK,XTERM);
 
        case KEY_undef:
-           UNI(OP_UNDEF);
+           UNIDOR(OP_UNDEF);
 
        case KEY_unpack:
            LOP(OP_UNPACK,XTERM);
@@ -5064,13 +5189,7 @@ Perl_yylex(pTHX)
            LOP(OP_UTIME,XTERM);
 
        case KEY_umask:
-           if (ckWARN(WARN_UMASK)) {
-               for (d = s; d < PL_bufend && (isSPACE(*d) || *d == '('); d++) ;
-               if (*d != '0' && isDIGIT(*d)) 
-                   Perl_warner(aTHX_ WARN_UMASK,
-                               "umask: argument is missing initial 0");
-           }
-           UNI(OP_UMASK);
+           UNIDOR(OP_UMASK);
 
        case KEY_unshift:
            LOP(OP_UNSHIFT,XTERM);
@@ -5080,15 +5199,19 @@ Perl_yylex(pTHX)
                yyerror("\"use\" not allowed in expression");
            s = skipspace(s);
            if (isDIGIT(*s) || (*s == 'v' && isDIGIT(s[1]))) {
-               s = force_version(s);
+               s = force_version(s, TRUE);
                if (*s == ';' || (s = skipspace(s), *s == ';')) {
                    PL_nextval[PL_nexttoke].opval = Nullop;
                    force_next(WORD);
                }
+               else if (*s == 'v') {
+                   s = force_word(s,WORD,FALSE,TRUE,FALSE);
+                   s = force_version(s, FALSE);
+               }
            }
            else {
                s = force_word(s,WORD,FALSE,TRUE,FALSE);
-               s = force_version(s);
+               s = force_version(s, FALSE);
            }
            yylval.ival = 1;
            OPERATOR(USE);
@@ -5119,10 +5242,9 @@ Perl_yylex(pTHX)
        case KEY_write:
 #ifdef EBCDIC
        {
-           static char ctl_l[2];
-
-           if (ctl_l[0] == '\0') 
-               ctl_l[0] = toCTRL('L');
+           char ctl_l[2];
+           ctl_l[0] = toCTRL('L');
+           ctl_l[1] = '\0';
            gv_fetchpv(ctl_l,TRUE, SVt_PV);
        }
 #else
@@ -5150,6 +5272,127 @@ Perl_yylex(pTHX)
 #pragma segment Main
 #endif
 
+static int
+S_pending_ident(pTHX)
+{
+    register char *d;
+    register I32 tmp = 0;
+    /* pit holds the identifier we read and pending_ident is reset */
+    char pit = PL_pending_ident;
+    PL_pending_ident = 0;
+
+    DEBUG_T({ PerlIO_printf(Perl_debug_log,
+          "### Tokener saw identifier '%s'\n", PL_tokenbuf); });
+
+    /* if we're in a my(), we can't allow dynamics here.
+       $foo'bar has already been turned into $foo::bar, so
+       just check for colons.
+
+       if it's a legal name, the OP is a PADANY.
+    */
+    if (PL_in_my) {
+        if (PL_in_my == KEY_our) {     /* "our" is merely analogous to "my" */
+            if (strchr(PL_tokenbuf,':'))
+                yyerror(Perl_form(aTHX_ "No package name allowed for "
+                                  "variable %s in \"our\"",
+                                  PL_tokenbuf));
+            tmp = allocmy(PL_tokenbuf);
+        }
+        else {
+            if (strchr(PL_tokenbuf,':'))
+                yyerror(Perl_form(aTHX_ PL_no_myglob,PL_tokenbuf));
+
+            yylval.opval = newOP(OP_PADANY, 0);
+            yylval.opval->op_targ = allocmy(PL_tokenbuf);
+            return PRIVATEREF;
+        }
+    }
+
+    /*
+       build the ops for accesses to a my() variable.
+
+       Deny my($a) or my($b) in a sort block, *if* $a or $b is
+       then used in a comparison.  This catches most, but not
+       all cases.  For instance, it catches
+           sort { my($a); $a <=> $b }
+       but not
+           sort { my($a); $a < $b ? -1 : $a == $b ? 0 : 1; }
+       (although why you'd do that is anyone's guess).
+    */
+
+    if (!strchr(PL_tokenbuf,':')) {
+       if (!PL_in_my)
+           tmp = pad_findmy(PL_tokenbuf);
+        if (tmp != NOT_IN_PAD) {
+            /* might be an "our" variable" */
+            if (PAD_COMPNAME_FLAGS(tmp) & SVpad_OUR) {
+                /* build ops for a bareword */
+                SV *sym = newSVpv(HvNAME(PAD_COMPNAME_OURSTASH(tmp)), 0);
+                sv_catpvn(sym, "::", 2);
+                sv_catpv(sym, PL_tokenbuf+1);
+                yylval.opval = (OP*)newSVOP(OP_CONST, 0, sym);
+                yylval.opval->op_private = OPpCONST_ENTERED;
+                gv_fetchpv(SvPVX(sym),
+                    (PL_in_eval
+                        ? (GV_ADDMULTI | GV_ADDINEVAL)
+                        : GV_ADDMULTI
+                    ),
+                    ((PL_tokenbuf[0] == '$') ? SVt_PV
+                     : (PL_tokenbuf[0] == '@') ? SVt_PVAV
+                     : SVt_PVHV));
+                return WORD;
+            }
+
+            /* if it's a sort block and they're naming $a or $b */
+            if (PL_last_lop_op == OP_SORT &&
+                PL_tokenbuf[0] == '$' &&
+                (PL_tokenbuf[1] == 'a' || PL_tokenbuf[1] == 'b')
+                && !PL_tokenbuf[2])
+            {
+                for (d = PL_in_eval ? PL_oldoldbufptr : PL_linestart;
+                     d < PL_bufend && *d != '\n';
+                     d++)
+                {
+                    if (strnEQ(d,"<=>",3) || strnEQ(d,"cmp",3)) {
+                        Perl_croak(aTHX_ "Can't use \"my %s\" in sort comparison",
+                              PL_tokenbuf);
+                    }
+                }
+            }
+
+            yylval.opval = newOP(OP_PADANY, 0);
+            yylval.opval->op_targ = tmp;
+            return PRIVATEREF;
+        }
+    }
+
+    /*
+       Whine if they've said @foo in a doublequoted string,
+       and @foo isn't a variable we can find in the symbol
+       table.
+    */
+    if (pit == '@' && PL_lex_state != LEX_NORMAL && !PL_lex_brackets) {
+        GV *gv = gv_fetchpv(PL_tokenbuf+1, FALSE, SVt_PVAV);
+        if ((!gv || ((PL_tokenbuf[0] == '@') ? !GvAV(gv) : !GvHV(gv)))
+             && ckWARN(WARN_AMBIGUOUS))
+        {
+            /* Downgraded from fatal to warning 20000522 mjd */
+            Perl_warner(aTHX_ packWARN(WARN_AMBIGUOUS),
+                        "Possible unintended interpolation of %s in string",
+                         PL_tokenbuf);
+        }
+    }
+
+    /* build ops for a bareword */
+    yylval.opval = (OP*)newSVOP(OP_CONST, 0, newSVpv(PL_tokenbuf+1, 0));
+    yylval.opval->op_private = OPpCONST_ENTERED;
+    gv_fetchpv(PL_tokenbuf+1, PL_in_eval ? (GV_ADDMULTI | GV_ADDINEVAL) : TRUE,
+               ((PL_tokenbuf[0] == '$') ? SVt_PV
+                : (PL_tokenbuf[0] == '@') ? SVt_PVAV
+                : SVt_PVHV));
+    return WORD;
+}
+
 I32
 Perl_keyword(pTHX_ register char *d, I32 len)
 {
@@ -5260,6 +5503,7 @@ Perl_keyword(pTHX_ register char *d, I32 len)
            break;
        case 3:
            if (strEQ(d,"eof"))                 return -KEY_eof;
+           if (strEQ(d,"err"))                 return -KEY_err;
            if (strEQ(d,"exp"))                 return -KEY_exp;
            break;
        case 4:
@@ -5493,7 +5737,7 @@ Perl_keyword(pTHX_ register char *d, I32 len)
     case 'p':
        switch (len) {
        case 3:
-           if (strEQ(d,"pop"))                 return -KEY_pop; 
+           if (strEQ(d,"pop"))                 return -KEY_pop;
            if (strEQ(d,"pos"))                 return KEY_pos;
            break;
        case 4:
@@ -5545,7 +5789,7 @@ Perl_keyword(pTHX_ register char *d, I32 len)
            if (strEQ(d,"rindex"))              return -KEY_rindex;
            break;
        case 7:
-           if (strEQ(d,"require"))             return -KEY_require;
+           if (strEQ(d,"require"))             return KEY_require;
            if (strEQ(d,"reverse"))             return -KEY_reverse;
            if (strEQ(d,"readdir"))             return -KEY_readdir;
            break;
@@ -5759,7 +6003,6 @@ S_checkcomma(pTHX_ register char *s, char *name, char *what)
     char *w;
 
     if (*s == ' ' && s[1] == '(') {    /* XXX gotta be a better way */
-       dTHR;                           /* only for ckWARN */
        if (ckWARN(WARN_SYNTAX)) {
            int level = 1;
            for (w = s+2; *w && level; w++) {
@@ -5771,7 +6014,7 @@ S_checkcomma(pTHX_ register char *s, char *name, char *what)
            if (*w)
                for (; *w && isSPACE(*w); w++) ;
            if (!*w || !strchr(";|})]oaiuw!=", *w))     /* an advisory hack only... */
-               Perl_warner(aTHX_ WARN_SYNTAX,
+               Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
                            "%s (...) interpreted as function",name);
        }
     }
@@ -5814,14 +6057,14 @@ S_new_constant(pTHX_ char *s, STRLEN len, const char *key, SV *sv, SV *pv,
     SV **cvp;
     SV *cv, *typesv;
     const char *why1, *why2, *why3;
-    
+
     if (!table || !(PL_hints & HINT_LOCALIZE_HH)) {
        SV *msg;
        
        why2 = strEQ(key,"charnames")
               ? "(possibly a missing \"use charnames ...\")"
               : "";
-       msg = Perl_newSVpvf(aTHX_ "Constant(%s) unknown: %s", 
+       msg = Perl_newSVpvf(aTHX_ "Constant(%s) unknown: %s",
                            (type ? type: "undef"), why2);
 
        /* This is convoluted and evil ("goto considered harmful")
@@ -5832,7 +6075,7 @@ S_new_constant(pTHX_ char *s, STRLEN len, const char *key, SV *sv, SV *pv,
        goto msgdone;
 
     report:
-       msg = Perl_newSVpvf(aTHX_ "Constant(%s): %s%s%s", 
+       msg = Perl_newSVpvf(aTHX_ "Constant(%s): %s%s%s",
                            (type ? type: "undef"), why1, why2, why3);
     msgdone:
        yyerror(SvPVX(msg));
@@ -5854,11 +6097,11 @@ S_new_constant(pTHX_ char *s, STRLEN len, const char *key, SV *sv, SV *pv,
        typesv = sv_2mortal(newSVpv(type, 0));
     else
        typesv = &PL_sv_undef;
-    
+
     PUSHSTACKi(PERLSI_OVERLOAD);
     ENTER ;
     SAVETMPS;
-    
+
     PUSHMARK(SP) ;
     EXTEND(sp, 3);
     if (pv)
@@ -5868,9 +6111,9 @@ S_new_constant(pTHX_ char *s, STRLEN len, const char *key, SV *sv, SV *pv,
        PUSHs(typesv);
     PUTBACK;
     call_sv(cv, G_SCALAR | ( PL_in_eval ? 0 : G_EVAL));
-    
+
     SPAGAIN ;
-    
+
     /* Check the eval first */
     if (!PL_in_eval && SvTRUE(ERRSV)) {
        STRLEN n_a;
@@ -5883,12 +6126,12 @@ S_new_constant(pTHX_ char *s, STRLEN len, const char *key, SV *sv, SV *pv,
        res = POPs;
        (void)SvREFCNT_inc(res);
     }
-    
+
     PUTBACK ;
     FREETMPS ;
     LEAVE ;
     POPSTACK;
-    
+
     if (!SvOK(res)) {
        why1 = "Call to &{$^H{";
        why2 = key;
@@ -5899,7 +6142,7 @@ S_new_constant(pTHX_ char *s, STRLEN len, const char *key, SV *sv, SV *pv,
 
     return res;
 }
-  
+
 STATIC char *
 S_scan_word(pTHX_ register char *s, char *dest, STRLEN destlen, int allow_package, STRLEN *slp)
 {
@@ -5919,9 +6162,9 @@ S_scan_word(pTHX_ register char *s, char *dest, STRLEN destlen, int allow_packag
            *d++ = *s++;
            *d++ = *s++;
        }
-       else if (UTF && *(U8*)s >= 0xc0 && isALNUM_utf8((U8*)s)) {
+       else if (UTF && UTF8_IS_START(*s) && isALNUM_utf8((U8*)s)) {
            char *t = s + UTF8SKIP(s);
-           while (*t & 0x80 && is_utf8_mark((U8*)t))
+           while (UTF8_IS_CONTINUED(*t) && is_utf8_mark((U8*)t))
                t += UTF8SKIP(t);
            if (d + (t - s) > e)
                Perl_croak(aTHX_ ident_too_long);
@@ -5971,9 +6214,9 @@ S_scan_ident(pTHX_ register char *s, register char *send, char *dest, STRLEN des
                *d++ = *s++;
                *d++ = *s++;
            }
-           else if (UTF && *(U8*)s >= 0xc0 && isALNUM_utf8((U8*)s)) {
+           else if (UTF && UTF8_IS_START(*s) && isALNUM_utf8((U8*)s)) {
                char *t = s + UTF8SKIP(s);
-               while (*t & 0x80 && is_utf8_mark((U8*)t))
+               while (UTF8_IS_CONTINUED(*t) && is_utf8_mark((U8*)t))
                    t += UTF8SKIP(t);
                if (d + (t - s) > e)
                    Perl_croak(aTHX_ ident_too_long);
@@ -6026,7 +6269,7 @@ S_scan_ident(pTHX_ register char *s, register char *send, char *dest, STRLEN des
                e = s;
                while ((e < send && isALNUM_lazy_if(e,UTF)) || *e == ':') {
                    e += UTF8SKIP(e);
-                   while (e < send && *e & 0x80 && is_utf8_mark((U8*)e))
+                   while (e < send && UTF8_IS_CONTINUED(*e) && is_utf8_mark((U8*)e))
                        e += UTF8SKIP(e);
                }
                Copy(s, d, e - s, char);
@@ -6042,10 +6285,9 @@ S_scan_ident(pTHX_ register char *s, register char *send, char *dest, STRLEN des
            *d = '\0';
            while (s < send && SPACE_OR_TAB(*s)) s++;
            if ((*s == '[' || (*s == '{' && strNE(dest, "sub")))) {
-               dTHR;                   /* only for ckWARN */
                if (ckWARN(WARN_AMBIGUOUS) && keyword(dest, d - dest)) {
                    const char *brack = *s == '[' ? "[...]" : "{...}";
-                   Perl_warner(aTHX_ WARN_AMBIGUOUS,
+                   Perl_warner(aTHX_ packWARN(WARN_AMBIGUOUS),
                        "Ambiguous use of %c{%s%s} resolved to %c%s%s",
                        funny, dest, brack, funny, dest, brack);
                }
@@ -6053,8 +6295,8 @@ S_scan_ident(pTHX_ register char *s, register char *send, char *dest, STRLEN des
                PL_lex_brackstack[PL_lex_brackets++] = (char)(XOPERATOR | XFAKEBRACK);
                return s;
            }
-       } 
-       /* Handle extended ${^Foo} variables 
+       }
+       /* Handle extended ${^Foo} variables
         * 1999-02-27 mjd-perl-patch@plover.com */
        else if (!isALNUM(*d) && !isPRINT(*d) /* isCTRL(d) */
                 && isALNUM(*s))
@@ -6069,16 +6311,17 @@ S_scan_ident(pTHX_ register char *s, register char *send, char *dest, STRLEN des
        }
        if (*s == '}') {
            s++;
-           if (PL_lex_state == LEX_INTERPNORMAL && !PL_lex_brackets)
+           if (PL_lex_state == LEX_INTERPNORMAL && !PL_lex_brackets) {
                PL_lex_state = LEX_INTERPEND;
+               PL_expect = XREF;
+           }
            if (funny == '#')
                funny = '@';
            if (PL_lex_state == LEX_NORMAL) {
-               dTHR;                   /* only for ckWARN */
                if (ckWARN(WARN_AMBIGUOUS) &&
                    (keyword(dest, d - dest) || get_cv(dest, FALSE)))
                {
-                   Perl_warner(aTHX_ WARN_AMBIGUOUS,
+                   Perl_warner(aTHX_ packWARN(WARN_AMBIGUOUS),
                        "Ambiguous use of %c{%s} resolved to %c%s",
                        funny, dest, funny, dest);
                }
@@ -6095,7 +6338,7 @@ S_scan_ident(pTHX_ register char *s, register char *send, char *dest, STRLEN des
 }
 
 void
-Perl_pmflag(pTHX_ U16 *pmfl, int ch)
+Perl_pmflag(pTHX_ U32* pmfl, int ch)
 {
     if (ch == 'i')
        *pmfl |= PMf_FOLD;
@@ -6120,12 +6363,8 @@ S_scan_pat(pTHX_ char *start, I32 type)
     char *s;
 
     s = scan_str(start,FALSE,FALSE);
-    if (!s) {
-       if (PL_lex_stuff)
-           SvREFCNT_dec(PL_lex_stuff);
-       PL_lex_stuff = Nullsv;
+    if (!s)
        Perl_croak(aTHX_ "Search pattern not terminated");
-    }
 
     pm = (PMOP*)newPMOP(type, 0);
     if (PL_multi_open == '?')
@@ -6138,6 +6377,13 @@ S_scan_pat(pTHX_ char *start, I32 type)
        while (*s && strchr("iogcmsx", *s))
            pmflag(&pm->op_pmflags,*s++);
     }
+    /* issue a warning if /c is specified,but /g is not */
+    if (ckWARN(WARN_REGEXP) && 
+        (pm->op_pmflags & PMf_CONTINUE) && !(pm->op_pmflags & PMf_GLOBAL))
+    {
+        Perl_warner(aTHX_ packWARN(WARN_REGEXP), c_without_g);
+    }
+
     pm->op_pmpermflags = pm->op_pmflags;
 
     PL_lex_op = (OP*)pm;
@@ -6157,12 +6403,8 @@ S_scan_subst(pTHX_ char *start)
 
     s = scan_str(start,FALSE,FALSE);
 
-    if (!s) {
-       if (PL_lex_stuff)
-           SvREFCNT_dec(PL_lex_stuff);
-       PL_lex_stuff = Nullsv;
+    if (!s)
        Perl_croak(aTHX_ "Substitution pattern not terminated");
-    }
 
     if (s[-1] == PL_multi_open)
        s--;
@@ -6170,12 +6412,10 @@ S_scan_subst(pTHX_ char *start)
     first_start = PL_multi_start;
     s = scan_str(s,FALSE,FALSE);
     if (!s) {
-       if (PL_lex_stuff)
+       if (PL_lex_stuff) {
            SvREFCNT_dec(PL_lex_stuff);
-       PL_lex_stuff = Nullsv;
-       if (PL_lex_repl)
-           SvREFCNT_dec(PL_lex_repl);
-       PL_lex_repl = Nullsv;
+           PL_lex_stuff = Nullsv;
+       }
        Perl_croak(aTHX_ "Substitution replacement not terminated");
     }
     PL_multi_start = first_start;      /* so whole substitution is taken together */
@@ -6192,6 +6432,12 @@ S_scan_subst(pTHX_ char *start)
            break;
     }
 
+    /* /c is not meaningful with s/// */
+    if (ckWARN(WARN_REGEXP) && (pm->op_pmflags & PMf_CONTINUE))
+    {
+        Perl_warner(aTHX_ packWARN(WARN_REGEXP), c_in_subst);
+    }
+
     if (es) {
        SV *repl;
        PL_sublex_info.super_bufptr = s;
@@ -6224,35 +6470,24 @@ S_scan_trans(pTHX_ char *start)
     I32 squash;
     I32 del;
     I32 complement;
-    I32 utf8;
-    I32 count = 0;
 
     yylval.ival = OP_NULL;
 
     s = scan_str(start,FALSE,FALSE);
-    if (!s) {
-       if (PL_lex_stuff)
-           SvREFCNT_dec(PL_lex_stuff);
-       PL_lex_stuff = Nullsv;
+    if (!s)
        Perl_croak(aTHX_ "Transliteration pattern not terminated");
-    }
     if (s[-1] == PL_multi_open)
        s--;
 
     s = scan_str(s,FALSE,FALSE);
     if (!s) {
-       if (PL_lex_stuff)
+       if (PL_lex_stuff) {
            SvREFCNT_dec(PL_lex_stuff);
-       PL_lex_stuff = Nullsv;
-       if (PL_lex_repl)
-           SvREFCNT_dec(PL_lex_repl);
-       PL_lex_repl = Nullsv;
+           PL_lex_stuff = Nullsv;
+       }
        Perl_croak(aTHX_ "Transliteration replacement not terminated");
     }
 
-    New(803,tbl,256,short);
-    o = newPVOP(OP_TRANS, 0, (char*)tbl);
-
     complement = del = squash = 0;
     while (strchr("cds", *s)) {
        if (*s == 'c')
@@ -6263,7 +6498,12 @@ S_scan_trans(pTHX_ char *start)
            squash = OPpTRANS_SQUASH;
        s++;
     }
-    o->op_private = del|squash|complement;
+
+    New(803, tbl, complement&&!del?258:256, short);
+    o = newPVOP(OP_TRANS, 0, (char*)tbl);
+    o->op_private = del|squash|complement|
+      (DO_UTF8(PL_lex_stuff)? OPpTRANS_FROM_UTF : 0)|
+      (DO_UTF8(PL_lex_repl) ? OPpTRANS_TO_UTF   : 0);
 
     PL_lex_op = o;
     yylval.ival = OP_TRANS;
@@ -6273,7 +6513,6 @@ S_scan_trans(pTHX_ char *start)
 STATIC char *
 S_scan_heredoc(pTHX_ register char *s)
 {
-    dTHR;
     SV *herewas;
     I32 op_type = OP_SCALAR;
     I32 len;
@@ -6304,7 +6543,7 @@ S_scan_heredoc(pTHX_ register char *s)
        else
            term = '"';
        if (!isALNUM_lazy_if(s,UTF))
-           deprecate("bare << to mean <<\"\"");
+           deprecate_old("bare << to mean <<\"\"");
        for (; isALNUM_lazy_if(s,UTF); s++) {
            if (d < e)
                *d++ = *s;
@@ -6375,7 +6614,7 @@ S_scan_heredoc(pTHX_ register char *s)
                CopLINE_inc(PL_curcop);
        }
        if (s >= bufend) {
-           CopLINE_set(PL_curcop, PL_multi_start);
+           CopLINE_set(PL_curcop, (line_t)PL_multi_start);
            missingterm(PL_tokenbuf);
        }
        sv_setpvn(herewas,bufptr,d-bufptr+1);
@@ -6395,7 +6634,7 @@ S_scan_heredoc(pTHX_ register char *s)
                CopLINE_inc(PL_curcop);
        }
        if (s >= PL_bufend) {
-           CopLINE_set(PL_curcop, PL_multi_start);
+           CopLINE_set(PL_curcop, (line_t)PL_multi_start);
            missingterm(PL_tokenbuf);
        }
        sv_setpvn(tmpstr,d+1,s-d);
@@ -6406,17 +6645,19 @@ S_scan_heredoc(pTHX_ register char *s)
        sv_setsv(PL_linestr,herewas);
        PL_oldoldbufptr = PL_oldbufptr = PL_bufptr = s = PL_linestart = SvPVX(PL_linestr);
        PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
+       PL_last_lop = PL_last_uni = Nullch;
     }
     else
        sv_setpvn(tmpstr,"",0);   /* avoid "uninitialized" warning */
     while (s >= PL_bufend) {   /* multiple line string? */
        if (!outer ||
         !(PL_oldoldbufptr = PL_oldbufptr = s = PL_linestart = filter_gets(PL_linestr, PL_rsfp, 0))) {
-           CopLINE_set(PL_curcop, PL_multi_start);
+           CopLINE_set(PL_curcop, (line_t)PL_multi_start);
            missingterm(PL_tokenbuf);
        }
        CopLINE_inc(PL_curcop);
        PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
+       PL_last_lop = PL_last_uni = Nullch;
 #ifndef PERL_STRICT_CR
        if (PL_bufend - PL_linestart >= 2) {
            if ((PL_bufend[-2] == '\r' && PL_bufend[-1] == '\n') ||
@@ -6437,6 +6678,8 @@ S_scan_heredoc(pTHX_ register char *s)
 
            sv_upgrade(sv, SVt_PVMG);
            sv_setsv(sv,PL_linestr);
+            (void)SvIOK_on(sv);
+            SvIVX(sv) = 0;
            av_store(CopFILEAV(PL_curcop), (I32)CopLINE(PL_curcop),sv);
        }
        if (*s == term && memEQ(s,PL_tokenbuf,len)) {
@@ -6458,6 +6701,12 @@ retval:
        Renew(SvPVX(tmpstr), SvLEN(tmpstr), char);
     }
     SvREFCNT_dec(herewas);
+    if (!IN_BYTES) {
+       if (UTF && is_utf8_string((U8*)SvPVX(tmpstr), SvCUR(tmpstr)))
+           SvUTF8_on(tmpstr);
+       else if (PL_encoding)
+           sv_recode_to_utf8(tmpstr, PL_encoding);
+    }
     PL_lex_stuff = tmpstr;
     yylval.ival = op_type;
     return s;
@@ -6534,6 +6783,9 @@ S_scan_inputsymbol(pTHX_ char *start)
        return s;
     }
     else {
+       bool readline_overriden = FALSE;
+       GV *gv_readline = Nullgv;
+       GV **gvp;
        /* we're in a filehandle read situation */
        d = PL_tokenbuf;
 
@@ -6541,6 +6793,15 @@ S_scan_inputsymbol(pTHX_ char *start)
        if (!len)
            (void)strcpy(d,"ARGV");
 
+       /* Check whether readline() is overriden */
+       if (((gv_readline = gv_fetchpv("readline", FALSE, SVt_PVCV))
+               && GvCVu(gv_readline) && GvIMPORTED_CV(gv_readline))
+               ||
+               ((gvp = (GV**)hv_fetch(PL_globalstash, "readline", 8, FALSE))
+               && (gv_readline = *gvp) != (GV*)&PL_sv_undef
+               && GvCVu(gv_readline) && GvIMPORTED_CV(gv_readline)))
+           readline_overriden = TRUE;
+
        /* if <$fh>, create the ops to turn the variable into a
           filehandle
        */
@@ -6551,17 +6812,44 @@ S_scan_inputsymbol(pTHX_ char *start)
               add symbol table ops
            */
            if ((tmp = pad_findmy(d)) != NOT_IN_PAD) {
-               OP *o = newOP(OP_PADSV, 0);
-               o->op_targ = tmp;
-               PL_lex_op = (OP*)newUNOP(OP_READLINE, 0, o);
+               if (PAD_COMPNAME_FLAGS(tmp) & SVpad_OUR) {
+                   SV *sym = sv_2mortal(
+                           newSVpv(HvNAME(PAD_COMPNAME_OURSTASH(tmp)),0));
+                   sv_catpvn(sym, "::", 2);
+                   sv_catpv(sym, d+1);
+                   d = SvPVX(sym);
+                   goto intro_sym;
+               }
+               else {
+                   OP *o = newOP(OP_PADSV, 0);
+                   o->op_targ = tmp;
+                   PL_lex_op = readline_overriden
+                       ? (OP*)newUNOP(OP_ENTERSUB, OPf_STACKED,
+                               append_elem(OP_LIST, o,
+                                   newCVREF(0, newGVOP(OP_GV,0,gv_readline))))
+                       : (OP*)newUNOP(OP_READLINE, 0, o);
+               }
            }
            else {
-               GV *gv = gv_fetchpv(d+1,TRUE, SVt_PV);
-               PL_lex_op = (OP*)newUNOP(OP_READLINE, 0,
-                                           newUNOP(OP_RV2SV, 0,
-                                               newGVOP(OP_GV, 0, gv)));
-           }
-           PL_lex_op->op_flags |= OPf_SPECIAL;
+               GV *gv;
+               ++d;
+intro_sym:
+               gv = gv_fetchpv(d,
+                               (PL_in_eval
+                                ? (GV_ADDMULTI | GV_ADDINEVAL)
+                                : GV_ADDMULTI),
+                               SVt_PV);
+               PL_lex_op = readline_overriden
+                   ? (OP*)newUNOP(OP_ENTERSUB, OPf_STACKED,
+                           append_elem(OP_LIST,
+                               newUNOP(OP_RV2SV, 0, newGVOP(OP_GV, 0, gv)),
+                               newCVREF(0, newGVOP(OP_GV, 0, gv_readline))))
+                   : (OP*)newUNOP(OP_READLINE, 0,
+                           newUNOP(OP_RV2SV, 0,
+                               newGVOP(OP_GV, 0, gv)));
+           }
+           if (!readline_overriden)
+               PL_lex_op->op_flags |= OPf_SPECIAL;
            /* we created the ops in PL_lex_op, so make yylval.ival a null op */
            yylval.ival = OP_NULL;
        }
@@ -6570,7 +6858,12 @@ S_scan_inputsymbol(pTHX_ char *start)
           (<Foo::BAR> or <FOO>) so build a simple readline OP */
        else {
            GV *gv = gv_fetchpv(d,TRUE, SVt_PVIO);
-           PL_lex_op = (OP*)newUNOP(OP_READLINE, 0, newGVOP(OP_GV, 0, gv));
+           PL_lex_op = readline_overriden
+               ? (OP*)newUNOP(OP_ENTERSUB, OPf_STACKED,
+                       append_elem(OP_LIST,
+                           newGVOP(OP_GV, 0, gv),
+                           newCVREF(0, newGVOP(OP_GV, 0, gv_readline))))
+               : (OP*)newUNOP(OP_READLINE, 0, newGVOP(OP_GV, 0, gv));
            yylval.ival = OP_NULL;
        }
     }
@@ -6608,24 +6901,23 @@ S_scan_inputsymbol(pTHX_ char *start)
    calls scan_str().  s/// makes yylex() call scan_subst() which calls
    scan_str().  tr/// and y/// make yylex() call scan_trans() which
    calls scan_str().
-      
+
    It skips whitespace before the string starts, and treats the first
    character as the delimiter.  If the delimiter is one of ([{< then
    the corresponding "close" character )]}> is used as the closing
    delimiter.  It allows quoting of delimiters, and if the string has
    balanced delimiters ([{<>}]) it allows nesting.
 
-   The lexer always reads these strings into lex_stuff, except in the
-   case of the operators which take *two* arguments (s/// and tr///)
-   when it checks to see if lex_stuff is full (presumably with the 1st
-   arg to s or tr) and if so puts the string into lex_repl.
-
+   On success, the SV with the resulting string is put into lex_stuff or,
+   if that is already non-NULL, into lex_repl. The second case occurs only
+   when parsing the RHS of the special constructs s/// and tr/// (y///).
+   For convenience, the terminating delimiter character is stuffed into
+   SvIVX of the SV.
 */
 
 STATIC char *
 S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
 {
-    dTHR;
     SV *sv;                            /* scalar value: string */
     char *tmps;                                /* temp string, used for delimiter matching */
     register char *s = start;          /* current position in the buffer */
@@ -6633,6 +6925,10 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
     register char *to;                 /* current position in the sv's data */
     I32 brackets = 1;                  /* bracket nesting level */
     bool has_utf8 = FALSE;             /* is there any utf8 content? */
+    I32 termcode;                      /* terminating char. code */
+    U8 termstr[UTF8_MAXLEN];           /* terminating string */
+    STRLEN termlen;                    /* length of terminating string */
+    char *last = NULL;                 /* last position for nesting bracket */
 
     /* skip space before the delimiter */
     if (isSPACE(*s))
@@ -6643,8 +6939,16 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
 
     /* after skipping whitespace, the next character is the terminator */
     term = *s;
-    if ((term & 0x80) && UTF)
-       has_utf8 = TRUE;
+    if (!UTF) {
+       termcode = termstr[0] = term;
+       termlen = 1;
+    }
+    else {
+       termcode = utf8_to_uvchr((U8*)s, &termlen);
+       Copy(s, termstr, termlen, U8);
+       if (!UTF8_IS_INVARIANT(term))
+           has_utf8 = TRUE;
+    }
 
     /* mark where we are */
     PL_multi_start = CopLINE(PL_curcop);
@@ -6652,21 +6956,92 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
 
     /* find corresponding closing delimiter */
     if (term && (tmps = strchr("([{< )]}> )]}>",term)))
-       term = tmps[5];
+       termcode = termstr[0] = term = tmps[5];
+
     PL_multi_close = term;
 
     /* create a new SV to hold the contents.  87 is leak category, I'm
        assuming.  79 is the SV's initial length.  What a random number. */
     sv = NEWSV(87,79);
     sv_upgrade(sv, SVt_PVIV);
-    SvIVX(sv) = term;
+    SvIVX(sv) = termcode;
     (void)SvPOK_only(sv);              /* validate pointer */
 
     /* move past delimiter and try to read a complete string */
     if (keep_delims)
-       sv_catpvn(sv, s, 1);
-    s++;
+       sv_catpvn(sv, s, termlen);
+    s += termlen;
     for (;;) {
+       if (PL_encoding && !UTF) {
+           bool cont = TRUE;
+
+           while (cont) {
+               int offset = s - SvPVX(PL_linestr);
+               bool found = sv_cat_decode(sv, PL_encoding, PL_linestr,
+                                          &offset, (char*)termstr, termlen);
+               char *ns = SvPVX(PL_linestr) + offset;
+               char *svlast = SvEND(sv) - 1;
+
+               for (; s < ns; s++) {
+                   if (*s == '\n' && !PL_rsfp)
+                       CopLINE_inc(PL_curcop);
+               }
+               if (!found)
+                   goto read_more_line;
+               else {
+                   /* handle quoted delimiters */
+                   if (*(svlast-1) == '\\') {
+                       char *t;
+                       for (t = svlast-2; t >= SvPVX(sv) && *t == '\\';)
+                           t--;
+                       if ((svlast-1 - t) % 2) {
+                           if (!keep_quoted) {
+                               *(svlast-1) = term;
+                               *svlast = '\0';
+                               SvCUR_set(sv, SvCUR(sv) - 1);
+                           }
+                           continue;
+                       }
+                   }
+                   if (PL_multi_open == PL_multi_close) {
+                       cont = FALSE;
+                   }
+                   else {
+                       char *t, *w;
+                       if (!last)
+                           last = SvPVX(sv);
+                       for (w = t = last; t < svlast; w++, t++) {
+                           /* At here, all closes are "was quoted" one,
+                              so we don't check PL_multi_close. */
+                           if (*t == '\\') {
+                               if (!keep_quoted && *(t+1) == PL_multi_open)
+                                   t++;
+                               else
+                                   *w++ = *t++;
+                           }
+                           else if (*t == PL_multi_open)
+                               brackets++;
+
+                           *w = *t;
+                       }
+                       if (w < t) {
+                           *w++ = term;
+                           *w = '\0';
+                           SvCUR_set(sv, w - SvPVX(sv));
+                       }
+                       last = w;
+                       if (--brackets <= 0)
+                           cont = FALSE;
+                   }
+               }
+           }
+           if (!keep_delims) {
+               SvCUR_set(sv, SvCUR(sv) - 1);
+               *SvEND(sv) = '\0';
+           }
+           break;
+       }
+
        /* extend sv if need be */
        SvGROW(sv, SvCUR(sv) + (PL_bufend - s) + 1);
        /* set 'to' to the next character in the sv's string */
@@ -6688,9 +7063,13 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
                }
                /* terminate when run out of buffer (the for() condition), or
                   have found the terminator */
-               else if (*s == term)
-                   break;
-               else if (!has_utf8 && (*s & 0x80) && UTF)
+               else if (*s == term) {
+                   if (termlen == 1)
+                       break;
+                   if (s+termlen <= PL_bufend && memEQ(s, (char*)termstr, termlen))
+                       break;
+               }
+               else if (!has_utf8 && !UTF8_IS_INVARIANT((U8)*s) && UTF)
                    has_utf8 = TRUE;
                *to = *s;
            }
@@ -6719,7 +7098,7 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
                    break;
                else if (*s == PL_multi_open)
                    brackets++;
-               else if (!has_utf8 && (*s & 0x80) && UTF)
+               else if (!has_utf8 && !UTF8_IS_INVARIANT((U8)*s) && UTF)
                    has_utf8 = TRUE;
                *to = *s;
            }
@@ -6751,13 +7130,14 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
            to[-1] = '\n';
 #endif
        
+     read_more_line:
        /* if we're out of file, or a read fails, bail and reset the current
           line marker so we can report where the unterminated string began
        */
        if (!PL_rsfp ||
         !(PL_oldoldbufptr = PL_oldbufptr = s = PL_linestart = filter_gets(PL_linestr, PL_rsfp, 0))) {
            sv_free(sv);
-           CopLINE_set(PL_curcop, PL_multi_start);
+           CopLINE_set(PL_curcop, (line_t)PL_multi_start);
            return Nullch;
        }
        /* we read a line, so increment our line counter */
@@ -6769,21 +7149,27 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
 
            sv_upgrade(sv, SVt_PVMG);
            sv_setsv(sv,PL_linestr);
+            (void)SvIOK_on(sv);
+            SvIVX(sv) = 0;
            av_store(CopFILEAV(PL_curcop), (I32)CopLINE(PL_curcop), sv);
        }
 
        /* having changed the buffer, we must update PL_bufend */
        PL_bufend = SvPVX(PL_linestr) + SvCUR(PL_linestr);
+       PL_last_lop = PL_last_uni = Nullch;
     }
-    
+
     /* at this point, we have successfully read the delimited string */
 
-    if (keep_delims)
-       sv_catpvn(sv, s, 1);
-    if (has_utf8)
+    if (!PL_encoding || UTF) {
+       if (keep_delims)
+           sv_catpvn(sv, s, termlen);
+       s += termlen;
+    }
+    if (has_utf8 || PL_encoding)
        SvUTF8_on(sv);
+
     PL_multi_end = CopLINE(PL_curcop);
-    s++;
 
     /* if we allocated too much space, give some back */
     if (SvCUR(sv) + 5 < SvLEN(sv)) {
@@ -6794,7 +7180,7 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
     /* decide whether this is the first or second quoted string we've read
        for this op
     */
-    
+
     if (PL_lex_stuff)
        PL_lex_repl = sv;
     else
@@ -6810,11 +7196,11 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
 
   Read a number in any of the formats that Perl accepts:
 
-  0(x[0-7A-F]+)|([0-7]+)|(b[01])
-  [\d_]+(\.[\d_]*)?[Ee](\d+)
-
-  Underbars (_) are allowed in decimal numbers.  If -w is on,
-  underbars before a decimal point must be at three digit intervals.
+  \d(_?\d)*(\.(\d(_?\d)*)?)?[Ee][\+\-]?(\d(_?\d)*)     12 12.34 12.
+  \.\d(_?\d)*[Ee][\+\-]?(\d(_?\d)*)                    .34
+  0b[01](_?[01])*
+  0[0-7](_?[0-7])*
+  0x[0-9A-Fa-f](_?[0-9A-Fa-f])*
 
   Like most scan_ routines, it uses the PL_tokenbuf buffer to hold the
   thing it reads.
@@ -6823,7 +7209,7 @@ S_scan_str(pTHX_ char *start, int keep_quoted, int keep_delims)
   try converting the number to an integer and see if it can do so
   without loss of precision.
 */
-  
+
 char *
 Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
 {
@@ -6841,7 +7227,7 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
     switch (*s) {
     default:
       Perl_croak(aTHX_ "panic: scan_num");
-      
+
     /* if it starts with a 0, it could be an octal number, a decimal in
        0.13 disguise, or a hexadecimal number, or a binary number. */
     case '0':
@@ -6856,7 +7242,6 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
             we in octal/hex/binary?" indicator to disallow hex characters
             when in octal mode.
           */
-           dTHR;
            NV n = 0.0;
            UV u = 0;
            I32 shift;
@@ -6885,8 +7270,17 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
            else if (s[1] == '.' || s[1] == 'e' || s[1] == 'E')
                goto decimal;
            /* so it must be octal */
-           else
+           else {
                shift = 3;
+               s++;
+           }
+
+           if (*s == '_') {
+              if (ckWARN(WARN_SYNTAX))
+                  Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
+                              "Misplaced _ in number");
+              lastub = s++;
+           }
 
            base = bases[shift];
            Base = Bases[shift];
@@ -6904,9 +7298,12 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
                default:
                    goto out;
 
-               /* _ are ignored */
+               /* _ are ignored -- but warned about if consecutive */
                case '_':
-                   s++;
+                   if (ckWARN(WARN_SYNTAX) && lastub && s == lastub + 1)
+                       Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
+                                   "Misplaced _ in number");
+                   lastub = s++;
                    break;
 
                /* 8 and 9 are not octal */
@@ -6944,11 +7341,10 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
 
                        if ((x >> shift) != u
                            && !(PL_hints & HINT_NEW_BINARY)) {
-                           dTHR;
                            overflowed = TRUE;
                            n = (NV) u;
                            if (ckWARN_d(WARN_OVERFLOW))
-                               Perl_warner(aTHX_ WARN_OVERFLOW,
+                               Perl_warner(aTHX_ packWARN(WARN_OVERFLOW),
                                            "Integer overflow in %s number",
                                            base);
                        } else
@@ -6974,20 +7370,25 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
             the number.
          */
          out:
+
+           /* final misplaced underbar check */
+           if (s[-1] == '_') {
+               if (ckWARN(WARN_SYNTAX))
+                   Perl_warner(aTHX_ packWARN(WARN_SYNTAX), "Misplaced _ in number");
+           }
+
            sv = NEWSV(92,0);
            if (overflowed) {
-               dTHR;
                if (ckWARN(WARN_PORTABLE) && n > 4294967295.0)
-                   Perl_warner(aTHX_ WARN_PORTABLE,
+                   Perl_warner(aTHX_ packWARN(WARN_PORTABLE),
                                "%s number > %s non-portable",
                                Base, max);
                sv_setnv(sv, n);
            }
            else {
 #if UVSIZE > 4
-               dTHR;
                if (ckWARN(WARN_PORTABLE) && u > 0xffffffff)
-                   Perl_warner(aTHX_ WARN_PORTABLE,
+                   Perl_warner(aTHX_ packWARN(WARN_PORTABLE),
                                "%s number > %s non-portable",
                                Base, max);
 #endif
@@ -7011,14 +7412,14 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
 
        /* read next group of digits and _ and copy into d */
        while (isDIGIT(*s) || *s == '_') {
-           /* skip underscores, checking for misplaced ones 
+           /* skip underscores, checking for misplaced ones
               if -w is on
            */
            if (*s == '_') {
-               dTHR;                   /* only for ckWARN */
-               if (ckWARN(WARN_SYNTAX) && lastub && s - lastub != 3)
-                   Perl_warner(aTHX_ WARN_SYNTAX, "Misplaced _ in number");
-               lastub = ++s;
+               if (ckWARN(WARN_SYNTAX) && lastub && s == lastub + 1)
+                   Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
+                               "Misplaced _ in number");
+               lastub = s++;
            }
            else {
                /* check for end of fixed-length buffer */
@@ -7030,10 +7431,9 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
        }
 
        /* final misplaced underbar check */
-       if (lastub && s - lastub != 3) {
-           dTHR;
+       if (lastub && s == lastub + 1) {
            if (ckWARN(WARN_SYNTAX))
-               Perl_warner(aTHX_ WARN_SYNTAX, "Misplaced _ in number");
+               Perl_warner(aTHX_ packWARN(WARN_SYNTAX), "Misplaced _ in number");
        }
 
        /* read a decimal portion if there is one.  avoid
@@ -7044,131 +7444,123 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
            floatit = TRUE;
            *d++ = *s++;
 
-           /* copy, ignoring underbars, until we run out of
-              digits.  Note: no misplaced underbar checks!
+           if (*s == '_') {
+               if (ckWARN(WARN_SYNTAX))
+                   Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
+                               "Misplaced _ in number");
+               lastub = s;
+           }
+
+           /* copy, ignoring underbars, until we run out of digits.
            */
            for (; isDIGIT(*s) || *s == '_'; s++) {
                /* fixed length buffer check */
                if (d >= e)
                    Perl_croak(aTHX_ number_too_long);
-               if (*s != '_')
+               if (*s == '_') {
+                  if (ckWARN(WARN_SYNTAX) && lastub && s == lastub + 1)
+                      Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
+                                  "Misplaced _ in number");
+                  lastub = s;
+               }
+               else
                    *d++ = *s;
            }
+           /* fractional part ending in underbar? */
+           if (s[-1] == '_') {
+               if (ckWARN(WARN_SYNTAX))
+                   Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
+                               "Misplaced _ in number");
+           }
            if (*s == '.' && isDIGIT(s[1])) {
                /* oops, it's really a v-string, but without the "v" */
-               s = start - 1;
+               s = start;
                goto vstring;
            }
        }
 
        /* read exponent part, if present */
-       if (*s && strchr("eE",*s) && strchr("+-0123456789",s[1])) {
+       if (*s && strchr("eE",*s) && strchr("+-0123456789_", s[1])) {
            floatit = TRUE;
            s++;
 
            /* regardless of whether user said 3E5 or 3e5, use lower 'e' */
            *d++ = 'e';         /* At least some Mach atof()s don't grok 'E' */
 
+           /* stray preinitial _ */
+           if (*s == '_') {
+               if (ckWARN(WARN_SYNTAX))
+                   Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
+                               "Misplaced _ in number");
+               lastub = s++;
+           }
+
            /* allow positive or negative exponent */
            if (*s == '+' || *s == '-')
                *d++ = *s++;
 
-           /* read digits of exponent (no underbars :-) */
-           while (isDIGIT(*s)) {
-               if (d >= e)
-                   Perl_croak(aTHX_ number_too_long);
-               *d++ = *s++;
+           /* stray initial _ */
+           if (*s == '_') {
+               if (ckWARN(WARN_SYNTAX))
+                   Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
+                               "Misplaced _ in number");
+               lastub = s++;
+           }
+
+           /* read digits of exponent */
+           while (isDIGIT(*s) || *s == '_') {
+               if (isDIGIT(*s)) {
+                   if (d >= e)
+                       Perl_croak(aTHX_ number_too_long);
+                   *d++ = *s++;
+               }
+               else {
+                  if (ckWARN(WARN_SYNTAX) &&
+                      ((lastub && s == lastub + 1) ||
+                       (!isDIGIT(s[1]) && s[1] != '_')))
+                      Perl_warner(aTHX_ packWARN(WARN_SYNTAX),
+                                  "Misplaced _ in number");
+                  lastub = s++;
+               }
            }
        }
 
-       /* terminate the string */
-       *d = '\0';
 
        /* make an sv from the string */
        sv = NEWSV(92,0);
 
-#if defined(Strtol) && defined(Strtoul)
-
        /*
-          strtol/strtoll sets errno to ERANGE if the number is too big
-          for an integer. We try to do an integer conversion first
-          if no characters indicating "float" have been found.
+           We try to do an integer conversion first if no characters
+           indicating "float" have been found.
         */
 
        if (!floatit) {
-           IV iv;
            UV uv;
-           errno = 0;
-           if (*PL_tokenbuf == '-')
-               iv = Strtol(PL_tokenbuf, (char**)NULL, 10);
-           else
-               uv = Strtoul(PL_tokenbuf, (char**)NULL, 10);
-           if (errno)
-               floatit = TRUE; /* Probably just too large. */
-           else if (*PL_tokenbuf == '-')
-               sv_setiv(sv, iv);
-           else if (uv <= IV_MAX)
+            int flags = grok_number (PL_tokenbuf, d - PL_tokenbuf, &uv);
+
+            if (flags == IS_NUMBER_IN_UV) {
+              if (uv <= IV_MAX)
                sv_setiv(sv, uv); /* Prefer IVs over UVs. */
-           else
+              else
                sv_setuv(sv, uv);
-       }
+            } else if (flags == (IS_NUMBER_IN_UV | IS_NUMBER_NEG)) {
+              if (uv <= (UV) IV_MIN)
+                sv_setiv(sv, -(IV)uv);
+              else
+               floatit = TRUE;
+            } else
+              floatit = TRUE;
+        }
        if (floatit) {
+           /* terminate the string */
+           *d = '\0';
            nv = Atof(PL_tokenbuf);
            sv_setnv(sv, nv);
        }
-#else
-       /*
-          No working strtou?ll?.
-
-          Unfortunately atol() doesn't do range checks (returning
-          LONG_MIN/LONG_MAX, and setting errno to ERANGE on overflows)
-          everywhere [1], so we cannot use use atol() (or atoll()).
-          If we could, they would be used, as Atol(), very much like
-          Strtol() and Strtoul() are used above.
-
-          [1] XXX Configure test needed to check for atol()
-                  (and atoll()) overflow behaviour XXX
-
-          --jhi
 
-          We need to do this the hard way.  */
-
-       nv = Atof(PL_tokenbuf);
-
-       /* See if we can make do with an integer value without loss of
-          precision.  We use U_V to cast to a UV, because some
-          compilers have issues.  Then we try casting it back and see
-          if it was the same [1].  We only do this if we know we
-          specifically read an integer.  If floatit is true, then we
-          don't need to do the conversion at all. 
-
-          [1] Note that this is lossy if our NVs cannot preserve our
-          UVs.  There are metaconfig defines NV_PRESERVES_UV (a boolean)
-          and NV_PRESERVES_UV_BITS (a number), but in general we really
-          do hope all such potentially lossy platforms have strtou?ll?
-          to do a lossless IV/UV conversion.
-
-          Maybe could do some tricks with DBL_DIG, LDBL_DIG and
-          DBL_MANT_DIG and LDBL_MANT_DIG (these are already available
-          as NV_DIG and NV_MANT_DIG)?
-          
-          --jhi
-          */
-       {
-           UV uv = U_V(nv);
-           if (!floatit && (NV)uv == nv) {
-               if (uv <= IV_MAX)
-                   sv_setiv(sv, uv); /* Prefer IVs over UVs. */
-               else
-                   sv_setuv(sv, uv);
-           }
-           else
-               sv_setnv(sv, nv);
-       }
-#endif
        if ( floatit ? (PL_hints & HINT_NEW_FLOAT) :
                       (PL_hints & HINT_NEW_INTEGER) )
-           sv = new_constant(PL_tokenbuf, d - PL_tokenbuf, 
+           sv = new_constant(PL_tokenbuf, d - PL_tokenbuf,
                              (floatit ? "float" : "integer"),
                              sv, Nullsv, NULL);
        break;
@@ -7176,62 +7568,8 @@ Perl_scan_num(pTHX_ char *start, YYSTYPE* lvalp)
     /* if it starts with a v, it could be a v-string */
     case 'v':
 vstring:
-       {
-           char *pos = s;
-           pos++;
-           while (isDIGIT(*pos) || *pos == '_')
-               pos++;
-           if (!isALPHA(*pos)) {
-               UV rev;
-               U8 tmpbuf[UTF8_MAXLEN];
-               U8 *tmpend;
-               bool utf8 = FALSE;
-               s++;                            /* get past 'v' */
-
-               sv = NEWSV(92,5);
-               sv_setpvn(sv, "", 0);
-
-               for (;;) {
-                   if (*s == '0' && isDIGIT(s[1]))
-                       yyerror("Octal number in vector unsupported");
-                   rev = 0;
-                   {
-                       /* this is atoi() that tolerates underscores */
-                       char *end = pos;
-                       UV mult = 1;
-                       while (--end >= s) {
-                           UV orev;
-                           if (*end == '_')
-                               continue;
-                           orev = rev;
-                           rev += (*end - '0') * mult;
-                           mult *= 10;
-                           if (orev > rev && ckWARN_d(WARN_OVERFLOW))
-                               Perl_warner(aTHX_ WARN_OVERFLOW,
-                                           "Integer overflow in decimal number");
-                       }
-                   }
-                   tmpend = uv_to_utf8(tmpbuf, rev);
-                   utf8 = utf8 || rev > 127;
-                   sv_catpvn(sv, (const char*)tmpbuf, tmpend - tmpbuf);
-                   if (*pos == '.' && isDIGIT(pos[1]))
-                       s = ++pos;
-                   else {
-                       s = pos;
-                       break;
-                   }
-                   while (isDIGIT(*pos) || *pos == '_')
-                       pos++;
-               }
-
-               SvPOK_on(sv);
-               SvREADONLY_on(sv);
-               if (utf8) {
-                   SvUTF8_on(sv);
-                   sv_utf8_downgrade(sv, TRUE);
-               }
-           }
-       }
+               sv = NEWSV(92,5); /* preallocate storage space */
+               s = scan_vstring(s,sv);
        break;
     }
 
@@ -7248,7 +7586,6 @@ vstring:
 STATIC char *
 S_scan_formline(pTHX_ register char *s)
 {
-    dTHR;
     register char *eol;
     register char *t;
     SV *stuff = newSVpvn("",0);
@@ -7281,21 +7618,26 @@ S_scan_formline(pTHX_ register char *s)
                if (*t == '@' || *t == '^')
                    needargs = TRUE;
            }
-           sv_catpvn(stuff, s, eol-s);
+           if (eol > s) {
+               sv_catpvn(stuff, s, eol-s);
 #ifndef PERL_STRICT_CR
-           if (eol-s > 1 && eol[-2] == '\r' && eol[-1] == '\n') {
-               char *end = SvPVX(stuff) + SvCUR(stuff);
-               end[-2] = '\n';
-               end[-1] = '\0';
-               SvCUR(stuff)--;
-           }
+               if (eol-s > 1 && eol[-2] == '\r' && eol[-1] == '\n') {
+                   char *end = SvPVX(stuff) + SvCUR(stuff);
+                   end[-2] = '\n';
+                   end[-1] = '\0';
+                   SvCUR(stuff)--;
+               }
 #endif
+           }
+           else
+             break;
        }
        s = eol;
        if (PL_rsfp) {
            s = filter_gets(PL_linestr, PL_rsfp, 0);
            PL_oldoldbufptr = PL_oldbufptr = PL_bufptr = PL_linestart = SvPVX(PL_linestr);
            PL_bufend = PL_bufptr + SvCUR(PL_linestr);
+           PL_last_lop = PL_last_uni = Nullch;
            if (!s) {
                s = PL_bufptr;
                yyerror("Format not terminated");
@@ -7314,6 +7656,12 @@ S_scan_formline(pTHX_ register char *s)
        }
        else
            PL_lex_state = LEX_FORMLINE;
+       if (!IN_BYTES) {
+           if (UTF && is_utf8_string((U8*)SvPVX(stuff), SvCUR(stuff)))
+               SvUTF8_on(stuff);
+           else if (PL_encoding)
+               sv_recode_to_utf8(stuff, PL_encoding);
+       }
        PL_nextval[PL_nexttoke].opval = (OP*)newSVOP(OP_CONST, 0, stuff);
        force_next(THING);
        PL_nextval[PL_nexttoke].ival = OP_FORMLINE;
@@ -7339,63 +7687,34 @@ S_set_csh(pTHX)
 I32
 Perl_start_subparse(pTHX_ I32 is_format, U32 flags)
 {
-    dTHR;
     I32 oldsavestack_ix = PL_savestack_ix;
     CV* outsidecv = PL_compcv;
-    AV* comppadlist;
 
     if (PL_compcv) {
        assert(SvTYPE(PL_compcv) == SVt_PVCV);
     }
     SAVEI32(PL_subline);
     save_item(PL_subname);
-    SAVEI32(PL_padix);
-    SAVECOMPPAD();
-    SAVESPTR(PL_comppad_name);
     SAVESPTR(PL_compcv);
-    SAVEI32(PL_comppad_name_fill);
-    SAVEI32(PL_min_intro_pending);
-    SAVEI32(PL_max_intro_pending);
-    SAVEI32(PL_pad_reset_pending);
 
     PL_compcv = (CV*)NEWSV(1104,0);
     sv_upgrade((SV *)PL_compcv, is_format ? SVt_PVFM : SVt_PVCV);
     CvFLAGS(PL_compcv) |= flags;
 
-    PL_comppad = newAV();
-    av_push(PL_comppad, Nullsv);
-    PL_curpad = AvARRAY(PL_comppad);
-    PL_comppad_name = newAV();
-    PL_comppad_name_fill = 0;
-    PL_min_intro_pending = 0;
-    PL_padix = 0;
     PL_subline = CopLINE(PL_curcop);
-#ifdef USE_THREADS
-    av_store(PL_comppad_name, 0, newSVpvn("@_", 2));
-    PL_curpad[0] = (SV*)newAV();
-    SvPADMY_on(PL_curpad[0]);  /* XXX Needed? */
-#endif /* USE_THREADS */
-
-    comppadlist = newAV();
-    AvREAL_off(comppadlist);
-    av_store(comppadlist, 0, (SV*)PL_comppad_name);
-    av_store(comppadlist, 1, (SV*)PL_comppad);
-
-    CvPADLIST(PL_compcv) = comppadlist;
+    CvPADLIST(PL_compcv) = pad_new(padnew_SAVE|padnew_SAVESUB);
     CvOUTSIDE(PL_compcv) = (CV*)SvREFCNT_inc(outsidecv);
-#ifdef USE_THREADS
-    CvOWNER(PL_compcv) = 0;
-    New(666, CvMUTEXP(PL_compcv), 1, perl_mutex);
-    MUTEX_INIT(CvMUTEXP(PL_compcv));
-#endif /* USE_THREADS */
+    CvOUTSIDE_SEQ(PL_compcv) = PL_cop_seqmax;
 
     return oldsavestack_ix;
 }
 
+#ifdef __SC__
+#pragma segment Perl_yylex
+#endif
 int
 Perl_yywarn(pTHX_ char *s)
 {
-    dTHR;
     PL_in_eval |= EVAL_WARNONLY;
     yyerror(s);
     PL_in_eval &= ~EVAL_WARNONLY;
@@ -7405,7 +7724,6 @@ Perl_yywarn(pTHX_ char *s)
 int
 Perl_yyerror(pTHX_ char *s)
 {
-    dTHR;
     char *where = NULL;
     char *context = NULL;
     int contlen = -1;
@@ -7415,15 +7733,33 @@ Perl_yyerror(pTHX_ char *s)
        where = "at EOF";
     else if (PL_bufptr > PL_oldoldbufptr && PL_bufptr - PL_oldoldbufptr < 200 &&
       PL_oldoldbufptr != PL_oldbufptr && PL_oldbufptr != PL_bufptr) {
+       /*
+               Only for NetWare:
+               The code below is removed for NetWare because it abends/crashes on NetWare
+               when the script has error such as not having the closing quotes like:
+                   if ($var eq "value)
+               Checking of white spaces is anyway done in NetWare code.
+       */
+#ifndef NETWARE
        while (isSPACE(*PL_oldoldbufptr))
            PL_oldoldbufptr++;
+#endif
        context = PL_oldoldbufptr;
        contlen = PL_bufptr - PL_oldoldbufptr;
     }
     else if (PL_bufptr > PL_oldbufptr && PL_bufptr - PL_oldbufptr < 200 &&
       PL_oldbufptr != PL_bufptr) {
+       /*
+               Only for NetWare:
+               The code below is removed for NetWare because it abends/crashes on NetWare
+               when the script has error such as not having the closing quotes like:
+                   if ($var eq "value)
+               Checking of white spaces is anyway done in NetWare code.
+       */
+#ifndef NETWARE
        while (isSPACE(*PL_oldbufptr))
            PL_oldbufptr++;
+#endif
        context = PL_oldbufptr;
        contlen = PL_bufptr - PL_oldbufptr;
     }
@@ -7455,7 +7791,7 @@ Perl_yyerror(pTHX_ char *s)
     }
     msg = sv_2mortal(newSVpv(s, 0));
     Perl_sv_catpvf(aTHX_ msg, " at %s line %"IVdf", ",
-                  CopFILE(PL_curcop), (IV)CopLINE(PL_curcop));
+        OutCopFILE(PL_curcop), (IV)CopLINE(PL_curcop));
     if (context)
        Perl_sv_catpvf(aTHX_ msg, "near \"%.*s\"\n", contlen, context);
     else
@@ -7473,15 +7809,18 @@ Perl_yyerror(pTHX_ char *s)
     if (PL_error_count >= 10) {
        if (PL_in_eval && SvCUR(ERRSV))
            Perl_croak(aTHX_ "%"SVf"%s has too many errors.\n",
-                      ERRSV, CopFILE(PL_curcop));
+            ERRSV, OutCopFILE(PL_curcop));
        else
            Perl_croak(aTHX_ "%s has too many errors.\n",
-                      CopFILE(PL_curcop));
+            OutCopFILE(PL_curcop));
     }
     PL_in_my = 0;
     PL_in_my_stash = Nullhv;
     return 0;
 }
+#ifdef __SC__
+#pragma segment Main
+#endif
 
 STATIC char*
 S_swallow_bom(pTHX_ U8 *s)
@@ -7489,8 +7828,8 @@ S_swallow_bom(pTHX_ U8 *s)
     STRLEN slen;
     slen = SvCUR(PL_linestr);
     switch (*s) {
-    case 0xFF:       
-       if (s[1] == 0xFE) { 
+    case 0xFF:
+       if (s[1] == 0xFE) {
            /* UTF-16 little-endian */
            if (s[2] == 0 && s[3] == 0)  /* UTF-32 little-endian */
                Perl_croak(aTHX_ "Unsupported script encoding");
@@ -7558,17 +7897,13 @@ S_swallow_bom(pTHX_ U8 *s)
     return (char*)s;
 }
 
-#ifdef PERL_OBJECT
-#include "XSUB.h"
-#endif
-
 /*
  * restore_rsfp
  * Restore a source filter.
  */
 
 static void
-restore_rsfp(pTHXo_ void *f)
+restore_rsfp(pTHX_ void *f)
 {
     PerlIO *fp = (PerlIO*)f;
 
@@ -7581,7 +7916,7 @@ restore_rsfp(pTHXo_ void *f)
 
 #ifndef PERL_NO_UTF16_FILTER
 static I32
-utf16_textfilter(pTHXo_ int idx, SV *sv, int maxlen)
+utf16_textfilter(pTHX_ int idx, SV *sv, int maxlen)
 {
     I32 count = FILTER_READ(idx+1, sv, maxlen);
     if (count) {
@@ -7592,7 +7927,7 @@ utf16_textfilter(pTHXo_ int idx, SV *sv, int maxlen)
        if (!*SvPV_nolen(sv))
        /* Game over, but don't feed an odd-length string to utf16_to_utf8 */
        return count;
-       
+
        tend = utf16_to_utf8((U8*)SvPVX(sv), tmps, SvCUR(sv), &newlen);
        sv_usepvn(sv, (char*)tmps, tend - tmps);
     }
@@ -7600,7 +7935,7 @@ utf16_textfilter(pTHXo_ int idx, SV *sv, int maxlen)
 }
 
 static I32
-utf16rev_textfilter(pTHXo_ int idx, SV *sv, int maxlen)
+utf16rev_textfilter(pTHX_ int idx, SV *sv, int maxlen)
 {
     I32 count = FILTER_READ(idx+1, sv, maxlen);
     if (count) {
@@ -7618,3 +7953,4 @@ utf16rev_textfilter(pTHXo_ int idx, SV *sv, int maxlen)
     return count;
 }
 #endif
+