Merge S_utf16_textfilter and S_utf16rev_textfilter().

Use IoLINES() on the filter's SV to determine which encoding is in use.

diff --git a/embed.fnc b/embed.fnc
index 7bbb91a..0fd0a41 100644 (file)
--- a/embed.fnc
+++ b/embed.fnc
@@ -1795,7 +1795,6 @@ sR        |char*  |skipspace      |NN char *s
 sR     |char*  |swallow_bom    |NN U8 *s
 #ifndef PERL_NO_UTF16_FILTER
 s      |I32    |utf16_textfilter|int idx|NN SV *sv|int maxlen
-s      |I32    |utf16rev_textfilter|int idx|NN SV *sv|int maxlen
 #endif
 s      |void   |checkcomma     |NN const char *s|NN const char *name \
                                |NN const char *what

diff --git a/embed.h b/embed.h
index 7e69fec..66c3194 100644 (file)
--- a/embed.h
+++ b/embed.h
@@ -1584,7 +1584,6 @@
 #ifndef PERL_NO_UTF16_FILTER
 #ifdef PERL_CORE
 #define utf16_textfilter       S_utf16_textfilter
-#define utf16rev_textfilter    S_utf16rev_textfilter
 #endif
 #endif
 #ifdef PERL_CORE
@@ -3959,7 +3958,6 @@
 #ifndef PERL_NO_UTF16_FILTER
 #ifdef PERL_CORE
 #define utf16_textfilter(a,b,c)        S_utf16_textfilter(aTHX_ a,b,c)
-#define utf16rev_textfilter(a,b,c)     S_utf16rev_textfilter(aTHX_ a,b,c)
 #endif
 #endif
 #ifdef PERL_CORE

diff --git a/proto.h b/proto.h
index 69a6852..7d47e9b 100644 (file)
--- a/proto.h
+++ b/proto.h
@@ -5770,11 +5770,6 @@ STATIC I32       S_utf16_textfilter(pTHX_ int idx, SV *sv, int maxlen)
 #define PERL_ARGS_ASSERT_UTF16_TEXTFILTER      \
        assert(sv)
 
-STATIC I32     S_utf16rev_textfilter(pTHX_ int idx, SV *sv, int maxlen)
-                       __attribute__nonnull__(pTHX_2);
-#define PERL_ARGS_ASSERT_UTF16REV_TEXTFILTER   \
-       assert(sv)
-
 #endif
 STATIC void    S_checkcomma(pTHX_ const char *s, const char *name, const char *what)
                        __attribute__nonnull__(pTHX_1)

diff --git a/toke.c b/toke.c
index 8e47c9a..c7ec476 100644 (file)
--- a/toke.c
+++ b/toke.c
@@ -12708,7 +12708,7 @@ S_swallow_bom(pTHX_ U8 *s)
                U8 *news;
                I32 newlen;
 
-               filter_add(S_utf16rev_textfilter, NULL);
+               IoLINES(filter_add(S_utf16_textfilter, NULL)) = 1;
                Newx(news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8);
                utf16_to_utf8_reversed(s, news,
                                       PL_bufend - (char*)s - 1,
@@ -12810,8 +12810,10 @@ S_utf16_textfilter(pTHX_ int idx, SV *sv, int maxlen)
     dVAR;
     const STRLEN old = SvCUR(sv);
     const I32 count = FILTER_READ(idx+1, sv, maxlen);
+    const int reverse = IoLINES(sv);
     DEBUG_P(PerlIO_printf(Perl_debug_log,
-                         "utf16_textfilter(%p): %d %d (%d)\n",
+                         "utf16%s_textfilter(%p): %d %d (%d)\n",
+                         reverse ? "rev" : "",
                          FPTR2DPTR(void *, S_utf16_textfilter),
                          idx, maxlen, (int) count));
     if (count) {
@@ -12819,36 +12821,22 @@ S_utf16_textfilter(pTHX_ int idx, SV *sv, int maxlen)
        I32 newlen;
        Newx(tmps, SvCUR(sv) * 3 / 2 + 1, U8);
        Copy(SvPVX_const(sv), tmps, old, char);
-       utf16_to_utf8((U8*)SvPVX_const(sv) + old, tmps + old,
-                     SvCUR(sv) - old, &newlen);
+       if (reverse) {
+           /* You would expect this to be utf16_to_utf8_reversed()
+              It was, prior to 1de9afcdf18cf98bbdecaa782da93e907be6fe4e
+              Effectively, right now, UTF-16LE is being read in off-by-one
+              See RT #69678  */
+           utf16_to_utf8((U8*)SvPVX_const(sv) + old, tmps + old,
+                         SvCUR(sv) - old, &newlen);
+       } else {
+           utf16_to_utf8((U8*)SvPVX_const(sv) + old, tmps + old,
+                         SvCUR(sv) - old, &newlen);
+       }
        sv_usepvn(sv, (char*)tmps, (STRLEN)newlen + old);
     }
     DEBUG_P({sv_dump(sv);});
     return SvCUR(sv);
 }
-
-static I32
-S_utf16rev_textfilter(pTHX_ int idx, SV *sv, int maxlen)
-{
-    dVAR;
-    const STRLEN old = SvCUR(sv);
-    const I32 count = FILTER_READ(idx+1, sv, maxlen);
-    DEBUG_P(PerlIO_printf(Perl_debug_log,
-                         "utf16rev_textfilter(%p): %d %d (%d)\n",
-                         FPTR2DPTR(void *, utf16rev_textfilter),
-                         idx, maxlen, (int) count));
-    if (count) {
-       U8* tmps;
-       I32 newlen;
-       Newx(tmps, SvCUR(sv) * 3 / 2 + 1, U8);
-       Copy(SvPVX_const(sv), tmps, old, char);
-       utf16_to_utf8((U8*)SvPVX_const(sv) + old, tmps + old,
-                     SvCUR(sv) - old, &newlen);
-       sv_usepvn(sv, (char*)tmps, (STRLEN)newlen + old);
-    }
-    DEBUG_P({ sv_dump(sv); });
-    return count;
-}
 #endif
 
 /*