From: Nicholas Clark <nick@ccl4.org>
Date: Sun, 18 Oct 2009 16:47:10 +0000 (+0100)
Subject: Merge S_utf16_textfilter and S_utf16rev_textfilter().
X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=aa6dbd607b0a3d8a49c43b63b63b73ed167692c3;p=p5sagit%2Fp5-mst-13.2.git

Merge S_utf16_textfilter and S_utf16rev_textfilter().

Use IoLINES() on the filter's SV to determine which encoding is in use.
---

diff --git a/embed.fnc b/embed.fnc
index 7bbb91a..0fd0a41 100644
--- a/embed.fnc
+++ b/embed.fnc
@@ -1795,7 +1795,6 @@ sR	|char*	|skipspace	|NN char *s
 sR	|char*	|swallow_bom	|NN U8 *s
 #ifndef PERL_NO_UTF16_FILTER
 s	|I32	|utf16_textfilter|int idx|NN SV *sv|int maxlen
-s	|I32	|utf16rev_textfilter|int idx|NN SV *sv|int maxlen
 #endif
 s	|void	|checkcomma	|NN const char *s|NN const char *name \
 				|NN const char *what
diff --git a/embed.h b/embed.h
index 7e69fec..66c3194 100644
--- a/embed.h
+++ b/embed.h
@@ -1584,7 +1584,6 @@
 #ifndef PERL_NO_UTF16_FILTER
 #ifdef PERL_CORE
 #define utf16_textfilter	S_utf16_textfilter
-#define utf16rev_textfilter	S_utf16rev_textfilter
 #endif
 #endif
 #ifdef PERL_CORE
@@ -3959,7 +3958,6 @@
 #ifndef PERL_NO_UTF16_FILTER
 #ifdef PERL_CORE
 #define utf16_textfilter(a,b,c)	S_utf16_textfilter(aTHX_ a,b,c)
-#define utf16rev_textfilter(a,b,c)	S_utf16rev_textfilter(aTHX_ a,b,c)
 #endif
 #endif
 #ifdef PERL_CORE
diff --git a/proto.h b/proto.h
index 69a6852..7d47e9b 100644
--- a/proto.h
+++ b/proto.h
@@ -5770,11 +5770,6 @@ STATIC I32	S_utf16_textfilter(pTHX_ int idx, SV *sv, int maxlen)
 #define PERL_ARGS_ASSERT_UTF16_TEXTFILTER	\
 	assert(sv)
 
-STATIC I32	S_utf16rev_textfilter(pTHX_ int idx, SV *sv, int maxlen)
-			__attribute__nonnull__(pTHX_2);
-#define PERL_ARGS_ASSERT_UTF16REV_TEXTFILTER	\
-	assert(sv)
-
 #endif
 STATIC void	S_checkcomma(pTHX_ const char *s, const char *name, const char *what)
 			__attribute__nonnull__(pTHX_1)
diff --git a/toke.c b/toke.c
index 8e47c9a..c7ec476 100644
--- a/toke.c
+++ b/toke.c
@@ -12708,7 +12708,7 @@ S_swallow_bom(pTHX_ U8 *s)
 		U8 *news;
 		I32 newlen;
 
-		filter_add(S_utf16rev_textfilter, NULL);
+		IoLINES(filter_add(S_utf16_textfilter, NULL)) = 1;
 		Newx(news, (PL_bufend - (char*)s) * 3 / 2 + 1, U8);
 		utf16_to_utf8_reversed(s, news,
 				       PL_bufend - (char*)s - 1,
@@ -12810,8 +12810,10 @@ S_utf16_textfilter(pTHX_ int idx, SV *sv, int maxlen)
     dVAR;
     const STRLEN old = SvCUR(sv);
     const I32 count = FILTER_READ(idx+1, sv, maxlen);
+    const int reverse = IoLINES(sv);
     DEBUG_P(PerlIO_printf(Perl_debug_log,
-			  "utf16_textfilter(%p): %d %d (%d)\n",
+			  "utf16%s_textfilter(%p): %d %d (%d)\n",
+			  reverse ? "rev" : "",
 			  FPTR2DPTR(void *, S_utf16_textfilter),
 			  idx, maxlen, (int) count));
     if (count) {
@@ -12819,36 +12821,22 @@ S_utf16_textfilter(pTHX_ int idx, SV *sv, int maxlen)
 	I32 newlen;
 	Newx(tmps, SvCUR(sv) * 3 / 2 + 1, U8);
 	Copy(SvPVX_const(sv), tmps, old, char);
-	utf16_to_utf8((U8*)SvPVX_const(sv) + old, tmps + old,
-		      SvCUR(sv) - old, &newlen);
+	if (reverse) {
+	    /* You would expect this to be utf16_to_utf8_reversed()
+	       It was, prior to 1de9afcdf18cf98bbdecaa782da93e907be6fe4e
+	       Effectively, right now, UTF-16LE is being read in off-by-one
+	       See RT #69678  */
+	    utf16_to_utf8((U8*)SvPVX_const(sv) + old, tmps + old,
+			  SvCUR(sv) - old, &newlen);
+	} else {
+	    utf16_to_utf8((U8*)SvPVX_const(sv) + old, tmps + old,
+			  SvCUR(sv) - old, &newlen);
+	}
 	sv_usepvn(sv, (char*)tmps, (STRLEN)newlen + old);
     }
     DEBUG_P({sv_dump(sv);});
     return SvCUR(sv);
 }
-
-static I32
-S_utf16rev_textfilter(pTHX_ int idx, SV *sv, int maxlen)
-{
-    dVAR;
-    const STRLEN old = SvCUR(sv);
-    const I32 count = FILTER_READ(idx+1, sv, maxlen);
-    DEBUG_P(PerlIO_printf(Perl_debug_log,
-			  "utf16rev_textfilter(%p): %d %d (%d)\n",
-			  FPTR2DPTR(void *, utf16rev_textfilter),
-			  idx, maxlen, (int) count));
-    if (count) {
-	U8* tmps;
-	I32 newlen;
-	Newx(tmps, SvCUR(sv) * 3 / 2 + 1, U8);
-	Copy(SvPVX_const(sv), tmps, old, char);
-	utf16_to_utf8((U8*)SvPVX_const(sv) + old, tmps + old,
-		      SvCUR(sv) - old, &newlen);
-	sv_usepvn(sv, (char*)tmps, (STRLEN)newlen + old);
-    }
-    DEBUG_P({ sv_dump(sv); });
-    return count;
-}
 #endif
 
 /*