X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=blobdiff_plain;f=ext%2FEncode%2FEncode.xs;h=8fb41063aaa82e2e87e4e07353f9e3e43f630192;hb=071db25d4bd6237e4ead7e44b9c1420448a117ff;hp=68a7008aa0d8a3e1a6cf167f0bc1d001f5b1ca0e;hpb=4a83738abbec44c7097ec530f795374eb56f9a88;p=p5sagit%2Fp5-mst-13.2.git

diff --git a/ext/Encode/Encode.xs b/ext/Encode/Encode.xs
index 68a7008..8fb4106 100644
--- a/ext/Encode/Encode.xs
+++ b/ext/Encode/Encode.xs
@@ -1,15 +1,633 @@
+#define PERL_NO_GET_CONTEXT
+
 #include "EXTERN.h"
 #include "perl.h"
 #include "XSUB.h"
+#define U8 U8
+#include "encode.h"
+/* #include "8859.h" */
+/* #include "EBCDIC.h" */
+/* #include "Symbols.h" */
+#include "def_t.h"
 
-#define UNIMPLEMENTED(x,y) y x (SV *sv, char *encoding) {   \
-                         Perl_croak("panic_unimplemented"); \
+#define UNIMPLEMENTED(x,y) y x (SV *sv, char *encoding) {dTHX;   \
+                         Perl_croak(aTHX_ "panic_unimplemented"); \
 			 return (y)0; /* fool picky compilers */ \
-                         } 
+                         }
 UNIMPLEMENTED(_encoded_utf8_to_bytes, I32)
-UNIMPLEMENTED(_encoded_bytes_to_utf8, I32)
+    UNIMPLEMENTED(_encoded_bytes_to_utf8, I32)
+#if defined(USE_PERLIO) && !defined(USE_SFIO)
+/* Define an encoding "layer" in the perliol.h sense.
+   The layer defined here "inherits" in an object-oriented sense from the
+   "perlio" layer with its PerlIOBuf_* "methods".
+   The implementation is particularly efficient as until Encode settles down
+   there is no point in tryint to tune it.
+
+   The layer works by overloading the "fill" and "flush" methods.
+
+   "fill" calls "SUPER::fill" in perl terms, then calls the encode OO perl API
+   to convert the encoded data to UTF-8 form, then copies it back to the
+   buffer. The "base class's" read methods then see the UTF-8 data.
+
+   "flush" transforms the UTF-8 data deposited by the "base class's write
+   method in the buffer back into the encoded form using the encode OO perl API,
+   then copies data back into the buffer and calls "SUPER::flush.
+
+   Note that "flush" is _also_ called for read mode - we still do the (back)-translate
+   so that the the base class's "flush" sees the correct number of encoded chars
+   for positioning the seek pointer. (This double translation is the worst performance
+   issue - particularly with all-perl encode engine.)
+
+*/
+#include "perliol.h"
+typedef struct {
+    PerlIOBuf base;		/* PerlIOBuf stuff */
+    SV *bufsv;			/* buffer seen by layers above */
+    SV *dataSV;			/* data we have read from layer below */
+    SV *enc;			/* the encoding object */
+} PerlIOEncode;
+
+SV *
+PerlIOEncode_getarg(pTHX_ PerlIO * f, CLONE_PARAMS * param, int flags)
+{
+    PerlIOEncode *e = PerlIOSelf(f, PerlIOEncode);
+    SV *sv = &PL_sv_undef;
+    if (e->enc) {
+	dSP;
+	ENTER;
+	SAVETMPS;
+	PUSHMARK(sp);
+	XPUSHs(e->enc);
+	PUTBACK;
+	if (perl_call_method("name", G_SCALAR) == 1) {
+	    SPAGAIN;
+	    sv = newSVsv(POPs);
+	    PUTBACK;
+	}
+    }
+    return sv;
+}
+
+IV
+PerlIOEncode_pushed(pTHX_ PerlIO * f, const char *mode, SV * arg)
+{
+    PerlIOEncode *e = PerlIOSelf(f, PerlIOEncode);
+    dSP;
+    IV code;
+    code = PerlIOBuf_pushed(aTHX_ f, mode, Nullsv);
+    ENTER;
+    SAVETMPS;
+    PUSHMARK(sp);
+    XPUSHs(arg);
+    PUTBACK;
+    if (perl_call_pv("Encode::find_encoding", G_SCALAR) != 1) {
+	/* should never happen */
+	Perl_die(aTHX_ "Encode::find_encoding did not return a value");
+	return -1;
+    }
+    SPAGAIN;
+    e->enc = POPs;
+    PUTBACK;
+    if (!SvROK(e->enc)) {
+	e->enc = Nullsv;
+	errno = EINVAL;
+	Perl_warner(aTHX_ packWARN(WARN_IO), "Cannot find encoding \"%" SVf "\"",
+		    arg);
+	code = -1;
+    }
+    else {
+	SvREFCNT_inc(e->enc);
+	PerlIOBase(f)->flags |= PERLIO_F_UTF8;
+    }
+    FREETMPS;
+    LEAVE;
+    return code;
+}
+
+IV
+PerlIOEncode_popped(pTHX_ PerlIO * f)
+{
+    PerlIOEncode *e = PerlIOSelf(f, PerlIOEncode);
+    if (e->enc) {
+	SvREFCNT_dec(e->enc);
+	e->enc = Nullsv;
+    }
+    if (e->bufsv) {
+	SvREFCNT_dec(e->bufsv);
+	e->bufsv = Nullsv;
+    }
+    if (e->dataSV) {
+	SvREFCNT_dec(e->dataSV);
+	e->dataSV = Nullsv;
+    }
+    return 0;
+}
 
-void call_failure (SV *routine, U8* done, U8* dest, U8* orig) {}
+STDCHAR *
+PerlIOEncode_get_base(pTHX_ PerlIO * f)
+{
+    PerlIOEncode *e = PerlIOSelf(f, PerlIOEncode);
+    if (!e->base.bufsiz)
+	e->base.bufsiz = 1024;
+    if (!e->bufsv) {
+	e->bufsv = newSV(e->base.bufsiz);
+	sv_setpvn(e->bufsv, "", 0);
+    }
+    e->base.buf = (STDCHAR *) SvPVX(e->bufsv);
+    if (!e->base.ptr)
+	e->base.ptr = e->base.buf;
+    if (!e->base.end)
+	e->base.end = e->base.buf;
+    if (e->base.ptr < e->base.buf
+	|| e->base.ptr > e->base.buf + SvLEN(e->bufsv)) {
+	Perl_warn(aTHX_ " ptr %p(%p)%p", e->base.buf, e->base.ptr,
+		  e->base.buf + SvLEN(e->bufsv));
+	abort();
+    }
+    if (SvLEN(e->bufsv) < e->base.bufsiz) {
+	SSize_t poff = e->base.ptr - e->base.buf;
+	SSize_t eoff = e->base.end - e->base.buf;
+	e->base.buf = (STDCHAR *) SvGROW(e->bufsv, e->base.bufsiz);
+	e->base.ptr = e->base.buf + poff;
+	e->base.end = e->base.buf + eoff;
+    }
+    if (e->base.ptr < e->base.buf
+	|| e->base.ptr > e->base.buf + SvLEN(e->bufsv)) {
+	Perl_warn(aTHX_ " ptr %p(%p)%p", e->base.buf, e->base.ptr,
+		  e->base.buf + SvLEN(e->bufsv));
+	abort();
+    }
+    return e->base.buf;
+}
+
+IV
+PerlIOEncode_fill(pTHX_ PerlIO * f)
+{
+    PerlIOEncode *e = PerlIOSelf(f, PerlIOEncode);
+    dSP;
+    IV code = 0;
+    PerlIO *n;
+    SSize_t avail;
+    if (PerlIO_flush(f) != 0)
+	return -1;
+    n  = PerlIONext(f);
+    if (!PerlIO_fast_gets(n)) {
+	/* Things get too messy if we don't have a buffer layer
+	   push a :perlio to do the job */
+	char mode[8];
+	n  = PerlIO_push(aTHX_ n, &PerlIO_perlio, PerlIO_modestr(f,mode), Nullsv);
+	if (!n) {
+	    Perl_die(aTHX_ "panic: cannot push :perlio for %p",f);
+	}
+    }
+    ENTER;
+    SAVETMPS;
+  retry:
+    avail = PerlIO_get_cnt(n);
+    if (avail <= 0) {
+	avail = PerlIO_fill(n);
+	if (avail == 0) {
+	    avail = PerlIO_get_cnt(n);
+	}
+	else {
+	    if (!PerlIO_error(n) && PerlIO_eof(n))
+		avail = 0;
+	}
+    }
+    if (avail > 0) {
+	STDCHAR *ptr = PerlIO_get_ptr(n);
+	SSize_t use  = avail;
+	SV *uni;
+	char *s;
+	STRLEN len = 0;
+	e->base.ptr = e->base.end = (STDCHAR *) Nullch;
+	(void) PerlIOEncode_get_base(aTHX_ f);
+	if (!e->dataSV)
+	    e->dataSV = newSV(0);
+	if (SvTYPE(e->dataSV) < SVt_PV) {
+	    sv_upgrade(e->dataSV,SVt_PV);
+	}
+	if (SvCUR(e->dataSV)) {
+	    /* something left over from last time - create a normal
+	       SV with new data appended
+	     */
+	    if (use + SvCUR(e->dataSV) > e->base.bufsiz) {
+	       use = e->base.bufsiz - SvCUR(e->dataSV);
+	    }
+	    sv_catpvn(e->dataSV,(char*)ptr,use);
+	}
+	else {
+	    /* Create a "dummy" SV to represent the available data from layer below */
+	    if (SvLEN(e->dataSV) && SvPVX(e->dataSV)) {
+		Safefree(SvPVX(e->dataSV));
+	    }
+	    if (use > e->base.bufsiz) {
+	       use = e->base.bufsiz;
+	    }
+	    SvPVX(e->dataSV) = (char *) ptr;
+	    SvLEN(e->dataSV) = 0;  /* Hands off sv.c - it isn't yours */
+	    SvCUR_set(e->dataSV,use);
+	    SvPOK_only(e->dataSV);
+	}
+	SvUTF8_off(e->dataSV);
+	PUSHMARK(sp);
+	XPUSHs(e->enc);
+	XPUSHs(e->dataSV);
+	XPUSHs(&PL_sv_yes);
+	PUTBACK;
+	if (perl_call_method("decode", G_SCALAR) != 1) {
+	    Perl_die(aTHX_ "panic: decode did not return a value");
+	}
+	SPAGAIN;
+	uni = POPs;
+	PUTBACK;
+	/* Now get translated string (forced to UTF-8) and use as buffer */
+	if (SvPOK(uni)) {
+	    s = SvPVutf8(uni, len);
+	    if (len && !is_utf8_string((U8*)s,len)) {
+		Perl_warn(aTHX_ "panic: decode did not return UTF-8 '%.*s'",(int) len,s);
+	    }
+	}
+	if (len > 0) {
+	    /* Got _something */
+	    /* if decode gave us back dataSV then data may vanish when
+	       we do ptrcnt adjust - so take our copy now.
+	       (The copy is a pain - need a put-it-here option for decode.)
+	     */
+	    sv_setpvn(e->bufsv,s,len);
+	    e->base.ptr = e->base.buf = (STDCHAR*)SvPVX(e->bufsv);
+	    e->base.end = e->base.ptr + SvCUR(e->bufsv);
+	    PerlIOBase(f)->flags |= PERLIO_F_RDBUF;
+	    SvUTF8_on(e->bufsv);
+
+	    /* Adjust ptr/cnt not taking anything which
+	       did not translate - not clear this is a win */
+	    /* compute amount we took */
+	    use -= SvCUR(e->dataSV);
+	    PerlIO_set_ptrcnt(n, ptr+use, (avail-use));
+	    /* and as we did not take it it isn't pending */
+	    SvCUR_set(e->dataSV,0);
+	} else {
+	    /* Got nothing - assume partial character so we need some more */
+	    /* Make sure e->dataSV is a normal SV before re-filling as
+	       buffer alias will change under us
+	     */
+	    s = SvPV(e->dataSV,len);
+	    sv_setpvn(e->dataSV,s,len);
+	    PerlIO_set_ptrcnt(n, ptr+use, (avail-use));
+	    goto retry;
+	}
+	FREETMPS;
+	LEAVE;
+	return code;
+    }
+    else {
+	if (avail == 0)
+	    PerlIOBase(f)->flags |= PERLIO_F_EOF;
+	else
+	    PerlIOBase(f)->flags |= PERLIO_F_ERROR;
+	return -1;
+    }
+}
+
+IV
+PerlIOEncode_flush(pTHX_ PerlIO * f)
+{
+    PerlIOEncode *e = PerlIOSelf(f, PerlIOEncode);
+    IV code = 0;
+    if (e->bufsv && (e->base.ptr > e->base.buf)) {
+	dSP;
+	SV *str;
+	char *s;
+	STRLEN len;
+	SSize_t count = 0;
+	if (PerlIOBase(f)->flags & PERLIO_F_WRBUF) {
+	    /* Write case encode the buffer and write() to layer below */
+	    ENTER;
+	    SAVETMPS;
+	    PUSHMARK(sp);
+	    XPUSHs(e->enc);
+	    SvCUR_set(e->bufsv, e->base.ptr - e->base.buf);
+	    SvUTF8_on(e->bufsv);
+	    XPUSHs(e->bufsv);
+	    XPUSHs(&PL_sv_yes);
+	    PUTBACK;
+	    if (perl_call_method("encode", G_SCALAR) != 1) {
+		Perl_die(aTHX_ "panic: encode did not return a value");
+	    }
+	    SPAGAIN;
+	    str = POPs;
+	    PUTBACK;
+	    s = SvPV(str, len);
+	    count = PerlIO_write(PerlIONext(f),s,len);
+	    if (count != len) {
+		code = -1;
+	    }
+	    FREETMPS;
+	    LEAVE;
+	    if (PerlIO_flush(PerlIONext(f)) != 0) {
+		code = -1;
+	    }
+	    if (SvCUR(e->bufsv)) {
+		/* Did not all translate */
+		e->base.ptr = e->base.buf+SvCUR(e->bufsv);
+		return code;
+	    }
+	}
+	else if (PerlIOBase(f)->flags & PERLIO_F_RDBUF) {
+	    /* read case */
+	    /* if we have any untranslated stuff then unread that first */
+	    if (e->dataSV && SvCUR(e->dataSV)) {
+		s = SvPV(e->dataSV, len);
+		count = PerlIO_unread(PerlIONext(f),s,len);
+		if (count != len) {
+		    code = -1;
+		}
+	    }
+	    /* See if there is anything left in the buffer */
+	    if (e->base.ptr < e->base.end) {
+		/* Bother - have unread data.
+		   re-encode and unread() to layer below
+		 */
+		ENTER;
+		SAVETMPS;
+		str = sv_newmortal();
+		sv_upgrade(str, SVt_PV);
+		SvPVX(str) = (char*)e->base.ptr;
+		SvLEN(str) = 0;
+		SvCUR_set(str, e->base.end - e->base.ptr);
+		SvPOK_only(str);
+		SvUTF8_on(str);
+		PUSHMARK(sp);
+		XPUSHs(e->enc);
+		XPUSHs(str);
+		XPUSHs(&PL_sv_yes);
+		PUTBACK;
+		if (perl_call_method("encode", G_SCALAR) != 1) {
+		     Perl_die(aTHX_ "panic: encode did not return a value");
+		}
+		SPAGAIN;
+		str = POPs;
+		PUTBACK;
+		s = SvPV(str, len);
+		count = PerlIO_unread(PerlIONext(f),s,len);
+		if (count != len) {
+		    code = -1;
+		}
+		FREETMPS;
+		LEAVE;
+	    }
+	}
+	e->base.ptr = e->base.end = e->base.buf;
+	PerlIOBase(f)->flags &= ~(PERLIO_F_RDBUF | PERLIO_F_WRBUF);
+    }
+    return code;
+}
+
+IV
+PerlIOEncode_close(pTHX_ PerlIO * f)
+{
+    PerlIOEncode *e = PerlIOSelf(f, PerlIOEncode);
+    IV code = PerlIOBase_close(aTHX_ f);
+    if (e->bufsv) {
+	if (e->base.buf && e->base.ptr > e->base.buf) {
+	    Perl_croak(aTHX_ "Close with partial character");
+	}
+	SvREFCNT_dec(e->bufsv);
+	e->bufsv = Nullsv;
+    }
+    e->base.buf = NULL;
+    e->base.ptr = NULL;
+    e->base.end = NULL;
+    PerlIOBase(f)->flags &= ~(PERLIO_F_RDBUF | PERLIO_F_WRBUF);
+    return code;
+}
+
+Off_t
+PerlIOEncode_tell(pTHX_ PerlIO * f)
+{
+    PerlIOBuf *b = PerlIOSelf(f, PerlIOBuf);
+    /* Unfortunately the only way to get a postion is to (re-)translate,
+       the UTF8 we have in bufefr and then ask layer below
+     */
+    PerlIO_flush(f);
+    if (b->buf && b->ptr > b->buf) {
+	Perl_croak(aTHX_ "Cannot tell at partial character");
+    }
+    return PerlIO_tell(PerlIONext(f));
+}
+
+PerlIO *
+PerlIOEncode_dup(pTHX_ PerlIO * f, PerlIO * o,
+		 CLONE_PARAMS * params, int flags)
+{
+    if ((f = PerlIOBase_dup(aTHX_ f, o, params, flags))) {
+	PerlIOEncode *fe = PerlIOSelf(f, PerlIOEncode);
+	PerlIOEncode *oe = PerlIOSelf(o, PerlIOEncode);
+	if (oe->enc) {
+	    fe->enc = PerlIO_sv_dup(aTHX_ oe->enc, params);
+	}
+    }
+    return f;
+}
+
+PerlIO_funcs PerlIO_encode = {
+    "encoding",
+    sizeof(PerlIOEncode),
+    PERLIO_K_BUFFERED|PERLIO_K_DESTRUCT,
+    PerlIOEncode_pushed,
+    PerlIOEncode_popped,
+    PerlIOBuf_open,
+    PerlIOEncode_getarg,
+    PerlIOBase_fileno,
+    PerlIOEncode_dup,
+    PerlIOBuf_read,
+    PerlIOBuf_unread,
+    PerlIOBuf_write,
+    PerlIOBuf_seek,
+    PerlIOEncode_tell,
+    PerlIOEncode_close,
+    PerlIOEncode_flush,
+    PerlIOEncode_fill,
+    PerlIOBase_eof,
+    PerlIOBase_error,
+    PerlIOBase_clearerr,
+    PerlIOBase_setlinebuf,
+    PerlIOEncode_get_base,
+    PerlIOBuf_bufsiz,
+    PerlIOBuf_get_ptr,
+    PerlIOBuf_get_cnt,
+    PerlIOBuf_set_ptrcnt,
+};
+#endif				/* encode layer */
+
+void
+Encode_XSEncoding(pTHX_ encode_t * enc)
+{
+    dSP;
+    HV *stash = gv_stashpv("Encode::XS", TRUE);
+    SV *sv = sv_bless(newRV_noinc(newSViv(PTR2IV(enc))), stash);
+    int i = 0;
+    PUSHMARK(sp);
+    XPUSHs(sv);
+    while (enc->name[i]) {
+	const char *name = enc->name[i++];
+	XPUSHs(sv_2mortal(newSVpvn(name, strlen(name))));
+    }
+    PUTBACK;
+    call_pv("Encode::define_encoding", G_DISCARD);
+    SvREFCNT_dec(sv);
+}
+
+void
+call_failure(SV * routine, U8 * done, U8 * dest, U8 * orig)
+{
+ /* Exists for breakpointing */
+}
+
+static SV *
+encode_method(pTHX_ encode_t * enc, encpage_t * dir, SV * src,
+			 int check)
+{
+    STRLEN slen;
+    U8 *s = (U8 *) SvPV(src, slen);
+    STRLEN tlen  = slen;
+    STRLEN ddone = 0;
+    STRLEN sdone = 0;
+    SV *dst = sv_2mortal(newSV(slen+1));
+    if (slen) {
+	U8 *d = (U8 *) SvPVX(dst);
+	STRLEN dlen = SvLEN(dst)-1;
+	int code;
+	while ((code = do_encode(dir, s, &slen, d, dlen, &dlen, !check))) {
+	    SvCUR_set(dst, dlen+ddone);
+	    SvPOK_only(dst);
+
+#if 0
+	    Perl_warn(aTHX_ "code=%d @ s=%d/%d/%d d=%d/%d/%d",code,slen,sdone,tlen,dlen,ddone,SvLEN(dst)-1);
+#endif
+	
+	    if (code == ENCODE_FALLBACK || code == ENCODE_PARTIAL)
+		break;
+
+	    switch (code) {
+	    case ENCODE_NOSPACE:
+		{
+		    STRLEN need ;
+		    sdone += slen;
+		    ddone += dlen;
+		    if (sdone) {
+			need = (tlen*SvLEN(dst)+sdone-1)/sdone+UTF8_MAXLEN;
+		    }
+		    else {
+			need = SvLEN(dst) + UTF8_MAXLEN;
+		    }
+		
+		    d = (U8 *) SvGROW(dst, need);
+		    if (ddone >= SvLEN(dst)) {
+			Perl_croak(aTHX_ "Destination couldn't be grown.");
+		    }
+		    dlen = SvLEN(dst)-ddone-1;
+		    d   += ddone;
+		    s   += slen;
+		    slen = tlen-sdone;
+		    continue;
+		}
+
+	    case ENCODE_NOREP:
+		if (dir == enc->f_utf8) {
+		    if (!check && ckWARN_d(WARN_UTF8)) {
+			STRLEN clen;
+			UV ch =
+			    utf8n_to_uvuni(s + slen, (SvCUR(src) - slen),
+					   &clen, 0);
+			Perl_warner(aTHX_ packWARN(WARN_UTF8),
+				    "\"\\N{U+%" UVxf
+				    "}\" does not map to %s", ch,
+				    enc->name[0]);
+			/* FIXME: Skip over the character, copy in replacement and continue
+			 * but that is messy so for now just fail.
+			 */
+			return &PL_sv_undef;
+		    }
+		    else {
+			return &PL_sv_undef;
+		    }
+		}
+		else {
+		    /* UTF-8 is supposed to be "Universal" so should not happen
+		       for real characters, but some encodings have non-assigned
+		       codes which may occur.
+		     */
+		    Perl_croak(aTHX_ "%s \"\\x%02X\" does not map to Unicode (%d)",
+			       enc->name[0], (U8) s[slen], code);
+		}
+		break;
+
+	    default:
+		Perl_croak(aTHX_ "Unexpected code %d converting %s %s",
+			   code, (dir == enc->f_utf8) ? "to" : "from",
+			   enc->name[0]);
+		return &PL_sv_undef;
+	    }
+	}
+	SvCUR_set(dst, dlen+ddone);
+	SvPOK_only(dst);
+	if (check) {
+	    sdone = SvCUR(src) - (slen+sdone);
+	    if (sdone) {
+		Move(s + slen, SvPVX(src), sdone , U8);
+	    }
+	    SvCUR_set(src, sdone);
+	}
+    }
+    else {
+	SvCUR_set(dst, 0);
+	SvPOK_only(dst);
+    }
+    *SvEND(dst) = '\0';
+    return dst;
+}
+
+MODULE = Encode		PACKAGE = Encode::XS	PREFIX = Method_
+
+PROTOTYPES: ENABLE
+
+void
+Method_name(obj)
+SV *	obj
+CODE:
+ {
+  encode_t *enc = INT2PTR(encode_t *, SvIV(SvRV(obj)));
+  ST(0) = sv_2mortal(newSVpvn(enc->name[0],strlen(enc->name[0])));
+  XSRETURN(1);
+ }
+
+void
+Method_decode(obj,src,check = FALSE)
+SV *	obj
+SV *	src
+bool	check
+CODE:
+ {
+  encode_t *enc = INT2PTR(encode_t *, SvIV(SvRV(obj)));
+  ST(0) = encode_method(aTHX_ enc, enc->t_utf8, src, check);
+  SvUTF8_on(ST(0));
+  XSRETURN(1);
+ }
+
+void
+Method_encode(obj,src,check = FALSE)
+SV *	obj
+SV *	src
+bool	check
+CODE:
+ {
+  encode_t *enc = INT2PTR(encode_t *, SvIV(SvRV(obj)));
+  sv_utf8_upgrade(src);
+  ST(0) = encode_method(aTHX_ enc, enc->f_utf8, src, check);
+  XSRETURN(1);
+ }
 
 MODULE = Encode         PACKAGE = Encode
 
@@ -46,18 +664,18 @@ _utf8_to_bytes(sv, ...)
         {
           SV * to    = items > 1 ? ST(1) : Nullsv;
           SV * check = items > 2 ? ST(2) : Nullsv;
-          
+
           if (to)
             RETVAL = _encoded_utf8_to_bytes(sv, SvPV_nolen(to));
           else {
-            U8 *s;
             STRLEN len;
-            s = (U8*)SvPV(sv, len);
+            U8 *s = (U8*)SvPV(sv, len);
 
+	    RETVAL = 0;
             if (SvTRUE(check)) {
               /* Must do things the slow way */
               U8 *dest;
-              U8 *src  = (U8*)savepv((char *)s); /* We need a copy to pass to check() */ 
+              U8 *src  = (U8*)savepv((char *)s); /* We need a copy to pass to check() */
               U8 *send = s + len;
 
               New(83, dest, len, U8); /* I think */
@@ -66,10 +684,9 @@ _utf8_to_bytes(sv, ...)
                 if (*s < 0x80)
                   *dest++ = *s++;
                 else {
-                  I32 ulen;
-                  I32 byte;
-		  I32 uv = *s++;
-                  
+                  STRLEN ulen;
+		  UV uv = *s++;
+
                   /* Have to do it all ourselves because of error routine,
 		     aargh. */
 		  if (!(uv & 0x40))
@@ -81,15 +698,15 @@ _utf8_to_bytes(sv, ...)
 		  else if (!(uv & 0x02)) { ulen = 6;  uv &= 0x01; }
 		  else if (!(uv & 0x01)) { ulen = 7;  uv = 0; }
 		  else                   { ulen = 13; uv = 0; }
-		  
+		
 		  /* Note change to utf8.c variable naming, for variety */
 		  while (ulen--) {
 		    if ((*s & 0xc0) != 0x80)
 		      goto failure;
-		    
+		
 		    else
 		      uv = (uv << 6) | (*s++ & 0x3f);
-		  } 
+		  }
 		  if (uv > 256) {
 		  failure:
 		    call_failure(check, s, dest, src);
@@ -105,105 +722,36 @@ _utf8_to_bytes(sv, ...)
       OUTPUT:
 	RETVAL
 
-SV *
-_chars_to_utf8(sv, from, ...)
-	SV *	sv
-	SV *	from
-      CODE:
-	{
-	  SV * check = items == 3 ? ST(2) : Nullsv;
-	  RETVAL = &PL_sv_undef;
-	}
-      OUTPUT:
-	RETVAL
-
-SV *
-_utf8_to_chars(sv, to, ...)
-	SV *	sv
-	SV *	to
-      CODE:
-	{
-	  SV * check = items == 3 ? ST(2) : Nullsv;
-	  RETVAL = &PL_sv_undef;
-	}
-      OUTPUT:
-	RETVAL
-
-SV *
-_utf8_to_chars_check(sv, ...)
-	SV *	sv
-      CODE:
-	{
-	  SV * check = items == 2 ? ST(1) : Nullsv;
-	  RETVAL = &PL_sv_undef;
-	}
-      OUTPUT:
-	RETVAL
-
-SV *
-_bytes_to_chars(sv, from, ...)
-	SV *	sv
-	SV *	from
-      CODE:
-	{
-	  SV * check = items == 3 ? ST(2) : Nullsv;
-	  RETVAL = &PL_sv_undef;
-	}
-      OUTPUT:
-	RETVAL
-
-SV *
-_chars_to_bytes(sv, to, ...)
-	SV *	sv
-	SV *	to
-      CODE:
-	{
-	  SV * check = items == 3 ? ST(2) : Nullsv;
-	  RETVAL = &PL_sv_undef;
-	}
-      OUTPUT:
-	RETVAL
-
-SV *
-_from_to(sv, from, to, ...)
-	SV *	sv
-	SV *	from
-	SV *	to
-      CODE:
-	{
-	  SV * check = items == 4 ? ST(3) : Nullsv;
-	  RETVAL = &PL_sv_undef;
-	}
-      OUTPUT:
-	RETVAL
-
 bool
-_is_utf8(sv, ...)
-	SV *	sv
+is_utf8(sv, check = FALSE)
+SV *	sv
+bool	check
       CODE:
 	{
-	  SV *	check = items == 2 ? ST(1) : Nullsv;
+	  if (SvGMAGICAL(sv)) /* it could be $1, for example */
+	    sv = newSVsv(sv); /* GMAGIG will be done */
 	  if (SvPOK(sv)) {
-	    RETVAL = SvUTF8(sv);
+	    RETVAL = SvUTF8(sv) ? TRUE : FALSE;
 	    if (RETVAL &&
-		SvTRUE(check) &&
+		check  &&
 		!is_utf8_string((U8*)SvPVX(sv), SvCUR(sv)))
 	      RETVAL = FALSE;
 	  } else {
 	    RETVAL = FALSE;
 	  }
+	  if (sv != ST(0))
+	    SvREFCNT_dec(sv); /* it was a temp copy */
 	}
       OUTPUT:
 	RETVAL
 
 SV *
-_on_utf8(sv)
+_utf8_on(sv)
 	SV *	sv
       CODE:
 	{
 	  if (SvPOK(sv)) {
-	    SV *rsv = newSViv(SvUTF8(sv)); 
-	    sv_2mortal(rsv);
+	    SV *rsv = newSViv(SvUTF8(sv));
 	    RETVAL = rsv;
 	    SvUTF8_on(sv);
 	  } else {
@@ -214,13 +762,12 @@ _on_utf8(sv)
 	RETVAL
 
 SV *
-_off_utf8(sv)
+_utf8_off(sv)
 	SV *	sv
       CODE:
 	{
 	  if (SvPOK(sv)) {
-	    SV *rsv = newSViv(SvUTF8(sv)); 
-	    sv_2mortal(rsv);
+	    SV *rsv = newSViv(SvUTF8(sv));
 	    RETVAL = rsv;
 	    SvUTF8_off(sv);
 	  } else {
@@ -230,16 +777,13 @@ _off_utf8(sv)
       OUTPUT:
 	RETVAL
 
-SV *
-_utf_to_utf(sv, from, to, ...)
-	SV *	sv
-	SV *	from
-	SV *	to
-      CODE:
-	{
-	  SV * check = items == 4 ? ST(3) : Nullsv;
-	  RETVAL = &PL_sv_undef;
-	}
-      OUTPUT:
-	RETVAL
-
+BOOT:
+{
+#if defined(USE_PERLIO) && !defined(USE_SFIO)
+ PerlIO_define_layer(aTHX_ &PerlIO_encode);
+#endif
+/* #include "8859_def.h" */
+/* #include "EBCDIC_def.h" */
+/* #include "Symbols_def.h" */
+#include "def_t_def.h"
+}