From: Rafael Garcia-Suarez Date: Thu, 1 Jun 2006 14:50:50 +0000 (+0000) Subject: Add the perlunitut manpage, by Juerd Waalboer X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=aadaa45530a169c37ceb95c3ed76644ccbd2a644;p=p5sagit%2Fp5-mst-13.2.git Add the perlunitut manpage, by Juerd Waalboer p4raw-id: //depot/perl@28345 --- diff --git a/MANIFEST b/MANIFEST index 715a4f7..2e9ae71 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2963,6 +2963,7 @@ pod/perltoot.pod Perl OO tutorial, part 1 pod/perltrap.pod Perl traps for the unwary pod/perlunicode.pod Perl Unicode support pod/perluniintro.pod Perl Unicode introduction +pod/perlunitut.pod Perl Unicode tutorial pod/perlutil.pod utilities packaged with the Perl distribution pod/perlvar.pod Perl predefined variables pod/perlxs.pod Perl XS application programming interface diff --git a/pod.lst b/pod.lst index e3fb860..57014e0 100644 --- a/pod.lst +++ b/pod.lst @@ -78,6 +78,7 @@ h Reference Manual perllocale Perl locale support perluniintro Perl Unicode introduction perlunicode Perl Unicode support + perlunitut Perl Unicode tutorial perlebcdic Considerations for running Perl on EBCDIC platforms perlsec Perl security diff --git a/pod/perl.pod b/pod/perl.pod index c722d18..e00a758 100644 --- a/pod/perl.pod +++ b/pod/perl.pod @@ -95,6 +95,7 @@ For ease of access, the Perl manual has been split up into several sections. perllocale Perl locale support perluniintro Perl Unicode introduction perlunicode Perl Unicode support + perlunitut Perl Unicode tutorial perlebcdic Considerations for running Perl on EBCDIC platforms perlsec Perl security diff --git a/pod/perltoc.pod b/pod/perltoc.pod index d183bfb..1439bd9 100644 --- a/pod/perltoc.pod +++ b/pod/perltoc.pod @@ -3818,6 +3818,34 @@ autodetected, C needed to upgrade non-Latin-1 byte strings =back +=head2 perlunitut - Perl Unicode Tutorial + +=over 4 + +=item DESCRIPTION + +=over 4 + +=item Definitions + +=item Your new toolkit + +=item I/O flow (the actual 5 minute tutorial) + +=item Q and A + +=back + +=item SUMMARY + +=item ACKNOWLEDGEMENTS + +=item AUTHOR + +=item SEE ALSO + +=back + =head2 perlebcdic - Considerations for running Perl on EBCDIC platforms =over 4 @@ -5336,7 +5364,7 @@ PL_sv_undef X, PL_sv_yes X GvSV X, gv_const_sv X, gv_fetchmeth X, gv_fetchmethod_autoload X, gv_fetchmeth_autoload X, gv_stashpv X, gv_stashpvn -X, gv_stashsv X +X, gv_stashpvs X, gv_stashsv X =item Handy Values @@ -5351,13 +5379,13 @@ X, HeSVKEY_set X, HeVAL X, HvNAME X, hv_assert X, hv_clear X, hv_clear_placeholders X, hv_delete X, hv_delete_ent X, hv_exists X, hv_exists_ent -X, hv_fetch X, hv_fetch_ent X, -hv_iterinit X, hv_iterkey X, hv_iterkeysv -X, hv_iternext X, hv_iternextsv -X, hv_iternext_flags X, hv_iterval -X, hv_magic X, hv_scalar X, hv_store -X, hv_store_ent X, hv_undef X, newHV -X +X, hv_fetch X, hv_fetchs X, +hv_fetch_ent X, hv_iterinit X, hv_iterkey +X, hv_iterkeysv X, hv_iternext X, +hv_iternextsv X, hv_iternext_flags X, +hv_iterval X, hv_magic X, hv_scalar X, +hv_store X, hv_stores X, hv_store_ent X, +hv_undef X, newHV X =item Magical Functions @@ -5372,11 +5400,11 @@ X =item Memory Management Copy X, CopyD X, Move X, MoveD X, Newx X, -Newxc X, Newxz X, Poison X, PoisonNew X, -PoisonWith X, Renew X, Renewc X, Safefree -X, savepv X, savepvn X, savesharedpv -X, savesvpv X, StructCopy X, Zero -X, ZeroD X +Newxc X, Newxz X, Poison X, PoisonFree X, +PoisonNew X, PoisonWith X, Renew X, Renewc +X, Safefree X, savepv X, savepvn X, +savepvs X, savesharedpv X, savesvpv X, +StructCopy X, Zero X, ZeroD X =item Miscellaneous Functions @@ -5464,61 +5492,64 @@ X, SvPV_nomg X, SvPV_set X, SvREFCNT X, SvREFCNT_dec X, SvREFCNT_inc X, SvREFCNT_inc_NN X, SvREFCNT_inc_simple X, SvREFCNT_inc_simple_NN X, -SvREFCNT_inc_simple_void X, SvREFCNT_inc_void -X, SvREFCNT_inc_void_NN X, SvROK -X, SvROK_off X, SvROK_on X, SvRV X, -SvRV_set X, SvSTASH X, SvSTASH_set X, -SvTAINT X, SvTAINTED X, SvTAINTED_off X, -SvTAINTED_on X, SvTRUE X, SvTYPE X, SvUOK -X, SvUPGRADE X, SvUTF8 X, SvUTF8_off -X, SvUTF8_on X, SvUV X, SvUVX X, SvUVx -X, SvUV_nomg X, SvUV_set X, SvVOK X, -sv_catpvn_nomg X, sv_catsv_nomg X, -sv_derived_from X, sv_report_used X, -sv_setsv_nomg X +SvREFCNT_inc_simple_void X, +SvREFCNT_inc_simple_void_NN X, +SvREFCNT_inc_void X, SvREFCNT_inc_void_NN +X, SvROK X, SvROK_off X, SvROK_on +X, SvRV X, SvRV_set X, SvSTASH X, +SvSTASH_set X, SvTAINT X, SvTAINTED X, +SvTAINTED_off X, SvTAINTED_on X, SvTRUE +X, SvTYPE X, SvUOK X, SvUPGRADE X, SvUTF8 +X, SvUTF8_off X, SvUTF8_on X, SvUV X, +SvUVX X, SvUVx X, SvUV_nomg X, SvUV_set +X, SvVOK X, sv_catpvn_nomg X, +sv_catsv_nomg X, sv_derived_from X, +sv_report_used X, sv_setsv_nomg X =item SV-Body Allocation looks_like_number X, newRV_noinc X, newSV X, newSVhek X, newSViv X, newSVnv X, newSVpv X, newSVpvf X, newSVpvn X, -newSVpvn_share X, newSVrv X, newSVsv X, -newSVuv X, sv_2bool X, sv_2cv X, sv_2io -X, sv_2iv_flags X, sv_2mortal X, sv_2nv -X, sv_2pvbyte X, sv_2pvutf8 X, sv_2pv_flags +newSVpvn_share X, newSVpvs X, newSVpvs_share +X, newSVrv X, newSVsv X, newSVuv +X, sv_2bool X, sv_2cv X, sv_2io X, +sv_2iv_flags X, sv_2mortal X, sv_2nv X, +sv_2pvbyte X, sv_2pvutf8 X, sv_2pv_flags X, sv_2uv_flags X, sv_backoff X, sv_bless X, sv_catpv X, sv_catpvf X, sv_catpvf_mg X, sv_catpvn X, sv_catpvn_flags -X, sv_catpv_mg X, sv_catsv X, -sv_catsv_flags X, sv_chop X, sv_clear X, -sv_cmp X, sv_cmp_locale X, sv_collxfrm -X, sv_copypv X, sv_dec X, sv_eq X, -sv_force_normal_flags X, sv_free X, sv_gets -X, sv_grow X, sv_inc X, sv_insert X, -sv_isa X, sv_isobject X, sv_len X, sv_len_utf8 -X, sv_magic X, sv_magicext X, -sv_mortalcopy X, sv_newmortal X, sv_newref -X, sv_pos_b2u X, sv_pos_u2b X, -sv_pvbyten_force X, sv_pvn_force X, -sv_pvn_force_flags X, sv_pvutf8n_force -X, sv_reftype X, sv_replace X, -sv_reset X, sv_rvweaken X, sv_setiv X, -sv_setiv_mg X, sv_setnv X, sv_setnv_mg +X, sv_catpvs X, sv_catpv_mg X, +sv_catsv X, sv_catsv_flags X, sv_chop X, +sv_clear X, sv_cmp X, sv_cmp_locale X, +sv_collxfrm X, sv_copypv X, sv_dec X, sv_eq +X, sv_force_normal_flags X, sv_free +X, sv_gets X, sv_grow X, sv_inc X, +sv_insert X, sv_isa X, sv_isobject X, +sv_len X, sv_len_utf8 X, sv_magic X, +sv_magicext X, sv_mortalcopy X, sv_newmortal +X, sv_newref X, sv_pos_b2u X, +sv_pos_u2b X, sv_pvbyten_force X, +sv_pvn_force X, sv_pvn_force_flags X, +sv_pvutf8n_force X, sv_reftype X, sv_replace +X, sv_reset X, sv_rvweaken X, sv_setiv +X, sv_setiv_mg X, sv_setnv X, sv_setnv_mg X, sv_setpv X, sv_setpvf X, sv_setpvf_mg X, sv_setpviv X, sv_setpviv_mg X, -sv_setpvn X, sv_setpvn_mg X, sv_setpv_mg -X, sv_setref_iv X, sv_setref_nv X, -sv_setref_pv X, sv_setref_pvn X, sv_setref_uv -X, sv_setsv X, sv_setsv_flags X, -sv_setsv_mg X, sv_setuv X, sv_setuv_mg -X, sv_tainted X, sv_true X, sv_unmagic -X, sv_unref_flags X, sv_untaint X, -sv_upgrade X, sv_usepvn_flags X, -sv_utf8_decode X, sv_utf8_downgrade X, -sv_utf8_encode X, sv_utf8_upgrade X, -sv_utf8_upgrade_flags X, sv_vcatpvf X, -sv_vcatpvfn X, sv_vcatpvf_mg X, sv_vsetpvf -X, sv_vsetpvfn X, sv_vsetpvf_mg X +sv_setpvn X, sv_setpvn_mg X, sv_setpvs +X, sv_setpv_mg X, sv_setref_iv X, +sv_setref_nv X, sv_setref_pv X, sv_setref_pvn +X, sv_setref_uv X, sv_setsv X, +sv_setsv_flags X, sv_setsv_mg X, sv_setuv +X, sv_setuv_mg X, sv_tainted X, sv_true +X, sv_unmagic X, sv_unref_flags X, +sv_untaint X, sv_upgrade X, sv_usepvn_flags +X, sv_utf8_decode X, sv_utf8_downgrade +X, sv_utf8_encode X, sv_utf8_upgrade +X, sv_utf8_upgrade_flags X, +sv_vcatpvf X, sv_vcatpvfn X, sv_vcatpvf_mg +X, sv_vsetpvf X, sv_vsetpvfn X, +sv_vsetpvf_mg X =item Unicode Support @@ -10597,6 +10628,16 @@ threads->detach(), threads->self(), $thr->tid(), threads->tid(), threads->object($tid), threads->yield(), threads->list(), $thr1->equal($thr2), async BLOCK;, $thr->_handle(), threads->_handle() +=item THREAD CONTEXT + +=over 4 + +=item Explicit context + +=item Implicit context + +=back + =item THREAD STACK SIZE threads->get_stack_size();, $size = $thr->get_stack_size();, $old_size = @@ -12160,7 +12201,7 @@ redoop, nextop, lastop =item B::COP Methods -label, stash, stashpv, file, cop_seq, arybase, line, warnings, io +label, stash, stashpv, file, cop_seq, arybase, line, warnings, io, hints =back @@ -12358,10 +12399,11 @@ B<~> B<#>I, B<#>II, B<#>I, B<#addr>, B<#arg>, B<#class>, B<#classsym>, B<#coplabel>, B<#exname>, B<#extarg>, B<#firstaddr>, -B<#flags>, B<#flagval>, B<#hyphseq>, B<#label>, B<#lastaddr>, B<#name>, -B<#NAME>, B<#next>, B<#nextaddr>, B<#noise>, B<#private>, B<#privval>, -B<#seq>, B<#seqnum>, B<#opt>, B<#static>, B<#sibaddr>, B<#svaddr>, -B<#svclass>, B<#svval>, B<#targ>, B<#targarg>, B<#targarglife>, B<#typenum> +B<#flags>, B<#flagval>, B<#hints>, B<#hintsval>, B<#hyphseq>, B<#label>, +B<#lastaddr>, B<#name>, B<#NAME>, B<#next>, B<#nextaddr>, B<#noise>, +B<#private>, B<#privval>, B<#seq>, B<#seqnum>, B<#opt>, B<#static>, +B<#sibaddr>, B<#svaddr>, B<#svclass>, B<#svval>, B<#targ>, B<#targarg>, +B<#targarglife>, B<#typenum> =back @@ -12469,8 +12511,8 @@ strict, $[, bytes, utf8, integer, re, warnings, hint_bits, warning_bits =item OPTIONS AND LINT CHECKS -B, B and B, B, -B, B, B, +B, B, B and B, +B, B, B, B, B, B, B =item NON LINT-CHECK OPTIONS @@ -12479,6 +12521,11 @@ B<-u Package> =item EXTENDING LINT +=item TODO + +while() stomps $_, strict oo, unchecked system calls, more tests, +validate against older perls + =item BUGS =item AUTHOR @@ -15006,40 +15053,39 @@ C, C, C, C, C, C, C, C, C, C, C, C, C, -C, C, C, C, C, -C, C, C, C, C, -C, C, C, C, C, -C, C, C, C, -C, C, C, C, -C, C, C, C, C, -C, C, C, C, C, -C, C, C, C, -C, C, C, C, -C, C, C, C, -C, C, C, C, C, -C, C, C, C, C, -C, C, C, C, C, -C, C, C, C, C, -C, C, C, C, -C, C, C, C, C, -C, C, C, C, C, -C, C, C, C, C, -C, C, C, C, -C, C, C, C, -C, C, C, -C, C, C, C, -C, C, C, C, -C, C, C, C, -C, C, C, C, -C, C, C, C, C, -C, C, C, -C, C, C, -C, C, C, C, -C, C, C, C, -C, C, C, -C, C, C, C, -C, C, C, C, C, -C, C, C, C, +C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, +C, C, C, C, +C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, +C, C, C, C, +C, C, C, C, +C, C, C, C, +C, C, C, +C, C, C, C, +C, C, C, C, +C, C, C, C, +C, C, C, C, +C, C, C, C, C, +C, C, C, +C, C, C, C, +C, C, C, C, +C, C, C, C, +C, C, C, C, +C, C, C, C, C, +C, C, C, C, C, C, C, C, C, C, C, C, C, C, C, C, C, C, @@ -15701,32 +15747,6 @@ Dumper =back -=head2 Devel::Arena - Perl extension for inspecting the core's arena -structures - -=over 4 - -=item SYNOPSIS - -=item DESCRIPTION - -=over 4 - -=item EXPORT - -sv_stats [DONT_SHARE], shared_string_table, sizes, HEK_size STRING, -shared_string_table_effectiveness, write_stats_at_END - -=back - -=item SEE ALSO - -=item AUTHOR - -=item COPYRIGHT AND LICENSE - -=back - =head2 Devel::DProf - a Perl code profiler =over 4 @@ -15781,9 +15801,9 @@ shared_string_table_effectiveness, write_stats_at_END =item Perl API not supported by ppport.h -perl 5.9.3, perl 5.9.2, perl 5.9.1, perl 5.9.0, perl 5.8.3, perl 5.8.1, -perl 5.8.0, perl 5.7.3, perl 5.7.2, perl 5.7.1, perl 5.6.1, perl 5.6.0, -perl 5.005_03, perl 5.005, perl 5.004_05, perl 5.004 +perl 5.9.4, perl 5.9.3, perl 5.9.2, perl 5.9.1, perl 5.9.0, perl 5.8.3, +perl 5.8.1, perl 5.8.0, perl 5.7.3, perl 5.7.2, perl 5.7.1, perl 5.6.1, +perl 5.6.0, perl 5.005_03, perl 5.005, perl 5.004_05, perl 5.004 =back @@ -15857,59 +15877,6 @@ perl 5.005_03, perl 5.005, perl 5.004_05, perl 5.004 =back -=head2 Devel::Size - Perl extension for finding the memory usage of Perl -variables - -=over 4 - -=item SYNOPSIS - -=item DESCRIPTION - -=item FUNCTIONS - -=over 4 - -=item size($ref) - -=item total_size($ref) - -=back - -=item EXPORT - -=item UNDERSTANDING MEMORY ALLOCATION - -=over 4 - -=item The C library - -=item Perl - -=back - -=item DANGERS - -=item Messages: texts originating from this module. - -=over 4 - -=item Errors - -=item warnings - -=back - -=item BUGS - -=item AUTHOR - -=item COPYRIGHT - -=item SEE ALSO - -=back - =head2 Digest - Modules that calculate message digests =over 4 @@ -15990,15 +15957,16 @@ B, B, B, B, B, B, B, B, B, B, B, B, B, -B, B, B, B, -B, B, B, -B, B, -B, B, -B, B, -B, B, -B, B, -B, B, -B +B, B<"b"> read file in binary mode, +B<"p"> use portable mode, B, B, +B, B, B, B, +B, B, +B, B, +B, B, +B, B, +B, B, +B, B, +B, B =item SEE ALSO @@ -26222,14 +26190,14 @@ path not available =item SEE ALSO -=item AUTHOR +=item AUTHORS =item BUGS =item SUPPORT AnnoCPAN: Annotated CPAN documentation, CPAN Ratings, RT: CPAN's request -tracker, Search CPAN +tracker, Search CPAN, Kobes' CPAN Search, Perl Documentation =item LICENSE @@ -26281,14 +26249,14 @@ path not available =item SEE ALSO -=item AUTHOR +=item AUTHORS =item BUGS =item SUPPORT AnnoCPAN: Annotated CPAN documentation, CPAN Ratings, RT: CPAN's request -tracker, Search CPAN +tracker, Search CPAN, Kobes' CPAN Search, Perl Documentation =item LICENSE diff --git a/pod/perlunitut.pod b/pod/perlunitut.pod new file mode 100644 index 0000000..ae8d0b1 --- /dev/null +++ b/pod/perlunitut.pod @@ -0,0 +1,425 @@ +=head1 NAME + +perlunitut - Perl Unicode Tutorial + +=head1 DESCRIPTION + +The days of just flinging strings around are over. It's well established that +modern programs need to be capable of communicating funny accented letters, and +things like euro symbols. This means that programmers need new habits. It's +easy to program Unicode capable software, but it does require discipline to do +it right. + +There's a lot to know about character sets, and text encodings. It's probably +best to spend a full day learning all this, but the basics can be learned in +minutes. + +These are not the very basics, though. It is assumed that you already +know the difference between bytes and characters, and realise (and accept!) +that there are many different character sets and encodings, and that your +program has to be explicit about them. Recommended reading is "The Absolute +Minimum Every Software Developer Absolutely, Positively Must Know About Unicode +and Character Sets (No Excuses!)" by Joel Spolsky, at +L. + +This tutorial speaks in rather absolute terms, and provides only a limited view +of the wealth of character string related features that Perl has to offer. For +most projects, this information will probably suffice. + +=head2 Definitions + +It's important to set a few things straight first. This is the most important +part of this tutorial. This view may conflict with other information that you +may have found on the web, but that's mostly because many sources are wrong. + +You may have to re-read this entire section a few times... + +=head3 Unicode + +B is a character set with room for lots of characters. The ordinal +value of a character is called a B. + +There are many, many code points, but computers work with bytes, and a byte can +have only 256 values. Unicode has many more characters, so you need a method +to make these accessible. + +Unicode is encoded using several competing encodings, of which UTF-8 is the +most used. In a Unicode encoding, multiple subsequent bytes can be used to +store a single code point, or simply: character. + +=head3 UTF-8 + +B is a Unicode encoding. Many people think that Unicode and UTF-8 are +the same thing, but they're not. There are more Unicode encodings, but much of +the world has standardized on UTF-8. + +UTF-8 treats the first 128 codepoints, 0..127, the same as ASCII. They take +only one byte per character. All other characters are encoded as two or more +(up to six) bytes using a complex scheme. Fortunately, Perl handles this for +us, so we don't have to worry about this. + +=head3 Text strings (character strings) + +B, or B are made of characters. Bytes are +irrelevant here, and so are encodings. Each character is just that: the +character. + +On a text string, you would do things like: + + $text =~ s/foo/bar/; + if ($string =~ /^\d+$/) { ... } + $text = ucfirst $text; + my $character_count = length $text; + +The value of a character (C, C) is the corresponding Unicode code +point. + +=head3 Binary strings (byte strings) + +B, or B are made of bytes. Here, you don't have +characters, just bytes. All communication with the outside world (anything +outside of your current Perl process) is done in binary. + +On a binary string, you would do things like: + + my (@length_content) = unpack "(V/a)*", $binary; + $binary =~ s/\x00\x0F/\xFF\xF0/; # for the brave :) + print {$fh} $binary; + my $byte_count = length $binary; + +=head3 Encoding + +B (as a verb) is the conversion from I to I. To encode, +you have to supply the target encoding, for example C or C. +Some encodings, like the C ("latin") range, do not support the full +Unicode standard; characters that can't be represented are lost in the +conversion. + +=head3 Decoding + +B is the conversion from I to I. To decode, you have to +know what encoding was used during the encoding phase. And most of all, it must +be something decodable. It doesn't make much sense to decode a PNG image into a +text string. + +=head3 Internal format + +Perl has an B, an encoding that it uses to encode text strings +so it can store them in memory. All text strings are in this internal format. +In fact, text strings are never in any other format! + +You shouldn't worry about what this format is, because conversion is +automatically done when you decode or encode. + +=head2 Your new toolkit + +Add to your standard heading the following line: + + use Encode qw(encode decode); + +Or, if you're lazy, just: + + use Encode; + +=head2 I/O flow (the actual 5 minute tutorial) + +The typical input/output flow of a program is: + + 1. Receive and decode + 2. Process + 3. Encode and output + +If your input is binary, and is supposed to remain binary, you shouldn't decode +it to a text string, of course. But in all other cases, you should decode it. + +Decoding can't happen reliably if you don't know how the data was encoded. If +you get to choose, it's a good idea to standardize on UTF-8. + + my $foo = decode('UTF-8', get 'http://example.com/'); + my $bar = decode('ISO-8859-1', readline STDIN); + my $xyzzy = decode('Windows-1251', $cgi->param('foo')); + +Processing happens as you knew before. The only difference is that you're now +using characters instead of bytes. That's very useful if you use things like +C, or C. + +It's important to realize that there are no bytes in a text string. Of course, +Perl has its internal encoding to store the string in memory, but ignore that. +If you have to do anything with the number of bytes, it's probably best to move +that part to step 3, just after you've encoded the string. Then you know +exactly how many bytes it will be in the destination string. + +The syntax for encoding text strings to binary strings is as simple as decoding: + + $body = encode('UTF-8', $body); + +If you needed to know the length of the string in bytes, now's the perfect time +for that. Because C<$body> is now a byte string, C will report the +number of bytes, instead of the number of characters. The number of +characters is no longer known, because characters only exist in text strings. + + my $byte_count = length $body; + +And if the protocol you're using supports a way of letting the recipient know +which character encoding you used, please help the receiving end by using that +feature! For example, E-mail and HTTP support MIME headers, so you can use the +C header. They can also have C to indicate the +number of I, which is always a good idea to supply if the number is +known. + + "Content-Type: text/plain; charset=UTF-8", + "Content-Length: $byte_count" + +=head2 Q and A + +=head3 This isn't really a Unicode tutorial, is it? + +No, Perl has an abstracted interface for all supported character encodings, so +this is actually a generic C tutorial. But many people think that +Unicode is special and magical, and I didn't want to disappoint them, so I +decided to call this document a Unicode tutorial. + +=head3 What about binary data, like images? + +Well, apart from a bare C, you shouldn't treat them specially. +(The binmode is needed because otherwise Perl may convert line endings on Win32 +systems.) + +Be careful, though, to never combine text strings with binary strings. If you +need text in a binary stream, encode your text strings first using the +appropriate encoding, then join them with binary strings. See also: "What if I +don't encode?". + +=head3 What about the UTF-8 flag? + +Please, unless you're hacking the internals, or debugging weirdness, don't +think about the UTF-8 flag at all. That means that you very probably shouldn't +use C, C<_utf8_on> or C<_utf8_off> at all. + +Perl's internal format happens to be UTF-8. Unfortunately, Perl can't keep a +secret, so everyone knows about this. That is the source of much confusion. +It's better to pretend that the internal format is some unknown encoding, +and that you always have to encode and decode explicitly. + +=head3 When should I decode or encode? + +Whenever you're communicating with anything that is external to your perl +process, like a database, a text file, a socket, or another program. Even if +the thing you're communicating with is also written in Perl. + +=head3 What if I don't decode? + +Whenever your encoded, binary string is used together with a text string, Perl +will assume that your binary string was encoded with ISO-8859-1, also known as +latin-1. If it wasn't latin-1, then your data is unpleasantly converted. For +example, if it was UTF-8, the individual bytes of multibyte characters are seen +as separate characters, and then again converted to UTF-8. Such double encoding +can be compared to double HTML encoding (C<&gt;>), or double URI encoding +(C<%253E>). + +This silent implicit decoding is known as "upgrading". That may sound +positive, but it's best to avoid it. + +=head3 What if I don't encode? + +Your text string will be sent using the bytes in Perl's internal format. In +some cases, Perl will warn you that you're doing something wrong, with a +friendly warning: + + Wide character in print at example.pl line 2. + +Because the internal format is often UTF-8, these bugs are hard to spot, +because UTF-8 is usually the encoding you wanted! But don't be lazy, and don't +use the fact that Perl's internal format is UTF-8 to your advantage. Encode +explicitly to avoid weird bugs, and to show to maintenance programmers that you +thought this through. + +=head3 Is there a way to automatically decode or encode? + +If all data that comes from a certain handle is encoded in exactly the same +way, you can tell the PerlIO system to automatically decode everything, with +the C layer. If you do this, you can't accidentally forget to decode +or encode anymore, on things that use the layered handle. + +You can provide this layer when Cing the file: + + open my $fh, '>:encoding(UTF-8)', $filename; # auto encoding on write + open my $fh, '<:encoding(UTF-8)', $filename; # auto decoding on read + +Or if you already have an open filehandle: + + binmode $fh, ':encoding(UTF-8)'; + +Some database drivers for DBI can also automatically encode and decode, but +that is typically limited to the UTF-8 encoding, because they cheat. + +=head3 Cheat?! Tell me, how can I cheat? + +Well, because Perl's internal format is UTF-8, you can just skip the encoding +or decoding step, and manipulate the UTF-8 flag directly. + +Instead of C<:encoding(UTF-8)>, you can simply use C<:utf8>. This is widely +accepted as good behavior. + +Instead of C and C, you could use C<_utf8_on> and C<_utf8_off>. +But this is, contrary to C<:utf8>, considered bad style. + +There are some shortcuts for oneliners; see C<-C> in L. + +=head3 What if I don't know which encoding was used? + +Do whatever you can to find out, and if you have to: guess. (Don't forget to +document your guess with a comment.) + +You could open the document in a web browser, and change the character set or +character encoding until you can visually confirm that all characters look the +way they should. + +There is no way to reliably detect the encoding automatically, so if people +keep sending you data without charset indication, you may have to educate them. + +=head3 Can I use Unicode in my Perl sources? + +Yes, you can! If your sources are UTF-8 encoded, you can indicate that with the +C pragma. + + use utf8; + +This doesn't do anything to your input, or to your output. It only influences +the way your sources are read. You can use Unicode in string literals, in +identifiers (but they still have to be "word characters" according to C<\w>), +and even in custom delimiters. + +=head3 Data::Dumper doesn't restore the UTF-8 flag; is it broken? + +No, Data::Dumper's Unicode abilities are as they should be. There have been +some complaints that it should restore the UTF-8 flag when the data is read +again with C. However, you should really not look at the flag, and +nothing indicates that Data::Dumper should break this rule. + +Here's what happens: when Perl reads in a string literal, it sticks to 8 bit +encoding as long as it can. (But perhaps originally it was internally encoded +as UTF-8, when you dumped it.) When it has to give that up because other +characters are added to the text string, it silently upgrades the string to +UTF-8. + +If you properly encode your strings for output, none of this is of your +concern, and you can just C dumped data as always. + +=head3 How can I determine if a string is a text string or a binary string? + +You can't. Some use the UTF-8 flag for this, but that's misuse, and makes well +behaved modules like Data::Dumper look bad. The flag is useless for this +purpose, because it's off when an 8 bit encoding (by default ISO-8859-1) is +used to store the string. + +This is something you, the programmer, has to keep track of; sorry. You could +consider adopting a kind of "Hungarian notation" to help with this. + +=head3 How do I convert from encoding FOO to encoding BAR? + +By first converting the FOO-encoded byte string to a text string, and then the +text string to a BAR-encoded byte string: + + my $text_string = decode('FOO', $foo_string); + my $bar_string = encode('BAR', $text_string); + +or by skipping the text string part, and going directly from one binary +encoding to the other: + + use Encode qw(from_to); + from_to($string, 'FOO', 'BAR'); # changes contents of $string + +or by letting automatic decoding and encoding do all the work: + + open my $foofh, '<:encoding(FOO)', 'example.foo.txt'; + open my $barfh, '>:encoding(BAR)', 'example.bar.txt'; + print { $barfh } $_ while <$foofh>; + +=head3 What about the C pragma? + +Don't use it. It makes no sense to deal with bytes in a text string, and it +makes no sense to deal with characters in a byte string. Do the proper +conversions (by decoding/encoding), and things will work out well: you get +character counts for decoded data, and byte counts for encoded data. + +C is usually a failed attempt to do something useful. Just forget +about it. + +=head3 What are C and C? + +These are alternate syntaxes for C and C. + +=head3 What's the difference between C and C? + +C is the official standard. C is Perl's way of being liberal in +what it accepts. If you have to communicate with things that aren't so liberal, +you may want to consider using C. If you have to communicate with things +that are too liberal, you may have to use C. The full explanation is in +L. + +C is internally known as C. This tutorial uses UTF-8 +consistently, even where utf8 is actually used internally, because the +distinction can be hard to make, and is mostly irrelevant. + +Okay, if you insist: the "internal format" is utf8, not UTF-8. (When it's not +some other encoding.) + +=head3 I lost track; what encoding is the internal format really? + +It's good that you lost track, because you shouldn't depend on the internal +format being any specific encoding. But since you asked: by default, the +internal format is either ISO-8859-1 (latin-1), or utf8, depending on the +history of the string. + +Perl knows how it stored the string internally, and will use that knowledge +when you C. In other words: don't try to find out what the internal +encoding for a certain string is, but instead just encode it into the encoding +that you want. + +=head3 What character encodings does Perl support? + +To find out which character encodings your Perl supports, run: + + perl -MEncode -le "print for Encode->encodings(':all')" + +=head3 Which version of perl should I use? + +Well, if you can, upgrade to the most recent, but certainly C<5.8.1> or newer. +This tutorial is based on the status quo as of C<5.8.7>. + +You should also check your modules, and upgrade them if necessary. For example, +HTML::Entities requires version >= 1.32 to function correctly, even though the +changelog is silent about this. + +=head1 SUMMARY + +Decode everything you receive, encode everything you send out. (If it's text +data.) + +=head1 ACKNOWLEDGEMENTS + +Thanks to Johan Vromans from Squirrel Consultancy. His UTF-8 rants during the +Amsterdam Perl Mongers meetings got me interested and determined to find out +how to use character encodings in Perl in ways that don't break easily. + +Thanks to Gerard Goossen from TTY. His presentation "UTF-8 in the wild" (Dutch +Perl Workshop 2006) inspired me to publish my thoughts and write this tutorial. + +Thanks to the people who asked about this kind of stuff in several Perl IRC +channels, and have constantly reminded me that a simpler explanation was +needed. + +Thanks to the people who reviewed this document for me, before it went public. +They are: Benjamin Smith, Jan-Pieter Cornet, Johan Vromans, Lukas Mai, Nathan +Gray. + +=head1 AUTHOR + +Juerd Waalboer + +=head1 SEE ALSO + +L, L, L + diff --git a/vms/descrip_mms.template b/vms/descrip_mms.template index 6a5bc9b..29c1e44 100644 --- a/vms/descrip_mms.template +++ b/vms/descrip_mms.template @@ -411,9 +411,10 @@ pod21 = [.lib.pods]perlpragma.pod [.lib.pods]perlqnx.pod [.lib.pods]perlre.pod [ pod22 = [.lib.pods]perlreref.pod [.lib.pods]perlretut.pod [.lib.pods]perlriscos.pod [.lib.pods]perlrun.pod [.lib.pods]perlsec.pod [.lib.pods]perlsolaris.pod pod23 = [.lib.pods]perlstyle.pod [.lib.pods]perlsub.pod [.lib.pods]perlsymbian.pod [.lib.pods]perlsyn.pod [.lib.pods]perlthrtut.pod [.lib.pods]perltie.pod pod24 = [.lib.pods]perltoc.pod [.lib.pods]perltodo.pod [.lib.pods]perltooc.pod [.lib.pods]perltoot.pod [.lib.pods]perltrap.pod [.lib.pods]perltru64.pod -pod25 = [.lib.pods]perltw.pod [.lib.pods]perlunicode.pod [.lib.pods]perluniintro.pod [.lib.pods]perlutil.pod [.lib.pods]perluts.pod [.lib.pods]perlvar.pod -pod26 = [.lib.pods]perlvmesa.pod [.lib.pods]perlvms.pod [.lib.pods]perlvos.pod [.lib.pods]perlwin32.pod [.lib.pods]perlxs.pod [.lib.pods]perlxstut.pod -pod = $(pod0) $(pod1) $(pod2) $(pod3) $(pod4) $(pod5) $(pod6) $(pod7) $(pod8) $(pod9) $(pod10) $(pod11) $(pod12) $(pod13) $(pod14) $(pod15) $(pod16) $(pod17) $(pod18) $(pod19) $(pod20) $(pod21) $(pod22) $(pod23) $(pod24) $(pod25) $(pod26) +pod25 = [.lib.pods]perltw.pod [.lib.pods]perlunicode.pod [.lib.pods]perluniintro.pod [.lib.pods]perlunitut.pod [.lib.pods]perlutil.pod [.lib.pods]perluts.pod +pod26 = [.lib.pods]perlvar.pod [.lib.pods]perlvmesa.pod [.lib.pods]perlvms.pod [.lib.pods]perlvos.pod [.lib.pods]perlwin32.pod [.lib.pods]perlxs.pod +pod27 = [.lib.pods]perlxstut.pod +pod = $(pod0) $(pod1) $(pod2) $(pod3) $(pod4) $(pod5) $(pod6) $(pod7) $(pod8) $(pod9) $(pod10) $(pod11) $(pod12) $(pod13) $(pod14) $(pod15) $(pod16) $(pod17) $(pod18) $(pod19) $(pod20) $(pod21) $(pod22) $(pod23) $(pod24) $(pod25) $(pod26) $(pod27) # Would be useful to automate the generation of this rule from pod/buildtoc # Plus its corresponding delete in the clean target. @@ -1254,6 +1255,10 @@ preplibrary : $(MINIPERL_EXE) $(LIBPREREQ) @ If F$Search("[.lib]pods.dir").eqs."" Then Create/Directory [.lib.pods] Copy/NoConfirm/Log $(MMS$SOURCE) [.lib.pods] +[.lib.pods]perlunitut.pod : [.pod]perlunitut.pod + @ If F$Search("[.lib]pods.dir").eqs."" Then Create/Directory [.lib.pods] + Copy/NoConfirm/Log $(MMS$SOURCE) [.lib.pods] + [.lib.pods]perlutil.pod : [.pod]perlutil.pod @ If F$Search("[.lib]pods.dir").eqs."" Then Create/Directory [.lib.pods] Copy/NoConfirm/Log $(MMS$SOURCE) [.lib.pods] diff --git a/win32/pod.mak b/win32/pod.mak index 1993d71..5f3bf61 100644 --- a/win32/pod.mak +++ b/win32/pod.mak @@ -120,6 +120,7 @@ POD = \ perltrap.pod \ perlunicode.pod \ perluniintro.pod \ + perlunitut.pod \ perlutil.pod \ perlvar.pod \ perlxs.pod \