--- /dev/null
+ <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+ <html xmlns="http://www.w3.org/1999/xhtml">
+<!--
+This file was generated by Devel::NYTProf version 4.06
+-->
+<head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+ <meta http-equiv="Content-Language" content="en-us" />
+ <title>Profile of PPI/Tokenizer.pm</title>
+<link rel="stylesheet" type="text/css" href="style.css" />
+ <script type="text/javascript" src="js/jquery-min.js"></script>
+
+ <script type="text/javascript" src="js/jquery-tablesorter-min.js"></script>
+ <link rel="stylesheet" type="text/css" href="js/style-tablesorter.css" />
+ <script type="text/javascript">
+ // when a column is first clicked on to sort it, use descending order
+ // XXX doesn't seem to work (and not just because the tablesorter formatSortingOrder() is broken)
+ $.tablesorter.defaults.sortInitialOrder = "desc";
+ // add parser through the tablesorter addParser method
+ $.tablesorter.addParser({
+ id: 'fmt_time', // name of this parser
+ is: function(s) {
+ return false; // return false so this parser is not auto detected
+ },
+ format: function(orig) { // format data for normalization
+ // console.log(orig);
+ val = orig.replace(/ns/,'');
+ if (val != orig) { return val / (1000*1000*1000); }
+ val = orig.replace(/µs/,''); /* XXX use µ ? */
+ if (val != orig) { return val / (1000*1000); }
+ var val = orig.replace(/ms/,'');
+ if (val != orig) { return val / (1000); }
+ var val = orig.replace(/s/,'');
+ if (val != orig) { return val; }
+ if (orig == '0') { return orig; }
+ console.log('no match for fmt_time of '.concat(orig));
+ return orig;
+ },
+ type: 'numeric' // set type, either numeric or text
+ });
+ </script>
+</head>
+
+<body >
+<div class="header" style="position: relative; overflow-x: hidden; overflow-y: hidden; z-index: 0; ">
+<div class="header_back">
+ <a href="index.html">← Index</a>
+ </div>
+<div class="headerForeground" style="float: left">
+ <span class="siteTitle">NYTProf Performance Profile</span>
+ <span class="siteSubtitle">  <span>« <span class="mode_btn mode_btn_selected">block view</span> • <span class="mode_btn"><a href="PPI-Tokenizer-pm-1003-line.html">line view</a></span> • <span class="mode_btn"><a href="PPI-Tokenizer-pm-1003-sub.html">sub view</a></span> »</span><br />
+ For script/nytprof.pl
+ </span>
+</div>
+<div class="headerForeground" style="float: right; text-align: right">
+ <span class="siteTitle"> </span>
+ <span class="siteSubtitle">Run on Thu May 31 16:29:39 2012<br />Reported on Thu May 31 16:36:54 2012</span>
+</div>
+<div style="position: absolute; left: 0px; top: 0%; width: 100%; height: 101%; z-index: -1; background-color: rgb(17, 136, 255); "></div>
+<div style="position: absolute; left: 0px; top: 2%; width: 100%; height: 99%; z-index: -1; background-color: rgb(16, 134, 253); "></div>
+<div style="position: absolute; left: 0px; top: 4%; width: 100%; height: 97%; z-index: -1; background-color: rgb(16, 133, 252); "></div>
+<div style="position: absolute; left: 0px; top: 6%; width: 100%; height: 95%; z-index: -1; background-color: rgb(15, 131, 250); "></div>
+<div style="position: absolute; left: 0px; top: 8%; width: 100%; height: 93%; z-index: -1; background-color: rgb(15, 130, 249); "></div>
+<div style="position: absolute; left: 0px; top: 10%; width: 100%; height: 91%; z-index: -1; background-color: rgb(15, 129, 248); "></div>
+<div style="position: absolute; left: 0px; top: 12%; width: 100%; height: 89%; z-index: -1; background-color: rgb(14, 127, 246); "></div>
+<div style="position: absolute; left: 0px; top: 14%; width: 100%; height: 87%; z-index: -1; background-color: rgb(14, 126, 245); "></div>
+<div style="position: absolute; left: 0px; top: 16%; width: 100%; height: 85%; z-index: -1; background-color: rgb(14, 125, 244); "></div>
+<div style="position: absolute; left: 0px; top: 18%; width: 100%; height: 83%; z-index: -1; background-color: rgb(13, 123, 242); "></div>
+<div style="position: absolute; left: 0px; top: 20%; width: 100%; height: 81%; z-index: -1; background-color: rgb(13, 122, 241); "></div>
+<div style="position: absolute; left: 0px; top: 22%; width: 100%; height: 79%; z-index: -1; background-color: rgb(13, 121, 240); "></div>
+<div style="position: absolute; left: 0px; top: 24%; width: 100%; height: 77%; z-index: -1; background-color: rgb(12, 119, 238); "></div>
+<div style="position: absolute; left: 0px; top: 26%; width: 100%; height: 75%; z-index: -1; background-color: rgb(12, 118, 237); "></div>
+<div style="position: absolute; left: 0px; top: 28%; width: 100%; height: 73%; z-index: -1; background-color: rgb(12, 116, 235); "></div>
+<div style="position: absolute; left: 0px; top: 30%; width: 100%; height: 71%; z-index: -1; background-color: rgb(11, 115, 234); "></div>
+<div style="position: absolute; left: 0px; top: 32%; width: 100%; height: 69%; z-index: -1; background-color: rgb(11, 114, 233); "></div>
+<div style="position: absolute; left: 0px; top: 34%; width: 100%; height: 67%; z-index: -1; background-color: rgb(11, 112, 231); "></div>
+<div style="position: absolute; left: 0px; top: 36%; width: 100%; height: 65%; z-index: -1; background-color: rgb(10, 111, 230); "></div>
+<div style="position: absolute; left: 0px; top: 38%; width: 100%; height: 63%; z-index: -1; background-color: rgb(10, 110, 229); "></div>
+<div style="position: absolute; left: 0px; top: 40%; width: 100%; height: 61%; z-index: -1; background-color: rgb(10, 108, 227); "></div>
+<div style="position: absolute; left: 0px; top: 42%; width: 100%; height: 59%; z-index: -1; background-color: rgb(9, 107, 226); "></div>
+<div style="position: absolute; left: 0px; top: 44%; width: 100%; height: 57%; z-index: -1; background-color: rgb(9, 106, 225); "></div>
+<div style="position: absolute; left: 0px; top: 46%; width: 100%; height: 55%; z-index: -1; background-color: rgb(9, 104, 223); "></div>
+<div style="position: absolute; left: 0px; top: 48%; width: 100%; height: 53%; z-index: -1; background-color: rgb(8, 103, 222); "></div>
+<div style="position: absolute; left: 0px; top: 50%; width: 100%; height: 51%; z-index: -1; background-color: rgb(8, 102, 221); "></div>
+<div style="position: absolute; left: 0px; top: 52%; width: 100%; height: 49%; z-index: -1; background-color: rgb(8, 100, 219); "></div>
+<div style="position: absolute; left: 0px; top: 54%; width: 100%; height: 47%; z-index: -1; background-color: rgb(7, 99, 218); "></div>
+<div style="position: absolute; left: 0px; top: 56%; width: 100%; height: 45%; z-index: -1; background-color: rgb(7, 97, 216); "></div>
+<div style="position: absolute; left: 0px; top: 58%; width: 100%; height: 43%; z-index: -1; background-color: rgb(7, 96, 215); "></div>
+<div style="position: absolute; left: 0px; top: 60%; width: 100%; height: 41%; z-index: -1; background-color: rgb(6, 95, 214); "></div>
+<div style="position: absolute; left: 0px; top: 62%; width: 100%; height: 39%; z-index: -1; background-color: rgb(6, 93, 212); "></div>
+<div style="position: absolute; left: 0px; top: 64%; width: 100%; height: 37%; z-index: -1; background-color: rgb(6, 92, 211); "></div>
+<div style="position: absolute; left: 0px; top: 66%; width: 100%; height: 35%; z-index: -1; background-color: rgb(5, 91, 210); "></div>
+<div style="position: absolute; left: 0px; top: 68%; width: 100%; height: 33%; z-index: -1; background-color: rgb(5, 89, 208); "></div>
+<div style="position: absolute; left: 0px; top: 70%; width: 100%; height: 31%; z-index: -1; background-color: rgb(5, 88, 207); "></div>
+<div style="position: absolute; left: 0px; top: 72%; width: 100%; height: 29%; z-index: -1; background-color: rgb(4, 87, 206); "></div>
+<div style="position: absolute; left: 0px; top: 74%; width: 100%; height: 27%; z-index: -1; background-color: rgb(4, 85, 204); "></div>
+<div style="position: absolute; left: 0px; top: 76%; width: 100%; height: 25%; z-index: -1; background-color: rgb(4, 84, 203); "></div>
+<div style="position: absolute; left: 0px; top: 78%; width: 100%; height: 23%; z-index: -1; background-color: rgb(3, 82, 201); "></div>
+<div style="position: absolute; left: 0px; top: 80%; width: 100%; height: 21%; z-index: -1; background-color: rgb(3, 81, 200); "></div>
+<div style="position: absolute; left: 0px; top: 82%; width: 100%; height: 19%; z-index: -1; background-color: rgb(3, 80, 199); "></div>
+<div style="position: absolute; left: 0px; top: 84%; width: 100%; height: 17%; z-index: -1; background-color: rgb(2, 78, 197); "></div>
+<div style="position: absolute; left: 0px; top: 86%; width: 100%; height: 15%; z-index: -1; background-color: rgb(2, 77, 196); "></div>
+<div style="position: absolute; left: 0px; top: 88%; width: 100%; height: 13%; z-index: -1; background-color: rgb(2, 76, 195); "></div>
+<div style="position: absolute; left: 0px; top: 90%; width: 100%; height: 11%; z-index: -1; background-color: rgb(1, 74, 193); "></div>
+<div style="position: absolute; left: 0px; top: 92%; width: 100%; height: 9%; z-index: -1; background-color: rgb(1, 73, 192); "></div>
+<div style="position: absolute; left: 0px; top: 94%; width: 100%; height: 7%; z-index: -1; background-color: rgb(1, 72, 191); "></div>
+<div style="position: absolute; left: 0px; top: 96%; width: 100%; height: 5%; z-index: -1; background-color: rgb(0, 70, 189); "></div>
+<div style="position: absolute; left: 0px; top: 98%; width: 100%; height: 3%; z-index: -1; background-color: rgb(0, 69, 188); "></div>
+<div style="position: absolute; left: 0px; top: 100%; width: 100%; height: 1%; z-index: -1; background-color: rgb(0, 68, 187); "></div>
+</div>
+
+<div class="body_content"><br />
+<table class="file_summary"><tr><td class="h">Filename</td><td align="left"><a href="file:///Users/edenc/perl5/lib/perl5/PPI/Tokenizer.pm">/Users/edenc/perl5/lib/perl5/PPI/Tokenizer.pm</a></td></tr>
+<tr><td class="h">Statements</td><td align="left">Executed 1862 statements in 5.66ms</td></tr></table>
+
+ <table id="subs_table" border="1" cellpadding="0" class="tablesorter">
+ <caption>Subroutines</caption>
+ <thead>
+ <tr>
+ <th>Calls</th>
+ <th><span title="Number of Places sub is called from">P</span></th>
+ <th><span title="Number of Files sub is called from">F</span></th>
+ <th>Exclusive<br />Time</th>
+ <th>Inclusive<br />Time</th>
+ <th>Subroutine</th>
+ </tr>
+ </thead>
+ <tbody>
+<tr><td class="c0">48</td><td class="c3">1</td><td class="c3">1</td><td class="c0"><span title="0.0%">501µs</span></td><td class="c0"><span title="0.0%">3.16ms</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_process_next_char</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#539">_process_next_char</a></span></td></tr>
+<tr><td class="c0">24</td><td class="c3">1</td><td class="c3">1</td><td class="c0"><span title="0.0%">433µs</span></td><td class="c0"><span title="0.0%">4.47ms</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_process_next_line</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#485">_process_next_line</a></span></td></tr>
+<tr><td class="c0">56</td><td class="c3">1</td><td class="c3">1</td><td class="c0"><span title="0.0%">404µs</span></td><td class="c0"><span title="0.0%">4.89ms</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::get_token</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#255">get_token</a></span></td></tr>
+<tr><td class="c3">8</td><td class="c3">1</td><td class="c3">1</td><td class="c0"><span title="0.0%">389µs</span></td><td class="c0"><span title="0.0%">538µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::new</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#119">new</a></span></td></tr>
+<tr><td class="c0">24</td><td class="c3">1</td><td class="c3">1</td><td class="c0"><span title="0.0%">341µs</span></td><td class="c0"><span title="0.0%">411µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_fill_line</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#438">_fill_line</a></span></td></tr>
+<tr><td class="c1">16</td><td class="c1">2</td><td class="c1">2</td><td class="c1"><span title="0.0%">193µs</span></td><td class="c0"><span title="0.0%">727µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_new_token</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#613">_new_token</a></span></td></tr>
+<tr><td class="c0">56</td><td class="c0">5</td><td class="c0">4</td><td class="c1"><span title="0.0%">185µs</span></td><td class="c1"><span title="0.0%">185µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_finalize_token</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#599">_finalize_token</a></span></td></tr>
+<tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c1"><span title="0.0%">168µs</span></td><td class="c0"><span title="0.0%">238µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@88</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#88">BEGIN@88</a></span></td></tr>
+<tr><td class="c3">8</td><td class="c3">1</td><td class="c3">1</td><td class="c2"><span title="0.0%">122µs</span></td><td class="c0"><span title="0.0%">286µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_previous_significant_tokens</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#691">_previous_significant_tokens</a></span></td></tr>
+<tr><td class="c3">8</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">70µs</span></td><td class="c3"><span title="0.0%">70µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::CORE:subst</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#PPI__Tokenizer__CORE_subst">CORE:subst</a> (opcode)</span></td></tr>
+<tr><td class="c0">24</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">69µs</span></td><td class="c3"><span title="0.0%">69µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_get_line</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#421">_get_line</a></span></td></tr>
+<tr><td class="c3">8</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">38µs</span></td><td class="c3"><span title="0.0%">38µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_clean_eof</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#631">_clean_eof</a></span></td></tr>
+<tr><td class="c1">16</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">22µs</span></td><td class="c3"><span title="0.0%">22µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::CORE:match</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#PPI__Tokenizer__CORE_match">CORE:match</a> (opcode)</span></td></tr>
+<tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">14µs</span></td><td class="c3"><span title="0.0%">17µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@81</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#81">BEGIN@81</a></span></td></tr>
+<tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">8µs</span></td><td class="c3"><span title="0.0%">41µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@82</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#82">BEGIN@82</a></span></td></tr>
+<tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">7µs</span></td><td class="c3"><span title="0.0%">25µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@90</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#90">BEGIN@90</a></span></td></tr>
+<tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">4µs</span></td><td class="c3"><span title="0.0%">4µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@91</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#91">BEGIN@91</a></span></td></tr>
+<tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">4µs</span></td><td class="c3"><span title="0.0%">4µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@87</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#87">BEGIN@87</a></span></td></tr>
+<tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">4µs</span></td><td class="c3"><span title="0.0%">4µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@83</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#83">BEGIN@83</a></span></td></tr>
+<tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">3µs</span></td><td class="c3"><span title="0.0%">3µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@84</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#84">BEGIN@84</a></span></td></tr>
+<tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">3µs</span></td><td class="c3"><span title="0.0%">3µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@85</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#85">BEGIN@85</a></span></td></tr>
+<tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">3µs</span></td><td class="c3"><span title="0.0%">3µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@86</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#86">BEGIN@86</a></span></td></tr>
+<tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::__ANON__[:211]</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#211">__ANON__[:211]</a></span></td></tr>
+<tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_char</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#469">_char</a></span></td></tr>
+<tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_last_significant_token</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#675">_last_significant_token</a></span></td></tr>
+<tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_last_token</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#671">_last_token</a></span></td></tr>
+<tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_opcontext</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#739">_opcontext</a></span></td></tr>
+<tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::all_tokens</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#332">all_tokens</a></span></td></tr>
+<tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::decrement_cursor</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#401">decrement_cursor</a></span></td></tr>
+<tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::increment_cursor</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#376">increment_cursor</a></span></td></tr>
+</tbody></table>
+ Call graph for these subroutines as a
+ <a href="http://en.wikipedia.org/wiki/Graphviz">Graphviz</a>
+ <a href="Users-edenc-perl5-lib-perl5-PPI-Tokenizer-pm.dot">dot language file</a>.
+
+ <table border="1" cellpadding="0">
+ <thead>
+ <tr><th>Line</th>
+ <th><span title="Number of statements executed">State<br />ments</span></th>
+ <th><span title="Time spend executing statements on the line,
+ excluding time spent executing statements in any called subroutines">Time<br />on line</span></th>
+ <th><span title="Number of subroutines calls">Calls</span></th>
+ <th><span title="Time spent in subroutines called (inclusive)">Time<br />in subs</span></th>
+ <th class="left_indent_header">Code</th>
+ </tr>
+
+ </thead>
+ <tbody>
+ <tr><td class="h"><a name="1"></a>1</td><td></td><td></td><td></td><td></td><td class="s">package PPI::Tokenizer;</td></tr>
+<tr><td class="h"><a name="2"></a>2</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="3"></a>3</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr>
+<tr><td class="h"><a name="4"></a>4</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="5"></a>5</td><td></td><td></td><td></td><td></td><td class="s">=head1 NAME</td></tr>
+<tr><td class="h"><a name="6"></a>6</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="7"></a>7</td><td></td><td></td><td></td><td></td><td class="s">PPI::Tokenizer - The Perl Document Tokenizer</td></tr>
+<tr><td class="h"><a name="8"></a>8</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="9"></a>9</td><td></td><td></td><td></td><td></td><td class="s">=head1 SYNOPSIS</td></tr>
+<tr><td class="h"><a name="10"></a>10</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="11"></a>11</td><td></td><td></td><td></td><td></td><td class="s"> # Create a tokenizer for a file, array or string</td></tr>
+<tr><td class="h"><a name="12"></a>12</td><td></td><td></td><td></td><td></td><td class="s"> $Tokenizer = PPI::Tokenizer->new( 'filename.pl' );</td></tr>
+<tr><td class="h"><a name="13"></a>13</td><td></td><td></td><td></td><td></td><td class="s"> $Tokenizer = PPI::Tokenizer->new( \@lines );</td></tr>
+<tr><td class="h"><a name="14"></a>14</td><td></td><td></td><td></td><td></td><td class="s"> $Tokenizer = PPI::Tokenizer->new( \$source );</td></tr>
+<tr><td class="h"><a name="15"></a>15</td><td></td><td></td><td></td><td></td><td class="s"> </td></tr>
+<tr><td class="h"><a name="16"></a>16</td><td></td><td></td><td></td><td></td><td class="s"> # Return all the tokens for the document</td></tr>
+<tr><td class="h"><a name="17"></a>17</td><td></td><td></td><td></td><td></td><td class="s"> my $tokens = $Tokenizer->all_tokens;</td></tr>
+<tr><td class="h"><a name="18"></a>18</td><td></td><td></td><td></td><td></td><td class="s"> </td></tr>
+<tr><td class="h"><a name="19"></a>19</td><td></td><td></td><td></td><td></td><td class="s"> # Or we can use it as an iterator</td></tr>
+<tr><td class="h"><a name="20"></a>20</td><td></td><td></td><td></td><td></td><td class="s"> while ( my $Token = $Tokenizer->get_token ) {</td></tr>
+<tr><td class="h"><a name="21"></a>21</td><td></td><td></td><td></td><td></td><td class="s"> print "Found token '$Token'\n";</td></tr>
+<tr><td class="h"><a name="22"></a>22</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="23"></a>23</td><td></td><td></td><td></td><td></td><td class="s"> </td></tr>
+<tr><td class="h"><a name="24"></a>24</td><td></td><td></td><td></td><td></td><td class="s"> # If we REALLY need to manually nudge the cursor, you</td></tr>
+<tr><td class="h"><a name="25"></a>25</td><td></td><td></td><td></td><td></td><td class="s"> # can do that to (The lexer needs this ability to do rollbacks)</td></tr>
+<tr><td class="h"><a name="26"></a>26</td><td></td><td></td><td></td><td></td><td class="s"> $is_incremented = $Tokenizer->increment_cursor;</td></tr>
+<tr><td class="h"><a name="27"></a>27</td><td></td><td></td><td></td><td></td><td class="s"> $is_decremented = $Tokenizer->decrement_cursor;</td></tr>
+<tr><td class="h"><a name="28"></a>28</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="29"></a>29</td><td></td><td></td><td></td><td></td><td class="s">=head1 DESCRIPTION</td></tr>
+<tr><td class="h"><a name="30"></a>30</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="31"></a>31</td><td></td><td></td><td></td><td></td><td class="s">PPI::Tokenizer is the class that provides Tokenizer objects for use in</td></tr>
+<tr><td class="h"><a name="32"></a>32</td><td></td><td></td><td></td><td></td><td class="s">breaking strings of Perl source code into Tokens.</td></tr>
+<tr><td class="h"><a name="33"></a>33</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="34"></a>34</td><td></td><td></td><td></td><td></td><td class="s">By the time you are reading this, you probably need to know a little</td></tr>
+<tr><td class="h"><a name="35"></a>35</td><td></td><td></td><td></td><td></td><td class="s">about the difference between how perl parses Perl "code" and how PPI</td></tr>
+<tr><td class="h"><a name="36"></a>36</td><td></td><td></td><td></td><td></td><td class="s">parsers Perl "documents".</td></tr>
+<tr><td class="h"><a name="37"></a>37</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="38"></a>38</td><td></td><td></td><td></td><td></td><td class="s">"perl" itself (the interpreter) uses a heavily modified lex specification</td></tr>
+<tr><td class="h"><a name="39"></a>39</td><td></td><td></td><td></td><td></td><td class="s">to specify its parsing logic, maintains several types of state as it</td></tr>
+<tr><td class="h"><a name="40"></a>40</td><td></td><td></td><td></td><td></td><td class="s">goes, and incrementally tokenizes, lexes AND EXECUTES at the same time.</td></tr>
+<tr><td class="h"><a name="41"></a>41</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="42"></a>42</td><td></td><td></td><td></td><td></td><td class="s">In fact, it is provably impossible to use perl's parsing method without</td></tr>
+<tr><td class="h"><a name="43"></a>43</td><td></td><td></td><td></td><td></td><td class="s">simultaneously executing code. A formal mathematical proof has been</td></tr>
+<tr><td class="h"><a name="44"></a>44</td><td></td><td></td><td></td><td></td><td class="s">published demonstrating the method.</td></tr>
+<tr><td class="h"><a name="45"></a>45</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="46"></a>46</td><td></td><td></td><td></td><td></td><td class="s">This is where the truism "Only perl can parse Perl" comes from.</td></tr>
+<tr><td class="h"><a name="47"></a>47</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="48"></a>48</td><td></td><td></td><td></td><td></td><td class="s">PPI uses a completely different approach by abandoning the (impossible)</td></tr>
+<tr><td class="h"><a name="49"></a>49</td><td></td><td></td><td></td><td></td><td class="s">ability to parse Perl the same way that the interpreter does, and instead</td></tr>
+<tr><td class="h"><a name="50"></a>50</td><td></td><td></td><td></td><td></td><td class="s">parsing the source as a document, using a document structure independantly</td></tr>
+<tr><td class="h"><a name="51"></a>51</td><td></td><td></td><td></td><td></td><td class="s">derived from the Perl documentation and approximating the perl interpreter</td></tr>
+<tr><td class="h"><a name="52"></a>52</td><td></td><td></td><td></td><td></td><td class="s">interpretation as closely as possible.</td></tr>
+<tr><td class="h"><a name="53"></a>53</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="54"></a>54</td><td></td><td></td><td></td><td></td><td class="s">It was touch and go for a long time whether we could get it close enough,</td></tr>
+<tr><td class="h"><a name="55"></a>55</td><td></td><td></td><td></td><td></td><td class="s">but in the end it turned out that it could be done.</td></tr>
+<tr><td class="h"><a name="56"></a>56</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="57"></a>57</td><td></td><td></td><td></td><td></td><td class="s">In this approach, the tokenizer C<PPI::Tokenizer> is implemented separately</td></tr>
+<tr><td class="h"><a name="58"></a>58</td><td></td><td></td><td></td><td></td><td class="s">from the lexer L<PPI::Lexer>.</td></tr>
+<tr><td class="h"><a name="59"></a>59</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="60"></a>60</td><td></td><td></td><td></td><td></td><td class="s">The job of C<PPI::Tokenizer> is to take pure source as a string and break it</td></tr>
+<tr><td class="h"><a name="61"></a>61</td><td></td><td></td><td></td><td></td><td class="s">up into a stream/set of tokens, and contains most of the "black magic" used</td></tr>
+<tr><td class="h"><a name="62"></a>62</td><td></td><td></td><td></td><td></td><td class="s">in PPI. By comparison, the lexer implements a relatively straight forward</td></tr>
+<tr><td class="h"><a name="63"></a>63</td><td></td><td></td><td></td><td></td><td class="s">tree structure, and has an implementation that is uncomplicated (compared</td></tr>
+<tr><td class="h"><a name="64"></a>64</td><td></td><td></td><td></td><td></td><td class="s">to the insanity in the tokenizer at least).</td></tr>
+<tr><td class="h"><a name="65"></a>65</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="66"></a>66</td><td></td><td></td><td></td><td></td><td class="s">The Tokenizer uses an immense amount of heuristics, guessing and cruft,</td></tr>
+<tr><td class="h"><a name="67"></a>67</td><td></td><td></td><td></td><td></td><td class="s">supported by a very B<VERY> flexible internal API, but fortunately it was</td></tr>
+<tr><td class="h"><a name="68"></a>68</td><td></td><td></td><td></td><td></td><td class="s">possible to largely encapsulate the black magic, so there is not a lot that</td></tr>
+<tr><td class="h"><a name="69"></a>69</td><td></td><td></td><td></td><td></td><td class="s">gets exposed to people using the C<PPI::Tokenizer> itself.</td></tr>
+<tr><td class="h"><a name="70"></a>70</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="71"></a>71</td><td></td><td></td><td></td><td></td><td class="s">=head1 METHODS</td></tr>
+<tr><td class="h"><a name="72"></a>72</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="73"></a>73</td><td></td><td></td><td></td><td></td><td class="s">Despite the incredible complexity, the Tokenizer itself only exposes a</td></tr>
+<tr><td class="h"><a name="74"></a>74</td><td></td><td></td><td></td><td></td><td class="s">relatively small number of methods, with most of the complexity implemented</td></tr>
+<tr><td class="h"><a name="75"></a>75</td><td></td><td></td><td></td><td></td><td class="s">in private methods.</td></tr>
+<tr><td class="h"><a name="76"></a>76</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="77"></a>77</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr>
+<tr><td class="h"><a name="78"></a>78</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="79"></a>79</td><td></td><td></td><td></td><td></td><td class="s"># Make sure everything we need is loaded so</td></tr>
+<tr><td class="h"><a name="80"></a>80</td><td></td><td></td><td></td><td></td><td class="s"># we don't have to go and load all of PPI.</td></tr>
+<tr><td class="h"><a name="81"></a>81</td><td class="c3">2</td><td class="c3"><span title="Avg 13µs">27µs</span></td><td class="c3">2</td><td class="c3">20µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 17µs (14+3) within PPI::Tokenizer::BEGIN@81 which was called:
+# once (14µs+3µs) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#81">line 81</a></div></div>use strict;<div class="calls"><div class="calls_out"># spent 17µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#81">PPI::Tokenizer::BEGIN@81</a>
+# spent 3µs making 1 call to <a href="strict-pm-3-block.html#34">strict::import</a></div></div></td></tr>
+<tr><td class="h"><a name="82"></a>82</td><td class="c3">2</td><td class="c3"><span title="Avg 12µs">23µs</span></td><td class="c3">2</td><td class="c3">74µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 41µs (8+33) within PPI::Tokenizer::BEGIN@82 which was called:
+# once (8µs+33µs) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#82">line 82</a></div></div>use Params::Util qw{_INSTANCE _SCALAR0 _ARRAY0};<div class="calls"><div class="calls_out"># spent 41µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#82">PPI::Tokenizer::BEGIN@82</a>
+# spent 33µs making 1 call to <a href="Exporter-pm-8-block.html#28">Exporter::import</a></div></div></td></tr>
+<tr><td class="h"><a name="83"></a>83</td><td class="c3">2</td><td class="c3"><span title="Avg 9µs">18µs</span></td><td class="c3">1</td><td class="c3">4µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 4µs within PPI::Tokenizer::BEGIN@83 which was called:
+# once (4µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#83">line 83</a></div></div>use List::MoreUtils ();<div class="calls"><div class="calls_out"># spent 4µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#83">PPI::Tokenizer::BEGIN@83</a></div></div></td></tr>
+<tr><td class="h"><a name="84"></a>84</td><td class="c3">2</td><td class="c3"><span title="Avg 9µs">17µs</span></td><td class="c3">1</td><td class="c3">3µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 3µs within PPI::Tokenizer::BEGIN@84 which was called:
+# once (3µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#84">line 84</a></div></div>use PPI::Util ();<div class="calls"><div class="calls_out"># spent 3µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#84">PPI::Tokenizer::BEGIN@84</a></div></div></td></tr>
+<tr><td class="h"><a name="85"></a>85</td><td class="c3">2</td><td class="c3"><span title="Avg 11µs">21µs</span></td><td class="c3">1</td><td class="c3">3µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 3µs within PPI::Tokenizer::BEGIN@85 which was called:
+# once (3µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#85">line 85</a></div></div>use PPI::Element ();<div class="calls"><div class="calls_out"># spent 3µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#85">PPI::Tokenizer::BEGIN@85</a></div></div></td></tr>
+<tr><td class="h"><a name="86"></a>86</td><td class="c3">2</td><td class="c3"><span title="Avg 10µs">20µs</span></td><td class="c3">1</td><td class="c3">3µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 3µs within PPI::Tokenizer::BEGIN@86 which was called:
+# once (3µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#86">line 86</a></div></div>use PPI::Token ();<div class="calls"><div class="calls_out"># spent 3µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#86">PPI::Tokenizer::BEGIN@86</a></div></div></td></tr>
+<tr><td class="h"><a name="87"></a>87</td><td class="c3">2</td><td class="c3"><span title="Avg 9µs">17µs</span></td><td class="c3">1</td><td class="c3">4µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 4µs within PPI::Tokenizer::BEGIN@87 which was called:
+# once (4µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#87">line 87</a></div></div>use PPI::Exception ();<div class="calls"><div class="calls_out"># spent 4µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#87">PPI::Tokenizer::BEGIN@87</a></div></div></td></tr>
+<tr><td class="h"><a name="88"></a>88</td><td class="c3">2</td><td class="c1"><span title="Avg 49µs">98µs</span></td><td class="c3">1</td><td class="c0">238µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 238µs (168+70) within PPI::Tokenizer::BEGIN@88 which was called:
+# once (168µs+70µs) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#88">line 88</a></div></div>use PPI::Exception::ParserRejection ();<div class="calls"><div class="calls_out"># spent 238µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#88">PPI::Tokenizer::BEGIN@88</a></div></div></td></tr>
+<tr><td class="h"><a name="89"></a>89</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="90"></a>90</td><td class="c3">2</td><td class="c3"><span title="Avg 14µs">27µs</span></td><td class="c3">2</td><td class="c3">43µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 25µs (7+18) within PPI::Tokenizer::BEGIN@90 which was called:
+# once (7µs+18µs) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#90">line 90</a></div></div>use vars qw{$VERSION};<div class="calls"><div class="calls_out"># spent 25µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#90">PPI::Tokenizer::BEGIN@90</a>
+# spent 18µs making 1 call to <a href="vars-pm-6-block.html#10">vars::import</a></div></div></td></tr>
+<tr><td class="h"><a name="91"></a>91</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 4µs within PPI::Tokenizer::BEGIN@91 which was called:
+# once (4µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#93">line 93</a></div></div>BEGIN {</td></tr>
+<tr><td class="h"><a name="92"></a>92</td><td class="c3">1</td><td class="c3"><span title="Avg 4µs">4µs</span></td><td></td><td></td><td class="s"> $VERSION = '1.215';</td></tr>
+<tr><td class="h"><a name="93"></a>93</td><td class="c3">1</td><td class="c0"><span title="Avg 2.73ms">2.73ms</span></td><td class="c3">1</td><td class="c3">4µs</td><td class="s">}<div class="calls"><div class="calls_out"># spent 4µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#91">PPI::Tokenizer::BEGIN@91</a></div></div></td></tr>
+<tr><td class="h"><a name="94"></a>94</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="99"></a>99</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr>
+<tr><td class="h"><a name="100"></a>100</td><td></td><td></td><td></td><td></td><td class="s"># Creation and Initialization</td></tr>
+<tr><td class="h"><a name="101"></a>101</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="102"></a>102</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr>
+<tr><td class="h"><a name="103"></a>103</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="104"></a>104</td><td></td><td></td><td></td><td></td><td class="s">=head2 new $file | \@lines | \$source</td></tr>
+<tr><td class="h"><a name="105"></a>105</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="106"></a>106</td><td></td><td></td><td></td><td></td><td class="s">The main C<new> constructor creates a new Tokenizer object. These</td></tr>
+<tr><td class="h"><a name="107"></a>107</td><td></td><td></td><td></td><td></td><td class="s">objects have no configuration parameters, and can only be used once,</td></tr>
+<tr><td class="h"><a name="108"></a>108</td><td></td><td></td><td></td><td></td><td class="s">to tokenize a single perl source file.</td></tr>
+<tr><td class="h"><a name="109"></a>109</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="110"></a>110</td><td></td><td></td><td></td><td></td><td class="s">It takes as argument either a normal scalar containing source code,</td></tr>
+<tr><td class="h"><a name="111"></a>111</td><td></td><td></td><td></td><td></td><td class="s">a reference to a scalar containing source code, or a reference to an</td></tr>
+<tr><td class="h"><a name="112"></a>112</td><td></td><td></td><td></td><td></td><td class="s">ARRAY containing newline-terminated lines of source code.</td></tr>
+<tr><td class="h"><a name="113"></a>113</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="114"></a>114</td><td></td><td></td><td></td><td></td><td class="s">Returns a new C<PPI::Tokenizer> object on success, or throws a</td></tr>
+<tr><td class="h"><a name="115"></a>115</td><td></td><td></td><td></td><td></td><td class="s">L<PPI::Exception> exception on error.</td></tr>
+<tr><td class="h"><a name="116"></a>116</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="117"></a>117</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr>
+<tr><td class="h"><a name="118"></a>118</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="119"></a>119</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 538µs (389+149) within PPI::Tokenizer::new which was called 8 times, avg 67µs/call:
+# 8 times (389µs+149µs) by PPI::Lexer::lex_source at <a href="PPI-Lexer-pm-1005-block.html#191">line 191 of PPI/Lexer.pm</a>, avg 67µs/call</div></div>sub new {</td></tr>
+<tr><td class="h"><a name="120"></a>120</td><td class="c0">56</td><td class="c0"><span title="Avg 5µs">255µs</span></td><td></td><td></td><td class="s"> my $class = ref($_[0]) || $_[0];</td></tr>
+<tr><td class="h"><a name="121"></a>121</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="122"></a>122</td><td></td><td></td><td></td><td></td><td class="s"> # Create the empty tokenizer struct</td></tr>
+<tr><td class="h"><a name="123"></a>123</td><td></td><td></td><td></td><td></td><td class="s"> my $self = bless {</td></tr>
+<tr><td class="h"><a name="124"></a>124</td><td></td><td></td><td></td><td></td><td class="s"> # Source code</td></tr>
+<tr><td class="h"><a name="125"></a>125</td><td></td><td></td><td></td><td></td><td class="s"> source => undef,</td></tr>
+<tr><td class="h"><a name="126"></a>126</td><td></td><td></td><td></td><td></td><td class="s"> source_bytes => undef,</td></tr>
+<tr><td class="h"><a name="127"></a>127</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="128"></a>128</td><td></td><td></td><td></td><td></td><td class="s"> # Line buffer</td></tr>
+<tr><td class="h"><a name="129"></a>129</td><td></td><td></td><td></td><td></td><td class="s"> line => undef,</td></tr>
+<tr><td class="h"><a name="130"></a>130</td><td></td><td></td><td></td><td></td><td class="s"> line_length => undef,</td></tr>
+<tr><td class="h"><a name="131"></a>131</td><td></td><td></td><td></td><td></td><td class="s"> line_cursor => undef,</td></tr>
+<tr><td class="h"><a name="132"></a>132</td><td></td><td></td><td></td><td></td><td class="s"> line_count => 0,</td></tr>
+<tr><td class="h"><a name="133"></a>133</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="134"></a>134</td><td></td><td></td><td></td><td></td><td class="s"> # Parse state</td></tr>
+<tr><td class="h"><a name="135"></a>135</td><td></td><td></td><td></td><td></td><td class="s"> token => undef,</td></tr>
+<tr><td class="h"><a name="136"></a>136</td><td></td><td></td><td></td><td></td><td class="s"> class => 'PPI::Token::BOM',</td></tr>
+<tr><td class="h"><a name="137"></a>137</td><td></td><td></td><td></td><td></td><td class="s"> zone => 'PPI::Token::Whitespace',</td></tr>
+<tr><td class="h"><a name="138"></a>138</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="139"></a>139</td><td></td><td></td><td></td><td></td><td class="s"> # Output token buffer</td></tr>
+<tr><td class="h"><a name="140"></a>140</td><td></td><td></td><td></td><td></td><td class="s"> tokens => [],</td></tr>
+<tr><td class="h"><a name="141"></a>141</td><td></td><td></td><td></td><td></td><td class="s"> token_cursor => 0,</td></tr>
+<tr><td class="h"><a name="142"></a>142</td><td></td><td></td><td></td><td></td><td class="s"> token_eof => 0,</td></tr>
+<tr><td class="h"><a name="143"></a>143</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="144"></a>144</td><td></td><td></td><td></td><td></td><td class="s"> # Perl 6 blocks</td></tr>
+<tr><td class="h"><a name="145"></a>145</td><td></td><td></td><td></td><td></td><td class="s"> perl6 => [],</td></tr>
+<tr><td class="h"><a name="146"></a>146</td><td></td><td></td><td></td><td></td><td class="s"> }, $class;</td></tr>
+<tr><td class="h"><a name="147"></a>147</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="148"></a>148</td><td></td><td></td><td class="c3">8</td><td class="c3">8µs</td><td class="s"> if ( ! defined $_[1] ) {<div class="calls"><div class="calls_out"> # spent 8µs making 8 calls to <a href="Params-Util-pm-70-block.html#Params__Util___SCALAR0">Params::Util::_SCALAR0</a>, avg 1µs/call</div></div></td></tr>
+<tr><td class="h"><a name="149"></a>149</td><td></td><td></td><td></td><td></td><td class="s"> # We weren't given anything</td></tr>
+<tr><td class="h"><a name="150"></a>150</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw("No source provided to Tokenizer");</td></tr>
+<tr><td class="h"><a name="151"></a>151</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="152"></a>152</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( ! ref $_[1] ) {</td></tr>
+<tr><td class="h"><a name="153"></a>153</td><td></td><td></td><td></td><td></td><td class="s"> my $source = PPI::Util::_slurp($_[1]);</td></tr>
+<tr><td class="h"><a name="154"></a>154</td><td></td><td></td><td></td><td></td><td class="s"> if ( ref $source ) {</td></tr>
+<tr><td class="h"><a name="155"></a>155</td><td></td><td></td><td></td><td></td><td class="s"> # Content returned by reference</td></tr>
+<tr><td class="h"><a name="156"></a>156</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = $$source;</td></tr>
+<tr><td class="h"><a name="157"></a>157</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr>
+<tr><td class="h"><a name="158"></a>158</td><td></td><td></td><td></td><td></td><td class="s"> # Errors returned as a string</td></tr>
+<tr><td class="h"><a name="159"></a>159</td><td></td><td></td><td></td><td></td><td class="s"> return( $source );</td></tr>
+<tr><td class="h"><a name="160"></a>160</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="161"></a>161</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="162"></a>162</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( _SCALAR0($_[1]) ) {</td></tr>
+<tr><td class="h"><a name="163"></a>163</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = ${$_[1]};</td></tr>
+<tr><td class="h"><a name="164"></a>164</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="165"></a>165</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( _ARRAY0($_[1]) ) {</td></tr>
+<tr><td class="h"><a name="166"></a>166</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = join '', map { "\n" } @{$_[1]};</td></tr>
+<tr><td class="h"><a name="167"></a>167</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="168"></a>168</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr>
+<tr><td class="h"><a name="169"></a>169</td><td></td><td></td><td></td><td></td><td class="s"> # We don't support whatever this is</td></tr>
+<tr><td class="h"><a name="170"></a>170</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw(ref($_[1]) . " is not supported as a source provider");</td></tr>
+<tr><td class="h"><a name="171"></a>171</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="172"></a>172</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="173"></a>173</td><td></td><td></td><td></td><td></td><td class="s"> # We can't handle a null string</td></tr>
+<tr><td class="h"><a name="174"></a>174</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_bytes} = length $self->{source};</td></tr>
+<tr><td class="h"><a name="175"></a>175</td><td class="c3">16</td><td class="c0"><span title="Avg 10µs">163µs</span></td><td></td><td></td><td class="s"> if ( $self->{source_bytes} > 1048576 ) {</td></tr>
+<tr><td class="h"><a name="176"></a>176</td><td></td><td></td><td></td><td></td><td class="s"> # Dammit! It's ALWAYS the "Perl" modules larger than a</td></tr>
+<tr><td class="h"><a name="177"></a>177</td><td></td><td></td><td></td><td></td><td class="s"> # meg that seems to blow up the Tokenizer/Lexer.</td></tr>
+<tr><td class="h"><a name="178"></a>178</td><td></td><td></td><td></td><td></td><td class="s"> # Nobody actually writes real programs larger than a meg</td></tr>
+<tr><td class="h"><a name="179"></a>179</td><td></td><td></td><td></td><td></td><td class="s"> # Perl::Tidy (the largest) is only 800k.</td></tr>
+<tr><td class="h"><a name="180"></a>180</td><td></td><td></td><td></td><td></td><td class="s"> # It is always these idiots with massive Data::Dumper</td></tr>
+<tr><td class="h"><a name="181"></a>181</td><td></td><td></td><td></td><td></td><td class="s"> # structs or huge RecDescent parser.</td></tr>
+<tr><td class="h"><a name="182"></a>182</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception::ParserRejection->throw("File is too large");</td></tr>
+<tr><td class="h"><a name="183"></a>183</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="184"></a>184</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( $self->{source_bytes} ) {</td></tr>
+<tr><td class="h"><a name="185"></a>185</td><td></td><td></td><td></td><td></td><td class="s"> # Split on local newlines</td></tr>
+<tr><td class="h"><a name="186"></a>186</td><td></td><td></td><td class="c3">8</td><td class="c3">70µs</td><td class="s"> $self->{source} =~ s/(?:\015{1,2}\012|\015|\012)/\n/g;<div class="calls"><div class="calls_out"> # spent 70µs making 8 calls to <a href="PPI-Tokenizer-pm-1003-block.html#PPI__Tokenizer__CORE_subst">PPI::Tokenizer::CORE:subst</a>, avg 9µs/call</div></div></td></tr>
+<tr><td class="h"><a name="187"></a>187</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = [ split /(?<=\n)/, $self->{source} ];</td></tr>
+<tr><td class="h"><a name="188"></a>188</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="189"></a>189</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr>
+<tr><td class="h"><a name="190"></a>190</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = [ ];</td></tr>
+<tr><td class="h"><a name="191"></a>191</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="192"></a>192</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="193"></a>193</td><td></td><td></td><td></td><td></td><td class="s"> ### EVIL</td></tr>
+<tr><td class="h"><a name="194"></a>194</td><td></td><td></td><td></td><td></td><td class="s"> # I'm explaining this earlier than I should so you can understand</td></tr>
+<tr><td class="h"><a name="195"></a>195</td><td></td><td></td><td></td><td></td><td class="s"> # why I'm about to do something that looks very strange. There's</td></tr>
+<tr><td class="h"><a name="196"></a>196</td><td></td><td></td><td></td><td></td><td class="s"> # a problem with the Tokenizer, in that tokens tend to change</td></tr>
+<tr><td class="h"><a name="197"></a>197</td><td></td><td></td><td></td><td></td><td class="s"> # classes as each letter is added, but they don't get allocated</td></tr>
+<tr><td class="h"><a name="198"></a>198</td><td></td><td></td><td></td><td></td><td class="s"> # their definite final class until the "end" of the token, the</td></tr>
+<tr><td class="h"><a name="199"></a>199</td><td></td><td></td><td></td><td></td><td class="s"> # detection of which occurs in about a hundred different places,</td></tr>
+<tr><td class="h"><a name="200"></a>200</td><td></td><td></td><td></td><td></td><td class="s"> # all through various crufty code (that triples the speed).</td></tr>
+<tr><td class="h"><a name="201"></a>201</td><td></td><td></td><td></td><td></td><td class="s"> #</td></tr>
+<tr><td class="h"><a name="202"></a>202</td><td></td><td></td><td></td><td></td><td class="s"> # However, in general, this does not apply to tokens in which a</td></tr>
+<tr><td class="h"><a name="203"></a>203</td><td></td><td></td><td></td><td></td><td class="s"> # whitespace character is valid, such as comments, whitespace and</td></tr>
+<tr><td class="h"><a name="204"></a>204</td><td></td><td></td><td></td><td></td><td class="s"> # big strings.</td></tr>
+<tr><td class="h"><a name="205"></a>205</td><td></td><td></td><td></td><td></td><td class="s"> #</td></tr>
+<tr><td class="h"><a name="206"></a>206</td><td></td><td></td><td></td><td></td><td class="s"> # So what we do is add a space to the end of the source. This</td></tr>
+<tr><td class="h"><a name="207"></a>207</td><td></td><td></td><td></td><td></td><td class="s"> # triggers normal "end of token" functionality for all cases. Then,</td></tr>
+<tr><td class="h"><a name="208"></a>208</td><td></td><td></td><td></td><td></td><td class="s"> # once the tokenizer hits end of file, it examines the last token to</td></tr>
+<tr><td class="h"><a name="209"></a>209</td><td></td><td></td><td></td><td></td><td class="s"> # manually either remove the ' ' token, or chop it off the end of</td></tr>
+<tr><td class="h"><a name="210"></a>210</td><td></td><td></td><td></td><td></td><td class="s"> # a longer one in which the space would be valid.</td></tr>
+<tr><td class="h"><a name="211"></a>211</td><td class="c3">8</td><td class="c0"><span title="Avg 14µs">115µs</span></td><td class="c0">24</td><td class="c3">78µs</td><td class="s"> if ( List::MoreUtils::any { /^__(?:DATA|END)__\s*$/ } @{$self->{source}} ) {<div class="calls"><div class="calls_out"> # spent 56µs making 8 calls to <a href="List-MoreUtils-pm-81-block.html#List__MoreUtils__any">List::MoreUtils::any</a>, avg 7µs/call
+ # spent 22µs making 16 calls to <a href="PPI-Tokenizer-pm-1003-block.html#PPI__Tokenizer__CORE_match">PPI::Tokenizer::CORE:match</a>, avg 1µs/call</div></div></td></tr>
+<tr><td class="h"><a name="212"></a>212</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_eof_chop} = '';</td></tr>
+<tr><td class="h"><a name="213"></a>213</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( ! defined $self->{source}->[0] ) {</td></tr>
+<tr><td class="h"><a name="214"></a>214</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_eof_chop} = '';</td></tr>
+<tr><td class="h"><a name="215"></a>215</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( $self->{source}->[-1] =~ /\s$/ ) {</td></tr>
+<tr><td class="h"><a name="216"></a>216</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_eof_chop} = '';</td></tr>
+<tr><td class="h"><a name="217"></a>217</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr>
+<tr><td class="h"><a name="218"></a>218</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_eof_chop} = 1;</td></tr>
+<tr><td class="h"><a name="219"></a>219</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source}->[-1] .= ' ';</td></tr>
+<tr><td class="h"><a name="220"></a>220</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="221"></a>221</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="222"></a>222</td><td></td><td></td><td></td><td></td><td class="s"> $self;</td></tr>
+<tr><td class="h"><a name="223"></a>223</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="224"></a>224</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="229"></a>229</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr>
+<tr><td class="h"><a name="230"></a>230</td><td></td><td></td><td></td><td></td><td class="s"># Main Public Methods</td></tr>
+<tr><td class="h"><a name="231"></a>231</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="232"></a>232</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr>
+<tr><td class="h"><a name="233"></a>233</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="234"></a>234</td><td></td><td></td><td></td><td></td><td class="s">=head2 get_token</td></tr>
+<tr><td class="h"><a name="235"></a>235</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="236"></a>236</td><td></td><td></td><td></td><td></td><td class="s">When using the PPI::Tokenizer object as an iterator, the C<get_token></td></tr>
+<tr><td class="h"><a name="237"></a>237</td><td></td><td></td><td></td><td></td><td class="s">method is the primary method that is used. It increments the cursor</td></tr>
+<tr><td class="h"><a name="238"></a>238</td><td></td><td></td><td></td><td></td><td class="s">and returns the next Token in the output array.</td></tr>
+<tr><td class="h"><a name="239"></a>239</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="240"></a>240</td><td></td><td></td><td></td><td></td><td class="s">The actual parsing of the file is done only as-needed, and a line at</td></tr>
+<tr><td class="h"><a name="241"></a>241</td><td></td><td></td><td></td><td></td><td class="s">a time. When C<get_token> hits the end of the token array, it will</td></tr>
+<tr><td class="h"><a name="242"></a>242</td><td></td><td></td><td></td><td></td><td class="s">cause the parser to pull in the next line and parse it, continuing</td></tr>
+<tr><td class="h"><a name="243"></a>243</td><td></td><td></td><td></td><td></td><td class="s">as needed until there are more tokens on the output array that</td></tr>
+<tr><td class="h"><a name="244"></a>244</td><td></td><td></td><td></td><td></td><td class="s">get_token can then return.</td></tr>
+<tr><td class="h"><a name="245"></a>245</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="246"></a>246</td><td></td><td></td><td></td><td></td><td class="s">This means that a number of Tokenizer objects can be created, and</td></tr>
+<tr><td class="h"><a name="247"></a>247</td><td></td><td></td><td></td><td></td><td class="s">won't consume significant CPU until you actually begin to pull tokens</td></tr>
+<tr><td class="h"><a name="248"></a>248</td><td></td><td></td><td></td><td></td><td class="s">from it.</td></tr>
+<tr><td class="h"><a name="249"></a>249</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="250"></a>250</td><td></td><td></td><td></td><td></td><td class="s">Return a L<PPI::Token> object on success, C<0> if the Tokenizer had</td></tr>
+<tr><td class="h"><a name="251"></a>251</td><td></td><td></td><td></td><td></td><td class="s">reached the end of the file, or C<undef> on error.</td></tr>
+<tr><td class="h"><a name="252"></a>252</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="253"></a>253</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr>
+<tr><td class="h"><a name="254"></a>254</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="255"></a>255</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 4.89ms (404µs+4.49) within PPI::Tokenizer::get_token which was called 56 times, avg 87µs/call:
+# 56 times (404µs+4.49ms) by PPI::Lexer::_get_token at <a href="PPI-Lexer-pm-1005-block.html#1413">line 1413 of PPI/Lexer.pm</a>, avg 87µs/call</div></div>sub get_token {</td></tr>
+<tr><td class="h"><a name="256"></a>256</td><td class="c0">256</td><td class="c0"><span title="Avg 851ns">218µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="257"></a>257</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="258"></a>258</td><td></td><td></td><td></td><td></td><td class="s"> # Shortcut for EOF</td></tr>
+<tr><td class="h"><a name="259"></a>259</td><td></td><td></td><td></td><td></td><td class="s"> if ( $self->{token_eof}</td></tr>
+<tr><td class="h"><a name="260"></a>260</td><td></td><td></td><td></td><td></td><td class="s"> and $self->{token_cursor} > scalar @{$self->{tokens}}</td></tr>
+<tr><td class="h"><a name="261"></a>261</td><td></td><td></td><td></td><td></td><td class="s"> ) {</td></tr>
+<tr><td class="h"><a name="262"></a>262</td><td></td><td></td><td></td><td></td><td class="s"> return 0;</td></tr>
+<tr><td class="h"><a name="263"></a>263</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="264"></a>264</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="265"></a>265</td><td></td><td></td><td></td><td></td><td class="s"> # Return the next token if we can</td></tr>
+<tr><td class="h"><a name="266"></a>266</td><td class="c0">64</td><td class="c2"><span title="Avg 1µs">72µs</span></td><td class="c0">32</td><td class="c3">18µs</td><td class="s"> if ( my $token = $self->{tokens}->[ $self->{token_cursor} ] ) {<div class="calls"><div class="calls_out"> # spent 18µs making 32 calls to <a href="PPI-Util-pm-915-block.html#PPI__Util__TRUE">PPI::Util::TRUE</a>, avg 550ns/call</div></div></td></tr>
+<tr><td class="h"><a name="267"></a>267</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token_cursor}++;</td></tr>
+<tr><td class="h"><a name="268"></a>268</td><td></td><td></td><td></td><td></td><td class="s"> return $token;</td></tr>
+<tr><td class="h"><a name="269"></a>269</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="270"></a>270</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="271"></a>271</td><td></td><td></td><td></td><td></td><td class="s"> my $line_rv;</td></tr>
+<tr><td class="h"><a name="272"></a>272</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="273"></a>273</td><td></td><td></td><td></td><td></td><td class="s"> # Catch exceptions and return undef, so that we</td></tr>
+<tr><td class="h"><a name="274"></a>274</td><td></td><td></td><td></td><td></td><td class="s"> # can start to convert code to exception-based code.</td></tr>
+<tr><td class="h"><a name="275"></a>275</td><td class="c1">40</td><td class="c3"><span title="Avg 1µs">40µs</span></td><td></td><td></td><td class="s"> my $rv = eval {</td></tr>
+<tr><td class="h"><a name="276"></a>276</td><td></td><td></td><td></td><td></td><td class="s"> # No token, we need to get some more</td></tr>
+<tr><td class="h"><a name="277"></a>277</td><td></td><td></td><td class="c0">24</td><td class="c0">4.47ms</td><td class="s"> while ( $line_rv = $self->_process_next_line ) {<div class="calls"><div class="calls_out"> # spent 4.47ms making 24 calls to <a href="PPI-Tokenizer-pm-1003-block.html#485">PPI::Tokenizer::_process_next_line</a>, avg 186µs/call</div></div></td></tr>
+<tr><td class="h"><a name="278"></a>278</td><td></td><td></td><td></td><td></td><td class="s"> # If there is something in the buffer, return it</td></tr>
+<tr><td class="h"><a name="279"></a>279</td><td></td><td></td><td></td><td></td><td class="s"> # The defined() prevents a ton of calls to PPI::Util::TRUE</td></tr>
+<tr><td class="h"><a name="280"></a>280</td><td class="c2">24</td><td class="c3"><span title="Avg 1µs">28µs</span></td><td></td><td></td><td class="s"> if ( defined( my $token = $self->{tokens}->[ $self->{token_cursor} ] ) ) {</td></tr>
+<tr><td class="h"><a name="281"></a>281</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token_cursor}++;</td></tr>
+<tr><td class="h"><a name="282"></a>282</td><td></td><td></td><td></td><td></td><td class="s"> return $token;</td></tr>
+<tr><td class="h"><a name="283"></a>283</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="284"></a>284</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="285"></a>285</td><td></td><td></td><td></td><td></td><td class="s"> return undef;</td></tr>
+<tr><td class="h"><a name="286"></a>286</td><td></td><td></td><td></td><td></td><td class="s"> };</td></tr>
+<tr><td class="h"><a name="287"></a>287</td><td></td><td></td><td class="c3">8</td><td class="c3">6µs</td><td class="s"> if ( $@ ) {<div class="calls"><div class="calls_out"> # spent 6µs making 8 calls to <a href="PPI-Util-pm-915-block.html#PPI__Util__TRUE">PPI::Util::TRUE</a>, avg 750ns/call</div></div></td></tr>
+<tr><td class="h"><a name="288"></a>288</td><td></td><td></td><td></td><td></td><td class="s"> if ( _INSTANCE($@, 'PPI::Exception') ) {</td></tr>
+<tr><td class="h"><a name="289"></a>289</td><td></td><td></td><td></td><td></td><td class="s"> $@->throw;</td></tr>
+<tr><td class="h"><a name="290"></a>290</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr>
+<tr><td class="h"><a name="291"></a>291</td><td></td><td></td><td></td><td></td><td class="s"> my $errstr = $@;</td></tr>
+<tr><td class="h"><a name="292"></a>292</td><td></td><td></td><td></td><td></td><td class="s"> $errstr =~ s/^(.*) at line .+$/$1/;</td></tr>
+<tr><td class="h"><a name="293"></a>293</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw( $errstr );</td></tr>
+<tr><td class="h"><a name="294"></a>294</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="295"></a>295</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( $rv ) {</td></tr>
+<tr><td class="h"><a name="296"></a>296</td><td></td><td></td><td></td><td></td><td class="s"> return $rv;</td></tr>
+<tr><td class="h"><a name="297"></a>297</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="298"></a>298</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="299"></a>299</td><td class="c0">48</td><td class="c3"><span title="Avg 973ns">47µs</span></td><td></td><td></td><td class="s"> if ( defined $line_rv ) {</td></tr>
+<tr><td class="h"><a name="300"></a>300</td><td></td><td></td><td></td><td></td><td class="s"> # End of file, but we can still return things from the buffer</td></tr>
+<tr><td class="h"><a name="301"></a>301</td><td></td><td></td><td></td><td></td><td class="s"> if ( my $token = $self->{tokens}->[ $self->{token_cursor} ] ) {</td></tr>
+<tr><td class="h"><a name="302"></a>302</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token_cursor}++;</td></tr>
+<tr><td class="h"><a name="303"></a>303</td><td></td><td></td><td></td><td></td><td class="s"> return $token;</td></tr>
+<tr><td class="h"><a name="304"></a>304</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="305"></a>305</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="306"></a>306</td><td></td><td></td><td></td><td></td><td class="s"> # Set our token end of file flag</td></tr>
+<tr><td class="h"><a name="307"></a>307</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token_eof} = 1;</td></tr>
+<tr><td class="h"><a name="308"></a>308</td><td></td><td></td><td></td><td></td><td class="s"> return 0;</td></tr>
+<tr><td class="h"><a name="309"></a>309</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="310"></a>310</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="311"></a>311</td><td></td><td></td><td></td><td></td><td class="s"> # Error, pass it up to our caller</td></tr>
+<tr><td class="h"><a name="312"></a>312</td><td></td><td></td><td></td><td></td><td class="s"> undef;</td></tr>
+<tr><td class="h"><a name="313"></a>313</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="314"></a>314</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="315"></a>315</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr>
+<tr><td class="h"><a name="316"></a>316</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="317"></a>317</td><td></td><td></td><td></td><td></td><td class="s">=head2 all_tokens</td></tr>
+<tr><td class="h"><a name="318"></a>318</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="319"></a>319</td><td></td><td></td><td></td><td></td><td class="s">When not being used as an iterator, the C<all_tokens> method tells</td></tr>
+<tr><td class="h"><a name="320"></a>320</td><td></td><td></td><td></td><td></td><td class="s">the Tokenizer to parse the entire file and return all of the tokens</td></tr>
+<tr><td class="h"><a name="321"></a>321</td><td></td><td></td><td></td><td></td><td class="s">in a single ARRAY reference.</td></tr>
+<tr><td class="h"><a name="322"></a>322</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="323"></a>323</td><td></td><td></td><td></td><td></td><td class="s">It should be noted that C<all_tokens> does B<NOT> interfere with the</td></tr>
+<tr><td class="h"><a name="324"></a>324</td><td></td><td></td><td></td><td></td><td class="s">use of the Tokenizer object as an iterator (does not modify the token</td></tr>
+<tr><td class="h"><a name="325"></a>325</td><td></td><td></td><td></td><td></td><td class="s">cursor) and use of the two different mechanisms can be mixed safely.</td></tr>
+<tr><td class="h"><a name="326"></a>326</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="327"></a>327</td><td></td><td></td><td></td><td></td><td class="s">Returns a reference to an ARRAY of L<PPI::Token> objects on success</td></tr>
+<tr><td class="h"><a name="328"></a>328</td><td></td><td></td><td></td><td></td><td class="s">or throws an exception on error.</td></tr>
+<tr><td class="h"><a name="329"></a>329</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="330"></a>330</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr>
+<tr><td class="h"><a name="331"></a>331</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="332"></a>332</td><td></td><td></td><td></td><td></td><td class="s">sub all_tokens {</td></tr>
+<tr><td class="h"><a name="333"></a>333</td><td></td><td></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="334"></a>334</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="335"></a>335</td><td></td><td></td><td></td><td></td><td class="s"> # Catch exceptions and return undef, so that we</td></tr>
+<tr><td class="h"><a name="336"></a>336</td><td></td><td></td><td></td><td></td><td class="s"> # can start to convert code to exception-based code.</td></tr>
+<tr><td class="h"><a name="337"></a>337</td><td></td><td></td><td></td><td></td><td class="s"> eval {</td></tr>
+<tr><td class="h"><a name="338"></a>338</td><td></td><td></td><td></td><td></td><td class="s"> # Process lines until we get EOF</td></tr>
+<tr><td class="h"><a name="339"></a>339</td><td></td><td></td><td></td><td></td><td class="s"> unless ( $self->{token_eof} ) {</td></tr>
+<tr><td class="h"><a name="340"></a>340</td><td></td><td></td><td></td><td></td><td class="s"> my $rv;</td></tr>
+<tr><td class="h"><a name="341"></a>341</td><td></td><td></td><td></td><td></td><td class="s"> while ( $rv = $self->_process_next_line ) {}</td></tr>
+<tr><td class="h"><a name="342"></a>342</td><td></td><td></td><td></td><td></td><td class="s"> unless ( defined $rv ) {</td></tr>
+<tr><td class="h"><a name="343"></a>343</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw("Error while processing source");</td></tr>
+<tr><td class="h"><a name="344"></a>344</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="345"></a>345</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="346"></a>346</td><td></td><td></td><td></td><td></td><td class="s"> # Clean up the end of the tokenizer</td></tr>
+<tr><td class="h"><a name="347"></a>347</td><td></td><td></td><td></td><td></td><td class="s"> $self->_clean_eof;</td></tr>
+<tr><td class="h"><a name="348"></a>348</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="349"></a>349</td><td></td><td></td><td></td><td></td><td class="s"> };</td></tr>
+<tr><td class="h"><a name="350"></a>350</td><td></td><td></td><td></td><td></td><td class="s"> if ( $@ ) {</td></tr>
+<tr><td class="h"><a name="351"></a>351</td><td></td><td></td><td></td><td></td><td class="s"> my $errstr = $@;</td></tr>
+<tr><td class="h"><a name="352"></a>352</td><td></td><td></td><td></td><td></td><td class="s"> $errstr =~ s/^(.*) at line .+$/$1/;</td></tr>
+<tr><td class="h"><a name="353"></a>353</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw( $errstr );</td></tr>
+<tr><td class="h"><a name="354"></a>354</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="355"></a>355</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="356"></a>356</td><td></td><td></td><td></td><td></td><td class="s"> # End of file, return a copy of the token array.</td></tr>
+<tr><td class="h"><a name="357"></a>357</td><td></td><td></td><td></td><td></td><td class="s"> return [ @{$self->{tokens}} ];</td></tr>
+<tr><td class="h"><a name="358"></a>358</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="359"></a>359</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="360"></a>360</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr>
+<tr><td class="h"><a name="361"></a>361</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="362"></a>362</td><td></td><td></td><td></td><td></td><td class="s">=head2 increment_cursor</td></tr>
+<tr><td class="h"><a name="363"></a>363</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="364"></a>364</td><td></td><td></td><td></td><td></td><td class="s">Although exposed as a public method, C<increment_method> is implemented</td></tr>
+<tr><td class="h"><a name="365"></a>365</td><td></td><td></td><td></td><td></td><td class="s">for expert use only, when writing lexers or other components that work</td></tr>
+<tr><td class="h"><a name="366"></a>366</td><td></td><td></td><td></td><td></td><td class="s">directly on token streams.</td></tr>
+<tr><td class="h"><a name="367"></a>367</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="368"></a>368</td><td></td><td></td><td></td><td></td><td class="s">It manually increments the token cursor forward through the file, in effect</td></tr>
+<tr><td class="h"><a name="369"></a>369</td><td></td><td></td><td></td><td></td><td class="s">"skipping" the next token.</td></tr>
+<tr><td class="h"><a name="370"></a>370</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="371"></a>371</td><td></td><td></td><td></td><td></td><td class="s">Return true if the cursor is incremented, C<0> if already at the end of</td></tr>
+<tr><td class="h"><a name="372"></a>372</td><td></td><td></td><td></td><td></td><td class="s">the file, or C<undef> on error.</td></tr>
+<tr><td class="h"><a name="373"></a>373</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="374"></a>374</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr>
+<tr><td class="h"><a name="375"></a>375</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="376"></a>376</td><td></td><td></td><td></td><td></td><td class="s">sub increment_cursor {</td></tr>
+<tr><td class="h"><a name="377"></a>377</td><td></td><td></td><td></td><td></td><td class="s"> # Do this via the get_token method, which makes sure there</td></tr>
+<tr><td class="h"><a name="378"></a>378</td><td></td><td></td><td></td><td></td><td class="s"> # is actually a token there to move to.</td></tr>
+<tr><td class="h"><a name="379"></a>379</td><td></td><td></td><td></td><td></td><td class="s"> $_[0]->get_token and 1;</td></tr>
+<tr><td class="h"><a name="380"></a>380</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="381"></a>381</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="382"></a>382</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr>
+<tr><td class="h"><a name="383"></a>383</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="384"></a>384</td><td></td><td></td><td></td><td></td><td class="s">=head2 decrement_cursor</td></tr>
+<tr><td class="h"><a name="385"></a>385</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="386"></a>386</td><td></td><td></td><td></td><td></td><td class="s">Although exposed as a public method, C<decrement_method> is implemented</td></tr>
+<tr><td class="h"><a name="387"></a>387</td><td></td><td></td><td></td><td></td><td class="s">for expert use only, when writing lexers or other components that work</td></tr>
+<tr><td class="h"><a name="388"></a>388</td><td></td><td></td><td></td><td></td><td class="s">directly on token streams.</td></tr>
+<tr><td class="h"><a name="389"></a>389</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="390"></a>390</td><td></td><td></td><td></td><td></td><td class="s">It manually decrements the token cursor backwards through the file, in</td></tr>
+<tr><td class="h"><a name="391"></a>391</td><td></td><td></td><td></td><td></td><td class="s">effect "rolling back" the token stream. And indeed that is what it is</td></tr>
+<tr><td class="h"><a name="392"></a>392</td><td></td><td></td><td></td><td></td><td class="s">primarily intended for, when the component that is consuming the token</td></tr>
+<tr><td class="h"><a name="393"></a>393</td><td></td><td></td><td></td><td></td><td class="s">stream needs to implement some sort of "roll back" feature in its use</td></tr>
+<tr><td class="h"><a name="394"></a>394</td><td></td><td></td><td></td><td></td><td class="s">of the token stream.</td></tr>
+<tr><td class="h"><a name="395"></a>395</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="396"></a>396</td><td></td><td></td><td></td><td></td><td class="s">Return true if the cursor is decremented, C<0> if already at the</td></tr>
+<tr><td class="h"><a name="397"></a>397</td><td></td><td></td><td></td><td></td><td class="s">beginning of the file, or C<undef> on error.</td></tr>
+<tr><td class="h"><a name="398"></a>398</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="399"></a>399</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr>
+<tr><td class="h"><a name="400"></a>400</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="401"></a>401</td><td></td><td></td><td></td><td></td><td class="s">sub decrement_cursor {</td></tr>
+<tr><td class="h"><a name="402"></a>402</td><td></td><td></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="403"></a>403</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="404"></a>404</td><td></td><td></td><td></td><td></td><td class="s"> # Check for the beginning of the file</td></tr>
+<tr><td class="h"><a name="405"></a>405</td><td></td><td></td><td></td><td></td><td class="s"> return 0 unless $self->{token_cursor};</td></tr>
+<tr><td class="h"><a name="406"></a>406</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="407"></a>407</td><td></td><td></td><td></td><td></td><td class="s"> # Decrement the token cursor</td></tr>
+<tr><td class="h"><a name="408"></a>408</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token_eof} = 0;</td></tr>
+<tr><td class="h"><a name="409"></a>409</td><td></td><td></td><td></td><td></td><td class="s"> --$self->{token_cursor};</td></tr>
+<tr><td class="h"><a name="410"></a>410</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="411"></a>411</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="416"></a>416</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr>
+<tr><td class="h"><a name="417"></a>417</td><td></td><td></td><td></td><td></td><td class="s"># Working With Source</td></tr>
+<tr><td class="h"><a name="418"></a>418</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="419"></a>419</td><td></td><td></td><td></td><td></td><td class="s"># Fetches the next line from the input line buffer</td></tr>
+<tr><td class="h"><a name="420"></a>420</td><td></td><td></td><td></td><td></td><td class="s"># Returns undef at EOF.</td></tr>
+<tr><td class="h"><a name="421"></a>421</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 69µs within PPI::Tokenizer::_get_line which was called 24 times, avg 3µs/call:
+# 24 times (69µs+0s) by PPI::Tokenizer::_fill_line at <a href="PPI-Tokenizer-pm-1003-block.html#443">line 443</a>, avg 3µs/call</div></div>sub _get_line {</td></tr>
+<tr><td class="h"><a name="422"></a>422</td><td class="c0">96</td><td class="c0"><span title="Avg 1µs">142µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="423"></a>423</td><td></td><td></td><td></td><td></td><td class="s"> return undef unless $self->{source}; # EOF hit previously</td></tr>
+<tr><td class="h"><a name="424"></a>424</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="425"></a>425</td><td></td><td></td><td></td><td></td><td class="s"> # Pull off the next line</td></tr>
+<tr><td class="h"><a name="426"></a>426</td><td></td><td></td><td></td><td></td><td class="s"> my $line = shift @{$self->{source}};</td></tr>
+<tr><td class="h"><a name="427"></a>427</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="428"></a>428</td><td></td><td></td><td></td><td></td><td class="s"> # Flag EOF if we hit it</td></tr>
+<tr><td class="h"><a name="429"></a>429</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = undef unless defined $line;</td></tr>
+<tr><td class="h"><a name="430"></a>430</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="431"></a>431</td><td></td><td></td><td></td><td></td><td class="s"> # Return the line (or EOF flag)</td></tr>
+<tr><td class="h"><a name="432"></a>432</td><td></td><td></td><td></td><td></td><td class="s"> return $line; # string or undef</td></tr>
+<tr><td class="h"><a name="433"></a>433</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="434"></a>434</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="435"></a>435</td><td></td><td></td><td></td><td></td><td class="s"># Fetches the next line, ready to process</td></tr>
+<tr><td class="h"><a name="436"></a>436</td><td></td><td></td><td></td><td></td><td class="s"># Returns 1 on success</td></tr>
+<tr><td class="h"><a name="437"></a>437</td><td></td><td></td><td></td><td></td><td class="s"># Returns 0 on EOF</td></tr>
+<tr><td class="h"><a name="438"></a>438</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 411µs (341+69) within PPI::Tokenizer::_fill_line which was called 24 times, avg 17µs/call:
+# 24 times (341µs+69µs) by PPI::Tokenizer::_process_next_line at <a href="PPI-Tokenizer-pm-1003-block.html#490">line 490</a>, avg 17µs/call</div></div>sub _fill_line {</td></tr>
+<tr><td class="h"><a name="439"></a>439</td><td class="c0">136</td><td class="c0"><span title="Avg 2µs">213µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="440"></a>440</td><td></td><td></td><td></td><td></td><td class="s"> my $inscan = shift;</td></tr>
+<tr><td class="h"><a name="441"></a>441</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="442"></a>442</td><td></td><td></td><td></td><td></td><td class="s"> # Get the next line</td></tr>
+<tr><td class="h"><a name="443"></a>443</td><td></td><td></td><td class="c0">24</td><td class="c3">69µs</td><td class="s"> my $line = $self->_get_line;<div class="calls"><div class="calls_out"> # spent 69µs making 24 calls to <a href="PPI-Tokenizer-pm-1003-block.html#421">PPI::Tokenizer::_get_line</a>, avg 3µs/call</div></div></td></tr>
+<tr><td class="h"><a name="444"></a>444</td><td class="c3">16</td><td class="c3"><span title="Avg 188ns">3µs</span></td><td></td><td></td><td class="s"> unless ( defined $line ) {</td></tr>
+<tr><td class="h"><a name="445"></a>445</td><td></td><td></td><td></td><td></td><td class="s"> # End of file</td></tr>
+<tr><td class="h"><a name="446"></a>446</td><td class="c0">64</td><td class="c3"><span title="Avg 792ns">51µs</span></td><td></td><td></td><td class="s"> unless ( $inscan ) {</td></tr>
+<tr><td class="h"><a name="447"></a>447</td><td></td><td></td><td></td><td></td><td class="s"> delete $self->{line};</td></tr>
+<tr><td class="h"><a name="448"></a>448</td><td></td><td></td><td></td><td></td><td class="s"> delete $self->{line_cursor};</td></tr>
+<tr><td class="h"><a name="449"></a>449</td><td></td><td></td><td></td><td></td><td class="s"> delete $self->{line_length};</td></tr>
+<tr><td class="h"><a name="450"></a>450</td><td></td><td></td><td></td><td></td><td class="s"> return 0;</td></tr>
+<tr><td class="h"><a name="451"></a>451</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="452"></a>452</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="453"></a>453</td><td></td><td></td><td></td><td></td><td class="s"> # In the scan version, just set the cursor to the end</td></tr>
+<tr><td class="h"><a name="454"></a>454</td><td></td><td></td><td></td><td></td><td class="s"> # of the line, and the rest should just cascade out.</td></tr>
+<tr><td class="h"><a name="455"></a>455</td><td></td><td></td><td></td><td></td><td class="s"> $self->{line_cursor} = $self->{line_length};</td></tr>
+<tr><td class="h"><a name="456"></a>456</td><td></td><td></td><td></td><td></td><td class="s"> return 0;</td></tr>
+<tr><td class="h"><a name="457"></a>457</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="458"></a>458</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="459"></a>459</td><td></td><td></td><td></td><td></td><td class="s"> # Populate the appropriate variables</td></tr>
+<tr><td class="h"><a name="460"></a>460</td><td></td><td></td><td></td><td></td><td class="s"> $self->{line} = $line;</td></tr>
+<tr><td class="h"><a name="461"></a>461</td><td></td><td></td><td></td><td></td><td class="s"> $self->{line_cursor} = -1;</td></tr>
+<tr><td class="h"><a name="462"></a>462</td><td></td><td></td><td></td><td></td><td class="s"> $self->{line_length} = length $line;</td></tr>
+<tr><td class="h"><a name="463"></a>463</td><td></td><td></td><td></td><td></td><td class="s"> $self->{line_count}++;</td></tr>
+<tr><td class="h"><a name="464"></a>464</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="465"></a>465</td><td></td><td></td><td></td><td></td><td class="s"> 1;</td></tr>
+<tr><td class="h"><a name="466"></a>466</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="467"></a>467</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="468"></a>468</td><td></td><td></td><td></td><td></td><td class="s"># Get the current character</td></tr>
+<tr><td class="h"><a name="469"></a>469</td><td></td><td></td><td></td><td></td><td class="s">sub _char {</td></tr>
+<tr><td class="h"><a name="470"></a>470</td><td></td><td></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="471"></a>471</td><td></td><td></td><td></td><td></td><td class="s"> substr( $self->{line}, $self->{line_cursor}, 1 );</td></tr>
+<tr><td class="h"><a name="472"></a>472</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="473"></a>473</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="478"></a>478</td><td></td><td></td><td></td><td></td><td class="s">####################################################################</td></tr>
+<tr><td class="h"><a name="479"></a>479</td><td></td><td></td><td></td><td></td><td class="s"># Per line processing methods</td></tr>
+<tr><td class="h"><a name="480"></a>480</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="481"></a>481</td><td></td><td></td><td></td><td></td><td class="s"># Processes the next line</td></tr>
+<tr><td class="h"><a name="482"></a>482</td><td></td><td></td><td></td><td></td><td class="s"># Returns 1 on success completion</td></tr>
+<tr><td class="h"><a name="483"></a>483</td><td></td><td></td><td></td><td></td><td class="s"># Returns 0 if EOF</td></tr>
+<tr><td class="h"><a name="484"></a>484</td><td></td><td></td><td></td><td></td><td class="s"># Returns undef on error</td></tr>
+<tr><td class="h"><a name="485"></a>485</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 4.47ms (433µs+4.03) within PPI::Tokenizer::_process_next_line which was called 24 times, avg 186µs/call:
+# 24 times (433µs+4.03ms) by PPI::Tokenizer::get_token at <a href="PPI-Tokenizer-pm-1003-block.html#277">line 277</a>, avg 186µs/call</div></div>sub _process_next_line {</td></tr>
+<tr><td class="h"><a name="486"></a>486</td><td class="c0">120</td><td class="c0"><span title="Avg 2µs">264µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="487"></a>487</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="488"></a>488</td><td></td><td></td><td></td><td></td><td class="s"> # Fill the line buffer</td></tr>
+<tr><td class="h"><a name="489"></a>489</td><td></td><td></td><td></td><td></td><td class="s"> my $rv;</td></tr>
+<tr><td class="h"><a name="490"></a>490</td><td class="c0">48</td><td class="c3"><span title="Avg 1µs">52µs</span></td><td class="c0">24</td><td class="c0">411µs</td><td class="s"> unless ( $rv = $self->_fill_line ) {<div class="calls"><div class="calls_out"> # spent 411µs making 24 calls to <a href="PPI-Tokenizer-pm-1003-block.html#438">PPI::Tokenizer::_fill_line</a>, avg 17µs/call</div></div></td></tr>
+<tr><td class="h"><a name="491"></a>491</td><td></td><td></td><td></td><td></td><td class="s"> return undef unless defined $rv;</td></tr>
+<tr><td class="h"><a name="492"></a>492</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="493"></a>493</td><td></td><td></td><td></td><td></td><td class="s"> # End of file, finalize last token</td></tr>
+<tr><td class="h"><a name="494"></a>494</td><td></td><td></td><td class="c1">16</td><td class="c3">47µs</td><td class="s"> $self->_finalize_token;<div class="calls"><div class="calls_out"> # spent 47µs making 16 calls to <a href="PPI-Tokenizer-pm-1003-block.html#599">PPI::Tokenizer::_finalize_token</a>, avg 3µs/call</div></div></td></tr>
+<tr><td class="h"><a name="495"></a>495</td><td></td><td></td><td></td><td></td><td class="s"> return 0;</td></tr>
+<tr><td class="h"><a name="496"></a>496</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="497"></a>497</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="498"></a>498</td><td></td><td></td><td></td><td></td><td class="s"> # Run the __TOKENIZER__on_line_start</td></tr>
+<tr><td class="h"><a name="499"></a>499</td><td></td><td></td><td class="c3">8</td><td class="c0">277µs</td><td class="s"> $rv = $self->{class}->__TOKENIZER__on_line_start( $self );<div class="calls"><div class="calls_out"> # spent 277µs making 8 calls to <a href="PPI-Token-BOM-pm-923-block.html#68">PPI::Token::BOM::__TOKENIZER__on_line_start</a>, avg 35µs/call</div></div></td></tr>
+<tr><td class="h"><a name="500"></a>500</td><td></td><td></td><td></td><td></td><td class="s"> unless ( $rv ) {</td></tr>
+<tr><td class="h"><a name="501"></a>501</td><td></td><td></td><td></td><td></td><td class="s"> # If there are no more source lines, then clean up</td></tr>
+<tr><td class="h"><a name="502"></a>502</td><td></td><td></td><td></td><td></td><td class="s"> if ( ref $self->{source} eq 'ARRAY' and ! @{$self->{source}} ) {</td></tr>
+<tr><td class="h"><a name="503"></a>503</td><td></td><td></td><td></td><td></td><td class="s"> $self->_clean_eof;</td></tr>
+<tr><td class="h"><a name="504"></a>504</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="505"></a>505</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="506"></a>506</td><td></td><td></td><td></td><td></td><td class="s"> # Defined but false means next line</td></tr>
+<tr><td class="h"><a name="507"></a>507</td><td></td><td></td><td></td><td></td><td class="s"> return 1 if defined $rv;</td></tr>
+<tr><td class="h"><a name="508"></a>508</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw("Error at line $self->{line_count}");</td></tr>
+<tr><td class="h"><a name="509"></a>509</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="510"></a>510</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="511"></a>511</td><td></td><td></td><td></td><td></td><td class="s"> # If we can't deal with the entire line, process char by char</td></tr>
+<tr><td class="h"><a name="512"></a>512</td><td></td><td></td><td class="c0">48</td><td class="c0">3.16ms</td><td class="s"> while ( $rv = $self->_process_next_char ) {}<div class="calls"><div class="calls_out"> # spent 3.16ms making 48 calls to <a href="PPI-Tokenizer-pm-1003-block.html#539">PPI::Tokenizer::_process_next_char</a>, avg 66µs/call</div></div></td></tr>
+<tr><td class="h"><a name="513"></a>513</td><td></td><td></td><td></td><td></td><td class="s"> unless ( defined $rv ) {</td></tr>
+<tr><td class="h"><a name="514"></a>514</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw("Error at line $self->{line_count}, character $self->{line_cursor}");</td></tr>
+<tr><td class="h"><a name="515"></a>515</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="516"></a>516</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="517"></a>517</td><td></td><td></td><td></td><td></td><td class="s"> # Trigger any action that needs to happen at the end of a line</td></tr>
+<tr><td class="h"><a name="518"></a>518</td><td></td><td></td><td class="c3">8</td><td class="c3">97µs</td><td class="s"> $self->{class}->__TOKENIZER__on_line_end( $self );<div class="calls"><div class="calls_out"> # spent 97µs making 8 calls to <a href="PPI-Token-Whitespace-pm-924-block.html#416">PPI::Token::Whitespace::__TOKENIZER__on_line_end</a>, avg 12µs/call</div></div></td></tr>
+<tr><td class="h"><a name="519"></a>519</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="520"></a>520</td><td></td><td></td><td></td><td></td><td class="s"> # If there are no more source lines, then clean up</td></tr>
+<tr><td class="h"><a name="521"></a>521</td><td></td><td></td><td class="c3">8</td><td class="c3">38µs</td><td class="s"> unless ( ref($self->{source}) eq 'ARRAY' and @{$self->{source}} ) {<div class="calls"><div class="calls_out"> # spent 38µs making 8 calls to <a href="PPI-Tokenizer-pm-1003-block.html#631">PPI::Tokenizer::_clean_eof</a>, avg 5µs/call</div></div></td></tr>
+<tr><td class="h"><a name="522"></a>522</td><td></td><td></td><td></td><td></td><td class="s"> return $self->_clean_eof;</td></tr>
+<tr><td class="h"><a name="523"></a>523</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="524"></a>524</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="525"></a>525</td><td></td><td></td><td></td><td></td><td class="s"> return 1;</td></tr>
+<tr><td class="h"><a name="526"></a>526</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="527"></a>527</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="532"></a>532</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr>
+<tr><td class="h"><a name="533"></a>533</td><td></td><td></td><td></td><td></td><td class="s"># Per-character processing methods</td></tr>
+<tr><td class="h"><a name="534"></a>534</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="535"></a>535</td><td></td><td></td><td></td><td></td><td class="s"># Process on a per-character basis.</td></tr>
+<tr><td class="h"><a name="536"></a>536</td><td></td><td></td><td></td><td></td><td class="s"># Note that due the the high number of times this gets</td></tr>
+<tr><td class="h"><a name="537"></a>537</td><td></td><td></td><td></td><td></td><td class="s"># called, it has been fairly heavily in-lined, so the code</td></tr>
+<tr><td class="h"><a name="538"></a>538</td><td></td><td></td><td></td><td></td><td class="s"># might look a bit ugly and duplicated.</td></tr>
+<tr><td class="h"><a name="539"></a>539</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 3.16ms (501µs+2.66) within PPI::Tokenizer::_process_next_char which was called 48 times, avg 66µs/call:
+# 48 times (501µs+2.66ms) by PPI::Tokenizer::_process_next_line at <a href="PPI-Tokenizer-pm-1003-block.html#512">line 512</a>, avg 66µs/call</div></div>sub _process_next_char {</td></tr>
+<tr><td class="h"><a name="540"></a>540</td><td class="c0">368</td><td class="c0"><span title="Avg 889ns">327µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="541"></a>541</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="542"></a>542</td><td></td><td></td><td></td><td></td><td class="s"> ### FIXME - This checks for a screwed up condition that triggers</td></tr>
+<tr><td class="h"><a name="543"></a>543</td><td></td><td></td><td></td><td></td><td class="s"> ### several warnings, amoungst other things.</td></tr>
+<tr><td class="h"><a name="544"></a>544</td><td></td><td></td><td></td><td></td><td class="s"> if ( ! defined $self->{line_cursor} or ! defined $self->{line_length} ) {</td></tr>
+<tr><td class="h"><a name="545"></a>545</td><td></td><td></td><td></td><td></td><td class="s"> # $DB::single = 1;</td></tr>
+<tr><td class="h"><a name="546"></a>546</td><td></td><td></td><td></td><td></td><td class="s"> return undef;</td></tr>
+<tr><td class="h"><a name="547"></a>547</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="548"></a>548</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="549"></a>549</td><td></td><td></td><td></td><td></td><td class="s"> # Increment the counter and check for end of line</td></tr>
+<tr><td class="h"><a name="550"></a>550</td><td></td><td></td><td></td><td></td><td class="s"> return 0 if ++$self->{line_cursor} >= $self->{line_length};</td></tr>
+<tr><td class="h"><a name="551"></a>551</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="552"></a>552</td><td></td><td></td><td></td><td></td><td class="s"> # Pass control to the token class</td></tr>
+<tr><td class="h"><a name="553"></a>553</td><td></td><td></td><td></td><td></td><td class="s"> my $result;</td></tr>
+<tr><td class="h"><a name="554"></a>554</td><td></td><td></td><td class="c0">40</td><td class="c0">1.98ms</td><td class="s"> unless ( $result = $self->{class}->__TOKENIZER__on_char( $self ) ) {<div class="calls"><div class="calls_out"> # spent 1.28ms making 24 calls to <a href="PPI-Token-Whitespace-pm-924-block.html#201">PPI::Token::Whitespace::__TOKENIZER__on_char</a>, avg 53µs/call
+ # spent 420µs making 8 calls to <a href="PPI-Token-Unknown-pm-939-block.html#48">PPI::Token::Unknown::__TOKENIZER__on_char</a>, avg 52µs/call
+ # spent 277µs making 8 calls to <a href="PPI-Token-Symbol-pm-936-block.html#160">PPI::Token::Symbol::__TOKENIZER__on_char</a>, avg 35µs/call</div></div></td></tr>
+<tr><td class="h"><a name="555"></a>555</td><td></td><td></td><td></td><td></td><td class="s"> # undef is error. 0 is "Did stuff ourself, you don't have to do anything"</td></tr>
+<tr><td class="h"><a name="556"></a>556</td><td></td><td></td><td></td><td></td><td class="s"> return defined $result ? 1 : undef;</td></tr>
+<tr><td class="h"><a name="557"></a>557</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="558"></a>558</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="559"></a>559</td><td></td><td></td><td></td><td></td><td class="s"> # We will need the value of the current character</td></tr>
+<tr><td class="h"><a name="560"></a>560</td><td></td><td></td><td></td><td></td><td class="s"> my $char = substr( $self->{line}, $self->{line_cursor}, 1 );</td></tr>
+<tr><td class="h"><a name="561"></a>561</td><td class="c3">16</td><td class="c3"><span title="Avg 1µs">24µs</span></td><td></td><td></td><td class="s"> if ( $result eq '1' ) {</td></tr>
+<tr><td class="h"><a name="562"></a>562</td><td></td><td></td><td></td><td></td><td class="s"> # If __TOKENIZER__on_char returns 1, it is signaling that it thinks that</td></tr>
+<tr><td class="h"><a name="563"></a>563</td><td></td><td></td><td></td><td></td><td class="s"> # the character is part of it.</td></tr>
+<tr><td class="h"><a name="564"></a>564</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="565"></a>565</td><td></td><td></td><td></td><td></td><td class="s"> # Add the character</td></tr>
+<tr><td class="h"><a name="566"></a>566</td><td></td><td></td><td></td><td></td><td class="s"> if ( defined $self->{token} ) {</td></tr>
+<tr><td class="h"><a name="567"></a>567</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token}->{content} .= $char;</td></tr>
+<tr><td class="h"><a name="568"></a>568</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr>
+<tr><td class="h"><a name="569"></a>569</td><td></td><td></td><td></td><td></td><td class="s"> defined($self->{token} = $self->{class}->new($char)) or return undef;</td></tr>
+<tr><td class="h"><a name="570"></a>570</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="571"></a>571</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="572"></a>572</td><td></td><td></td><td></td><td></td><td class="s"> return 1;</td></tr>
+<tr><td class="h"><a name="573"></a>573</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="574"></a>574</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="575"></a>575</td><td></td><td></td><td></td><td></td><td class="s"> # We have been provided with the name of a class</td></tr>
+<tr><td class="h"><a name="576"></a>576</td><td class="c2">24</td><td class="c2"><span title="Avg 3µs">68µs</span></td><td class="c3">8</td><td class="c0">339µs</td><td class="s"> if ( $self->{class} ne "PPI::Token::$result" ) {<div class="calls"><div class="calls_out"> # spent 339µs making 8 calls to <a href="PPI-Tokenizer-pm-1003-block.html#613">PPI::Tokenizer::_new_token</a>, avg 42µs/call</div></div></td></tr>
+<tr><td class="h"><a name="577"></a>577</td><td></td><td></td><td></td><td></td><td class="s"> # New class</td></tr>
+<tr><td class="h"><a name="578"></a>578</td><td></td><td></td><td></td><td></td><td class="s"> $self->_new_token( $result, $char );</td></tr>
+<tr><td class="h"><a name="579"></a>579</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( defined $self->{token} ) {</td></tr>
+<tr><td class="h"><a name="580"></a>580</td><td></td><td></td><td></td><td></td><td class="s"> # Same class as current</td></tr>
+<tr><td class="h"><a name="581"></a>581</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token}->{content} .= $char;</td></tr>
+<tr><td class="h"><a name="582"></a>582</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr>
+<tr><td class="h"><a name="583"></a>583</td><td></td><td></td><td></td><td></td><td class="s"> # Same class, but no current</td></tr>
+<tr><td class="h"><a name="584"></a>584</td><td></td><td></td><td class="c0">24</td><td class="c0">348µs</td><td class="s"> defined($self->{token} = $self->{class}->new($char)) or return undef;<div class="calls"><div class="calls_out"> # spent 348µs making 24 calls to <a href="PPI-Token-pm-922-block.html#85">PPI::Token::new</a>, avg 15µs/call</div></div></td></tr>
+<tr><td class="h"><a name="585"></a>585</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="586"></a>586</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="587"></a>587</td><td></td><td></td><td></td><td></td><td class="s"> 1;</td></tr>
+<tr><td class="h"><a name="588"></a>588</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="589"></a>589</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="594"></a>594</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr>
+<tr><td class="h"><a name="595"></a>595</td><td></td><td></td><td></td><td></td><td class="s"># Altering Tokens in Tokenizer</td></tr>
+<tr><td class="h"><a name="596"></a>596</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="597"></a>597</td><td></td><td></td><td></td><td></td><td class="s"># Finish the end of a token.</td></tr>
+<tr><td class="h"><a name="598"></a>598</td><td></td><td></td><td></td><td></td><td class="s"># Returns the resulting parse class as a convenience.</td></tr>
+<tr><td class="h"><a name="599"></a>599</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 185µs within PPI::Tokenizer::_finalize_token which was called 56 times, avg 3µs/call:
+# 16 times (57µs+0s) by PPI::Tokenizer::_new_token at <a href="PPI-Tokenizer-pm-1003-block.html#620">line 620</a>, avg 4µs/call
+# 16 times (47µs+0s) by PPI::Tokenizer::_process_next_line at <a href="PPI-Tokenizer-pm-1003-block.html#494">line 494</a>, avg 3µs/call
+# 8 times (30µs+0s) by PPI::Token::Symbol::__TOKENIZER__on_char at <a href="PPI-Token-Symbol-pm-936-block.html#216">line 216 of PPI/Token/Symbol.pm</a>, avg 4µs/call
+# 8 times (26µs+0s) by PPI::Token::Whitespace::__TOKENIZER__on_line_end at <a href="PPI-Token-Whitespace-pm-924-block.html#417">line 417 of PPI/Token/Whitespace.pm</a>, avg 3µs/call
+# 8 times (25µs+0s) by PPI::Token::Word::__TOKENIZER__commit at <a href="PPI-Token-Word-pm-934-block.html#539">line 539 of PPI/Token/Word.pm</a>, avg 3µs/call</div></div>sub _finalize_token {</td></tr>
+<tr><td class="h"><a name="600"></a>600</td><td class="c0">232</td><td class="c0"><span title="Avg 1µs">234µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="601"></a>601</td><td></td><td></td><td></td><td></td><td class="s"> return $self->{class} unless defined $self->{token};</td></tr>
+<tr><td class="h"><a name="602"></a>602</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="603"></a>603</td><td></td><td></td><td></td><td></td><td class="s"> # Add the token to the token buffer</td></tr>
+<tr><td class="h"><a name="604"></a>604</td><td></td><td></td><td></td><td></td><td class="s"> push @{ $self->{tokens} }, $self->{token};</td></tr>
+<tr><td class="h"><a name="605"></a>605</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token} = undef;</td></tr>
+<tr><td class="h"><a name="606"></a>606</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="607"></a>607</td><td></td><td></td><td></td><td></td><td class="s"> # Return the parse class to that of the zone we are in</td></tr>
+<tr><td class="h"><a name="608"></a>608</td><td></td><td></td><td></td><td></td><td class="s"> $self->{class} = $self->{zone};</td></tr>
+<tr><td class="h"><a name="609"></a>609</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="610"></a>610</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="611"></a>611</td><td></td><td></td><td></td><td></td><td class="s"># Creates a new token and sets it in the tokenizer</td></tr>
+<tr><td class="h"><a name="612"></a>612</td><td></td><td></td><td></td><td></td><td class="s"># The defined() in here prevent a ton of calls to PPI::Util::TRUE</td></tr>
+<tr><td class="h"><a name="613"></a>613</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 727µs (193+534) within PPI::Tokenizer::_new_token which was called 16 times, avg 45µs/call:
+# 8 times (106µs+282µs) by PPI::Token::Word::__TOKENIZER__commit at <a href="PPI-Token-Word-pm-934-block.html#533">line 533 of PPI/Token/Word.pm</a>, avg 49µs/call
+# 8 times (87µs+252µs) by PPI::Tokenizer::_process_next_char at <a href="PPI-Tokenizer-pm-1003-block.html#576">line 576</a>, avg 42µs/call</div></div>sub _new_token {</td></tr>
+<tr><td class="h"><a name="614"></a>614</td><td class="c0">96</td><td class="c0"><span title="Avg 2µs">158µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="615"></a>615</td><td></td><td></td><td></td><td></td><td class="s"> # throw PPI::Exception() unless @_;</td></tr>
+<tr><td class="h"><a name="616"></a>616</td><td></td><td></td><td></td><td></td><td class="s"> my $class = substr( $_[0], 0, 12 ) eq 'PPI::Token::'</td></tr>
+<tr><td class="h"><a name="617"></a>617</td><td></td><td></td><td></td><td></td><td class="s"> ? shift : 'PPI::Token::' . shift;</td></tr>
+<tr><td class="h"><a name="618"></a>618</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="619"></a>619</td><td></td><td></td><td></td><td></td><td class="s"> # Finalize any existing token</td></tr>
+<tr><td class="h"><a name="620"></a>620</td><td></td><td></td><td class="c1">16</td><td class="c3">57µs</td><td class="s"> $self->_finalize_token if defined $self->{token};<div class="calls"><div class="calls_out"> # spent 57µs making 16 calls to <a href="PPI-Tokenizer-pm-1003-block.html#599">PPI::Tokenizer::_finalize_token</a>, avg 4µs/call</div></div></td></tr>
+<tr><td class="h"><a name="621"></a>621</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="622"></a>622</td><td></td><td></td><td></td><td></td><td class="s"> # Create the new token and update the parse class</td></tr>
+<tr><td class="h"><a name="623"></a>623</td><td></td><td></td><td class="c1">16</td><td class="c0">477µs</td><td class="s"> defined($self->{token} = $class->new($_[0])) or PPI::Exception->throw;<div class="calls"><div class="calls_out"> # spent 477µs making 16 calls to <a href="PPI-Token-pm-922-block.html#85">PPI::Token::new</a>, avg 30µs/call</div></div></td></tr>
+<tr><td class="h"><a name="624"></a>624</td><td></td><td></td><td></td><td></td><td class="s"> $self->{class} = $class;</td></tr>
+<tr><td class="h"><a name="625"></a>625</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="626"></a>626</td><td></td><td></td><td></td><td></td><td class="s"> 1;</td></tr>
+<tr><td class="h"><a name="627"></a>627</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="628"></a>628</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="629"></a>629</td><td></td><td></td><td></td><td></td><td class="s"># At the end of the file, we need to clean up the results of the erroneous</td></tr>
+<tr><td class="h"><a name="630"></a>630</td><td></td><td></td><td></td><td></td><td class="s"># space that we inserted at the beginning of the process.</td></tr>
+<tr><td class="h"><a name="631"></a>631</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 38µs within PPI::Tokenizer::_clean_eof which was called 8 times, avg 5µs/call:
+# 8 times (38µs+0s) by PPI::Tokenizer::_process_next_line at <a href="PPI-Tokenizer-pm-1003-block.html#521">line 521</a>, avg 5µs/call</div></div>sub _clean_eof {</td></tr>
+<tr><td class="h"><a name="632"></a>632</td><td class="c0">48</td><td class="c3"><span title="Avg 902ns">43µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="633"></a>633</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="634"></a>634</td><td></td><td></td><td></td><td></td><td class="s"> # Finish any partially completed token</td></tr>
+<tr><td class="h"><a name="635"></a>635</td><td></td><td></td><td></td><td></td><td class="s"> $self->_finalize_token if $self->{token};</td></tr>
+<tr><td class="h"><a name="636"></a>636</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="637"></a>637</td><td></td><td></td><td></td><td></td><td class="s"> # Find the last token, and if it has no content, kill it.</td></tr>
+<tr><td class="h"><a name="638"></a>638</td><td></td><td></td><td></td><td></td><td class="s"> # There appears to be some evidence that such "null tokens" are</td></tr>
+<tr><td class="h"><a name="639"></a>639</td><td></td><td></td><td></td><td></td><td class="s"> # somehow getting created accidentally.</td></tr>
+<tr><td class="h"><a name="640"></a>640</td><td></td><td></td><td></td><td></td><td class="s"> my $last_token = $self->{tokens}->[ -1 ];</td></tr>
+<tr><td class="h"><a name="641"></a>641</td><td></td><td></td><td></td><td></td><td class="s"> unless ( length $last_token->{content} ) {</td></tr>
+<tr><td class="h"><a name="642"></a>642</td><td></td><td></td><td></td><td></td><td class="s"> pop @{$self->{tokens}};</td></tr>
+<tr><td class="h"><a name="643"></a>643</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="644"></a>644</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="645"></a>645</td><td></td><td></td><td></td><td></td><td class="s"> # Now, if the last character of the last token is a space we added,</td></tr>
+<tr><td class="h"><a name="646"></a>646</td><td></td><td></td><td></td><td></td><td class="s"> # chop it off, deleting the token if there's nothing else left.</td></tr>
+<tr><td class="h"><a name="647"></a>647</td><td></td><td></td><td></td><td></td><td class="s"> if ( $self->{source_eof_chop} ) {</td></tr>
+<tr><td class="h"><a name="648"></a>648</td><td></td><td></td><td></td><td></td><td class="s"> $last_token = $self->{tokens}->[ -1 ];</td></tr>
+<tr><td class="h"><a name="649"></a>649</td><td></td><td></td><td></td><td></td><td class="s"> $last_token->{content} =~ s/ $//;</td></tr>
+<tr><td class="h"><a name="650"></a>650</td><td></td><td></td><td></td><td></td><td class="s"> unless ( length $last_token->{content} ) {</td></tr>
+<tr><td class="h"><a name="651"></a>651</td><td></td><td></td><td></td><td></td><td class="s"> # Popping token</td></tr>
+<tr><td class="h"><a name="652"></a>652</td><td></td><td></td><td></td><td></td><td class="s"> pop @{$self->{tokens}};</td></tr>
+<tr><td class="h"><a name="653"></a>653</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="654"></a>654</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="655"></a>655</td><td></td><td></td><td></td><td></td><td class="s"> # The hack involving adding an extra space is now reversed, and</td></tr>
+<tr><td class="h"><a name="656"></a>656</td><td></td><td></td><td></td><td></td><td class="s"> # now nobody will ever know. The perfect crime!</td></tr>
+<tr><td class="h"><a name="657"></a>657</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_eof_chop} = '';</td></tr>
+<tr><td class="h"><a name="658"></a>658</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="659"></a>659</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="660"></a>660</td><td></td><td></td><td></td><td></td><td class="s"> 1;</td></tr>
+<tr><td class="h"><a name="661"></a>661</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="662"></a>662</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="667"></a>667</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr>
+<tr><td class="h"><a name="668"></a>668</td><td></td><td></td><td></td><td></td><td class="s"># Utility Methods</td></tr>
+<tr><td class="h"><a name="669"></a>669</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="670"></a>670</td><td></td><td></td><td></td><td></td><td class="s"># Context</td></tr>
+<tr><td class="h"><a name="671"></a>671</td><td></td><td></td><td></td><td></td><td class="s">sub _last_token {</td></tr>
+<tr><td class="h"><a name="672"></a>672</td><td></td><td></td><td></td><td></td><td class="s"> $_[0]->{tokens}->[-1];</td></tr>
+<tr><td class="h"><a name="673"></a>673</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="674"></a>674</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="675"></a>675</td><td></td><td></td><td></td><td></td><td class="s">sub _last_significant_token {</td></tr>
+<tr><td class="h"><a name="676"></a>676</td><td></td><td></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="677"></a>677</td><td></td><td></td><td></td><td></td><td class="s"> my $cursor = $#{ $self->{tokens} };</td></tr>
+<tr><td class="h"><a name="678"></a>678</td><td></td><td></td><td></td><td></td><td class="s"> while ( $cursor >= 0 ) {</td></tr>
+<tr><td class="h"><a name="679"></a>679</td><td></td><td></td><td></td><td></td><td class="s"> my $token = $self->{tokens}->[$cursor--];</td></tr>
+<tr><td class="h"><a name="680"></a>680</td><td></td><td></td><td></td><td></td><td class="s"> return $token if $token->significant;</td></tr>
+<tr><td class="h"><a name="681"></a>681</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="682"></a>682</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="683"></a>683</td><td></td><td></td><td></td><td></td><td class="s"> # Nothing...</td></tr>
+<tr><td class="h"><a name="684"></a>684</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Token::Whitespace->null;</td></tr>
+<tr><td class="h"><a name="685"></a>685</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="686"></a>686</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="687"></a>687</td><td></td><td></td><td></td><td></td><td class="s"># Get an array ref of previous significant tokens.</td></tr>
+<tr><td class="h"><a name="688"></a>688</td><td></td><td></td><td></td><td></td><td class="s"># Like _last_significant_token except it gets more than just one token</td></tr>
+<tr><td class="h"><a name="689"></a>689</td><td></td><td></td><td></td><td></td><td class="s"># Returns array ref on success.</td></tr>
+<tr><td class="h"><a name="690"></a>690</td><td></td><td></td><td></td><td></td><td class="s"># Returns 0 on not enough tokens</td></tr>
+<tr><td class="h"><a name="691"></a>691</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 286µs (122+165) within PPI::Tokenizer::_previous_significant_tokens which was called 8 times, avg 36µs/call:
+# 8 times (122µs+165µs) by PPI::Token::Word::__TOKENIZER__commit at <a href="PPI-Token-Word-pm-934-block.html#430">line 430 of PPI/Token/Word.pm</a>, avg 36µs/call</div></div>sub _previous_significant_tokens {</td></tr>
+<tr><td class="h"><a name="692"></a>692</td><td class="c0">56</td><td class="c2"><span title="Avg 1µs">72µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="693"></a>693</td><td></td><td></td><td></td><td></td><td class="s"> my $count = shift || 1;</td></tr>
+<tr><td class="h"><a name="694"></a>694</td><td></td><td></td><td></td><td></td><td class="s"> my $cursor = $#{ $self->{tokens} };</td></tr>
+<tr><td class="h"><a name="695"></a>695</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="696"></a>696</td><td></td><td></td><td></td><td></td><td class="s"> my ($token, @tokens);</td></tr>
+<tr><td class="h"><a name="697"></a>697</td><td></td><td></td><td></td><td></td><td class="s"> while ( $cursor >= 0 ) {</td></tr>
+<tr><td class="h"><a name="698"></a>698</td><td></td><td></td><td></td><td></td><td class="s"> $token = $self->{tokens}->[$cursor--];</td></tr>
+<tr><td class="h"><a name="699"></a>699</td><td></td><td></td><td></td><td></td><td class="s"> if ( $token->significant ) {</td></tr>
+<tr><td class="h"><a name="700"></a>700</td><td></td><td></td><td></td><td></td><td class="s"> push @tokens, $token;</td></tr>
+<tr><td class="h"><a name="701"></a>701</td><td></td><td></td><td></td><td></td><td class="s"> return \@tokens if scalar @tokens >= $count;</td></tr>
+<tr><td class="h"><a name="702"></a>702</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="703"></a>703</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="704"></a>704</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="705"></a>705</td><td></td><td></td><td></td><td></td><td class="s"> # Pad with empties</td></tr>
+<tr><td class="h"><a name="706"></a>706</td><td></td><td></td><td></td><td></td><td class="s"> foreach ( 1 .. ($count - scalar @tokens) ) {</td></tr>
+<tr><td class="h"><a name="707"></a>707</td><td class="c3">8</td><td class="c3"><span title="Avg 5µs">37µs</span></td><td class="c3">8</td><td class="c1">165µs</td><td class="s"> push @tokens, PPI::Token::Whitespace->null;<div class="calls"><div class="calls_out"> # spent 165µs making 8 calls to <a href="PPI-Token-Whitespace-pm-924-block.html#78">PPI::Token::Whitespace::null</a>, avg 21µs/call</div></div></td></tr>
+<tr><td class="h"><a name="708"></a>708</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr>
+<tr><td class="h"><a name="709"></a>709</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="710"></a>710</td><td></td><td></td><td></td><td></td><td class="s"> \@tokens;</td></tr>
+<tr><td class="h"><a name="711"></a>711</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="712"></a>712</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="713"></a>713</td><td class="c3">1</td><td class="c3"><span title="Avg 10µs">10µs</span></td><td></td><td></td><td class="s">my %OBVIOUS_CLASS = (</td></tr>
+<tr><td class="h"><a name="714"></a>714</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Symbol' => 'operator',</td></tr>
+<tr><td class="h"><a name="715"></a>715</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Magic' => 'operator',</td></tr>
+<tr><td class="h"><a name="716"></a>716</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Number' => 'operator',</td></tr>
+<tr><td class="h"><a name="717"></a>717</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::ArrayIndex' => 'operator',</td></tr>
+<tr><td class="h"><a name="718"></a>718</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Quote::Double' => 'operator',</td></tr>
+<tr><td class="h"><a name="719"></a>719</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Quote::Interpolate' => 'operator',</td></tr>
+<tr><td class="h"><a name="720"></a>720</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Quote::Literal' => 'operator',</td></tr>
+<tr><td class="h"><a name="721"></a>721</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Quote::Single' => 'operator',</td></tr>
+<tr><td class="h"><a name="722"></a>722</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::QuoteLike::Backtick' => 'operator',</td></tr>
+<tr><td class="h"><a name="723"></a>723</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::QuoteLike::Command' => 'operator',</td></tr>
+<tr><td class="h"><a name="724"></a>724</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::QuoteLike::Readline' => 'operator',</td></tr>
+<tr><td class="h"><a name="725"></a>725</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::QuoteLike::Regexp' => 'operator',</td></tr>
+<tr><td class="h"><a name="726"></a>726</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::QuoteLike::Words' => 'operator',</td></tr>
+<tr><td class="h"><a name="727"></a>727</td><td></td><td></td><td></td><td></td><td class="s">);</td></tr>
+<tr><td class="h"><a name="728"></a>728</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="729"></a>729</td><td class="c3">1</td><td class="c3"><span title="Avg 3µs">3µs</span></td><td></td><td></td><td class="s">my %OBVIOUS_CONTENT = (</td></tr>
+<tr><td class="h"><a name="730"></a>730</td><td></td><td></td><td></td><td></td><td class="s"> '(' => 'operand',</td></tr>
+<tr><td class="h"><a name="731"></a>731</td><td></td><td></td><td></td><td></td><td class="s"> '{' => 'operand',</td></tr>
+<tr><td class="h"><a name="732"></a>732</td><td></td><td></td><td></td><td></td><td class="s"> '[' => 'operand',</td></tr>
+<tr><td class="h"><a name="733"></a>733</td><td></td><td></td><td></td><td></td><td class="s"> ';' => 'operand',</td></tr>
+<tr><td class="h"><a name="734"></a>734</td><td></td><td></td><td></td><td></td><td class="s"> '}' => 'operator',</td></tr>
+<tr><td class="h"><a name="735"></a>735</td><td></td><td></td><td></td><td></td><td class="s">);</td></tr>
+<tr><td class="h"><a name="736"></a>736</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="737"></a>737</td><td></td><td></td><td></td><td></td><td class="s"># Try to determine operator/operand context, is possible.</td></tr>
+<tr><td class="h"><a name="738"></a>738</td><td></td><td></td><td></td><td></td><td class="s"># Returns "operator", "operand", or "" if unknown.</td></tr>
+<tr><td class="h"><a name="739"></a>739</td><td></td><td></td><td></td><td></td><td class="s">sub _opcontext {</td></tr>
+<tr><td class="h"><a name="740"></a>740</td><td></td><td></td><td></td><td></td><td class="s"> my $self = shift;</td></tr>
+<tr><td class="h"><a name="741"></a>741</td><td></td><td></td><td></td><td></td><td class="s"> my $tokens = $self->_previous_significant_tokens(1);</td></tr>
+<tr><td class="h"><a name="742"></a>742</td><td></td><td></td><td></td><td></td><td class="s"> my $p0 = $tokens->[0];</td></tr>
+<tr><td class="h"><a name="743"></a>743</td><td></td><td></td><td></td><td></td><td class="s"> my $c0 = ref $p0;</td></tr>
+<tr><td class="h"><a name="744"></a>744</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="745"></a>745</td><td></td><td></td><td></td><td></td><td class="s"> # Map the obvious cases</td></tr>
+<tr><td class="h"><a name="746"></a>746</td><td></td><td></td><td></td><td></td><td class="s"> return $OBVIOUS_CLASS{$c0} if defined $OBVIOUS_CLASS{$c0};</td></tr>
+<tr><td class="h"><a name="747"></a>747</td><td></td><td></td><td></td><td></td><td class="s"> return $OBVIOUS_CONTENT{$p0} if defined $OBVIOUS_CONTENT{$p0};</td></tr>
+<tr><td class="h"><a name="748"></a>748</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="749"></a>749</td><td></td><td></td><td></td><td></td><td class="s"> # Most of the time after an operator, we are an operand</td></tr>
+<tr><td class="h"><a name="750"></a>750</td><td></td><td></td><td></td><td></td><td class="s"> return 'operand' if $p0->isa('PPI::Token::Operator');</td></tr>
+<tr><td class="h"><a name="751"></a>751</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="752"></a>752</td><td></td><td></td><td></td><td></td><td class="s"> # If there's NOTHING, it's operand</td></tr>
+<tr><td class="h"><a name="753"></a>753</td><td></td><td></td><td></td><td></td><td class="s"> return 'operand' if $p0->content eq '';</td></tr>
+<tr><td class="h"><a name="754"></a>754</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="755"></a>755</td><td></td><td></td><td></td><td></td><td class="s"> # Otherwise, we don't know</td></tr>
+<tr><td class="h"><a name="756"></a>756</td><td></td><td></td><td></td><td></td><td class="s"> return ''</td></tr>
+<tr><td class="h"><a name="757"></a>757</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr>
+<tr><td class="h"><a name="758"></a>758</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="759"></a>759</td><td class="c3">1</td><td class="c3"><span title="Avg 12µs">12µs</span></td><td></td><td></td><td class="s">1;</td></tr>
+<tr><td class="h"><a name="760"></a>760</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="761"></a>761</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr>
+<tr><td class="h"><a name="762"></a>762</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="763"></a>763</td><td></td><td></td><td></td><td></td><td class="s">=head1 NOTES</td></tr>
+<tr><td class="h"><a name="764"></a>764</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="765"></a>765</td><td></td><td></td><td></td><td></td><td class="s">=head2 How the Tokenizer Works</td></tr>
+<tr><td class="h"><a name="766"></a>766</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="767"></a>767</td><td></td><td></td><td></td><td></td><td class="s">Understanding the Tokenizer is not for the feint-hearted. It is by far</td></tr>
+<tr><td class="h"><a name="768"></a>768</td><td></td><td></td><td></td><td></td><td class="s">the most complex and twisty piece of perl I've ever written that is actually</td></tr>
+<tr><td class="h"><a name="769"></a>769</td><td></td><td></td><td></td><td></td><td class="s">still built properly and isn't a terrible spaghetti-like mess. In fact, you</td></tr>
+<tr><td class="h"><a name="770"></a>770</td><td></td><td></td><td></td><td></td><td class="s">probably want to skip this section.</td></tr>
+<tr><td class="h"><a name="771"></a>771</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="772"></a>772</td><td></td><td></td><td></td><td></td><td class="s">But if you really want to understand, well then here goes.</td></tr>
+<tr><td class="h"><a name="773"></a>773</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="774"></a>774</td><td></td><td></td><td></td><td></td><td class="s">=head2 Source Input and Clean Up</td></tr>
+<tr><td class="h"><a name="775"></a>775</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="776"></a>776</td><td></td><td></td><td></td><td></td><td class="s">The Tokenizer starts by taking source in a variety of forms, sucking it</td></tr>
+<tr><td class="h"><a name="777"></a>777</td><td></td><td></td><td></td><td></td><td class="s">all in and merging into one big string, and doing our own internal line</td></tr>
+<tr><td class="h"><a name="778"></a>778</td><td></td><td></td><td></td><td></td><td class="s">split, using a "universal line separator" which allows the Tokenizer to</td></tr>
+<tr><td class="h"><a name="779"></a>779</td><td></td><td></td><td></td><td></td><td class="s">take source for any platform (and even supports a few known types of</td></tr>
+<tr><td class="h"><a name="780"></a>780</td><td></td><td></td><td></td><td></td><td class="s">broken newlines caused by mixed mac/pc/*nix editor screw ups).</td></tr>
+<tr><td class="h"><a name="781"></a>781</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="782"></a>782</td><td></td><td></td><td></td><td></td><td class="s">The resulting array of lines is used to feed the tokenizer, and is also</td></tr>
+<tr><td class="h"><a name="783"></a>783</td><td></td><td></td><td></td><td></td><td class="s">accessed directly by the heredoc-logic to do the line-oriented part of</td></tr>
+<tr><td class="h"><a name="784"></a>784</td><td></td><td></td><td></td><td></td><td class="s">here-doc support.</td></tr>
+<tr><td class="h"><a name="785"></a>785</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="786"></a>786</td><td></td><td></td><td></td><td></td><td class="s">=head2 Doing Things the Old Fashioned Way</td></tr>
+<tr><td class="h"><a name="787"></a>787</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="788"></a>788</td><td></td><td></td><td></td><td></td><td class="s">Due to the complexity of perl, and after 2 previously aborted parser</td></tr>
+<tr><td class="h"><a name="789"></a>789</td><td></td><td></td><td></td><td></td><td class="s">attempts, in the end the tokenizer was fashioned around a line-buffered</td></tr>
+<tr><td class="h"><a name="790"></a>790</td><td></td><td></td><td></td><td></td><td class="s">character-by-character method.</td></tr>
+<tr><td class="h"><a name="791"></a>791</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="792"></a>792</td><td></td><td></td><td></td><td></td><td class="s">That is, the Tokenizer pulls and holds a line at a time into a line buffer,</td></tr>
+<tr><td class="h"><a name="793"></a>793</td><td></td><td></td><td></td><td></td><td class="s">and then iterates a cursor along it. At each cursor position, a method is</td></tr>
+<tr><td class="h"><a name="794"></a>794</td><td></td><td></td><td></td><td></td><td class="s">called in whatever token class we are currently in, which will examine the</td></tr>
+<tr><td class="h"><a name="795"></a>795</td><td></td><td></td><td></td><td></td><td class="s">character at the current position, and handle it.</td></tr>
+<tr><td class="h"><a name="796"></a>796</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="797"></a>797</td><td></td><td></td><td></td><td></td><td class="s">As the handler methods in the various token classes are called, they</td></tr>
+<tr><td class="h"><a name="798"></a>798</td><td></td><td></td><td></td><td></td><td class="s">build up a output token array for the source code.</td></tr>
+<tr><td class="h"><a name="799"></a>799</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="800"></a>800</td><td></td><td></td><td></td><td></td><td class="s">Various parts of the Tokenizer use look-ahead, arbitrary-distance</td></tr>
+<tr><td class="h"><a name="801"></a>801</td><td></td><td></td><td></td><td></td><td class="s">look-behind (although currently the maximum is three significant tokens),</td></tr>
+<tr><td class="h"><a name="802"></a>802</td><td></td><td></td><td></td><td></td><td class="s">or both, and various other heuristic guesses.</td></tr>
+<tr><td class="h"><a name="803"></a>803</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="804"></a>804</td><td></td><td></td><td></td><td></td><td class="s">I've been told it is officially termed a I<"backtracking parser</td></tr>
+<tr><td class="h"><a name="805"></a>805</td><td></td><td></td><td></td><td></td><td class="s">with infinite lookaheads">.</td></tr>
+<tr><td class="h"><a name="806"></a>806</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="807"></a>807</td><td></td><td></td><td></td><td></td><td class="s">=head2 State Variables</td></tr>
+<tr><td class="h"><a name="808"></a>808</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="809"></a>809</td><td></td><td></td><td></td><td></td><td class="s">Aside from the current line and the character cursor, the Tokenizer</td></tr>
+<tr><td class="h"><a name="810"></a>810</td><td></td><td></td><td></td><td></td><td class="s">maintains a number of different state variables.</td></tr>
+<tr><td class="h"><a name="811"></a>811</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="812"></a>812</td><td></td><td></td><td></td><td></td><td class="s">=over</td></tr>
+<tr><td class="h"><a name="813"></a>813</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="814"></a>814</td><td></td><td></td><td></td><td></td><td class="s">=item Current Class</td></tr>
+<tr><td class="h"><a name="815"></a>815</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="816"></a>816</td><td></td><td></td><td></td><td></td><td class="s">The Tokenizer maintains the current token class at all times. Much of the</td></tr>
+<tr><td class="h"><a name="817"></a>817</td><td></td><td></td><td></td><td></td><td class="s">time is just going to be the "Whitespace" class, which is what the base of</td></tr>
+<tr><td class="h"><a name="818"></a>818</td><td></td><td></td><td></td><td></td><td class="s">a document is. As the tokenizer executes the various character handlers,</td></tr>
+<tr><td class="h"><a name="819"></a>819</td><td></td><td></td><td></td><td></td><td class="s">the class changes a lot as it moves a long. In fact, in some instances,</td></tr>
+<tr><td class="h"><a name="820"></a>820</td><td></td><td></td><td></td><td></td><td class="s">the character handler may not handle the character directly itself, but</td></tr>
+<tr><td class="h"><a name="821"></a>821</td><td></td><td></td><td></td><td></td><td class="s">rather change the "current class" and then hand off to the character</td></tr>
+<tr><td class="h"><a name="822"></a>822</td><td></td><td></td><td></td><td></td><td class="s">handler for the new class.</td></tr>
+<tr><td class="h"><a name="823"></a>823</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="824"></a>824</td><td></td><td></td><td></td><td></td><td class="s">Because of this, and some other things I'll deal with later, the number of</td></tr>
+<tr><td class="h"><a name="825"></a>825</td><td></td><td></td><td></td><td></td><td class="s">times the character handlers are called does not in fact have a direct</td></tr>
+<tr><td class="h"><a name="826"></a>826</td><td></td><td></td><td></td><td></td><td class="s">relationship to the number of actual characters in the document.</td></tr>
+<tr><td class="h"><a name="827"></a>827</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="828"></a>828</td><td></td><td></td><td></td><td></td><td class="s">=item Current Zone</td></tr>
+<tr><td class="h"><a name="829"></a>829</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="830"></a>830</td><td></td><td></td><td></td><td></td><td class="s">Rather than create a class stack to allow for infinitely nested layers of</td></tr>
+<tr><td class="h"><a name="831"></a>831</td><td></td><td></td><td></td><td></td><td class="s">classes, the Tokenizer recognises just a single layer.</td></tr>
+<tr><td class="h"><a name="832"></a>832</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="833"></a>833</td><td></td><td></td><td></td><td></td><td class="s">To put it a different way, in various parts of the file, the Tokenizer will</td></tr>
+<tr><td class="h"><a name="834"></a>834</td><td></td><td></td><td></td><td></td><td class="s">recognise different "base" or "substrate" classes. When a Token such as a</td></tr>
+<tr><td class="h"><a name="835"></a>835</td><td></td><td></td><td></td><td></td><td class="s">comment or a number is finalised by the tokenizer, it "falls back" to the</td></tr>
+<tr><td class="h"><a name="836"></a>836</td><td></td><td></td><td></td><td></td><td class="s">base state.</td></tr>
+<tr><td class="h"><a name="837"></a>837</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="838"></a>838</td><td></td><td></td><td></td><td></td><td class="s">This allows proper tokenization of special areas such as __DATA__</td></tr>
+<tr><td class="h"><a name="839"></a>839</td><td></td><td></td><td></td><td></td><td class="s">and __END__ blocks, which also contain things like comments and POD,</td></tr>
+<tr><td class="h"><a name="840"></a>840</td><td></td><td></td><td></td><td></td><td class="s">without allowing the creation of any significant Tokens inside these areas.</td></tr>
+<tr><td class="h"><a name="841"></a>841</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="842"></a>842</td><td></td><td></td><td></td><td></td><td class="s">For the main part of a document we use L<PPI::Token::Whitespace> for this,</td></tr>
+<tr><td class="h"><a name="843"></a>843</td><td></td><td></td><td></td><td></td><td class="s">with the idea being that code is "floating in a sea of whitespace".</td></tr>
+<tr><td class="h"><a name="844"></a>844</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="845"></a>845</td><td></td><td></td><td></td><td></td><td class="s">=item Current Token</td></tr>
+<tr><td class="h"><a name="846"></a>846</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="847"></a>847</td><td></td><td></td><td></td><td></td><td class="s">The final main state variable is the "current token". This is the Token</td></tr>
+<tr><td class="h"><a name="848"></a>848</td><td></td><td></td><td></td><td></td><td class="s">that is currently being built by the Tokenizer. For certain types, it</td></tr>
+<tr><td class="h"><a name="849"></a>849</td><td></td><td></td><td></td><td></td><td class="s">can be manipulated and morphed and change class quite a bit while being</td></tr>
+<tr><td class="h"><a name="850"></a>850</td><td></td><td></td><td></td><td></td><td class="s">assembled, as the Tokenizer's understanding of the token content changes.</td></tr>
+<tr><td class="h"><a name="851"></a>851</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="852"></a>852</td><td></td><td></td><td></td><td></td><td class="s">When the Tokenizer is confident that it has seen the end of the Token, it</td></tr>
+<tr><td class="h"><a name="853"></a>853</td><td></td><td></td><td></td><td></td><td class="s">will be "finalized", which adds it to the output token array and resets</td></tr>
+<tr><td class="h"><a name="854"></a>854</td><td></td><td></td><td></td><td></td><td class="s">the current class to that of the zone that we are currently in.</td></tr>
+<tr><td class="h"><a name="855"></a>855</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="856"></a>856</td><td></td><td></td><td></td><td></td><td class="s">I should also note at this point that the "current token" variable is</td></tr>
+<tr><td class="h"><a name="857"></a>857</td><td></td><td></td><td></td><td></td><td class="s">optional. The Tokenizer is capable of knowing what class it is currently</td></tr>
+<tr><td class="h"><a name="858"></a>858</td><td></td><td></td><td></td><td></td><td class="s">set to, without actually having accumulated any characters in the Token.</td></tr>
+<tr><td class="h"><a name="859"></a>859</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="860"></a>860</td><td></td><td></td><td></td><td></td><td class="s">=back</td></tr>
+<tr><td class="h"><a name="861"></a>861</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="862"></a>862</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster</td></tr>
+<tr><td class="h"><a name="863"></a>863</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="864"></a>864</td><td></td><td></td><td></td><td></td><td class="s">As I'm sure you can imagine, calling several different methods for each</td></tr>
+<tr><td class="h"><a name="865"></a>865</td><td></td><td></td><td></td><td></td><td class="s">character and running regexes and other complex heuristics made the first</td></tr>
+<tr><td class="h"><a name="866"></a>866</td><td></td><td></td><td></td><td></td><td class="s">fully working version of the tokenizer extremely slow.</td></tr>
+<tr><td class="h"><a name="867"></a>867</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="868"></a>868</td><td></td><td></td><td></td><td></td><td class="s">During testing, I created a metric to measure parsing speed called</td></tr>
+<tr><td class="h"><a name="869"></a>869</td><td></td><td></td><td></td><td></td><td class="s">LPGC, or "lines per gigacycle" . A gigacycle is simple a billion CPU</td></tr>
+<tr><td class="h"><a name="870"></a>870</td><td></td><td></td><td></td><td></td><td class="s">cycles on a typical single-core CPU, and so a Tokenizer running at</td></tr>
+<tr><td class="h"><a name="871"></a>871</td><td></td><td></td><td></td><td></td><td class="s">"1000 lines per gigacycle" should generate around 1200 lines of tokenized</td></tr>
+<tr><td class="h"><a name="872"></a>872</td><td></td><td></td><td></td><td></td><td class="s">code when running on a 1200 MHz processor.</td></tr>
+<tr><td class="h"><a name="873"></a>873</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="874"></a>874</td><td></td><td></td><td></td><td></td><td class="s">The first working version of the tokenizer ran at only 350 LPGC, so to</td></tr>
+<tr><td class="h"><a name="875"></a>875</td><td></td><td></td><td></td><td></td><td class="s">tokenize a typical large module such as L<ExtUtils::MakeMaker> took</td></tr>
+<tr><td class="h"><a name="876"></a>876</td><td></td><td></td><td></td><td></td><td class="s">10-15 seconds. This sluggishness made it unpractical for many uses.</td></tr>
+<tr><td class="h"><a name="877"></a>877</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="878"></a>878</td><td></td><td></td><td></td><td></td><td class="s">So in the current parser, there are multiple layers of optimisation</td></tr>
+<tr><td class="h"><a name="879"></a>879</td><td></td><td></td><td></td><td></td><td class="s">very carefully built in to the basic. This has brought the tokenizer</td></tr>
+<tr><td class="h"><a name="880"></a>880</td><td></td><td></td><td></td><td></td><td class="s">up to a more reasonable 1000 LPGC, at the expense of making the code</td></tr>
+<tr><td class="h"><a name="881"></a>881</td><td></td><td></td><td></td><td></td><td class="s">quite a bit twistier.</td></tr>
+<tr><td class="h"><a name="882"></a>882</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="883"></a>883</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster - Whole Line Classification</td></tr>
+<tr><td class="h"><a name="884"></a>884</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="885"></a>885</td><td></td><td></td><td></td><td></td><td class="s">The first step in the optimisation process was to add a hew handler to</td></tr>
+<tr><td class="h"><a name="886"></a>886</td><td></td><td></td><td></td><td></td><td class="s">enable several of the more basic classes (whitespace, comments) to be</td></tr>
+<tr><td class="h"><a name="887"></a>887</td><td></td><td></td><td></td><td></td><td class="s">able to be parsed a line at a time. At the start of each line, a</td></tr>
+<tr><td class="h"><a name="888"></a>888</td><td></td><td></td><td></td><td></td><td class="s">special optional handler (only supported by a few classes) is called to</td></tr>
+<tr><td class="h"><a name="889"></a>889</td><td></td><td></td><td></td><td></td><td class="s">check and see if the entire line can be parsed in one go.</td></tr>
+<tr><td class="h"><a name="890"></a>890</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="891"></a>891</td><td></td><td></td><td></td><td></td><td class="s">This is used mainly to handle things like POD, comments, empty lines,</td></tr>
+<tr><td class="h"><a name="892"></a>892</td><td></td><td></td><td></td><td></td><td class="s">and a few other minor special cases.</td></tr>
+<tr><td class="h"><a name="893"></a>893</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="894"></a>894</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster - Inlining</td></tr>
+<tr><td class="h"><a name="895"></a>895</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="896"></a>896</td><td></td><td></td><td></td><td></td><td class="s">The second stage of the optimisation involved inlining a small</td></tr>
+<tr><td class="h"><a name="897"></a>897</td><td></td><td></td><td></td><td></td><td class="s">number of critical methods that were repeated an extremely high number</td></tr>
+<tr><td class="h"><a name="898"></a>898</td><td></td><td></td><td></td><td></td><td class="s">of times. Profiling suggested that there were about 1,000,000 individual</td></tr>
+<tr><td class="h"><a name="899"></a>899</td><td></td><td></td><td></td><td></td><td class="s">method calls per gigacycle, and by cutting these by two thirds a significant</td></tr>
+<tr><td class="h"><a name="900"></a>900</td><td></td><td></td><td></td><td></td><td class="s">speed improvement was gained, in the order of about 50%.</td></tr>
+<tr><td class="h"><a name="901"></a>901</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="902"></a>902</td><td></td><td></td><td></td><td></td><td class="s">You may notice that many methods in the C<PPI::Tokenizer> code look</td></tr>
+<tr><td class="h"><a name="903"></a>903</td><td></td><td></td><td></td><td></td><td class="s">very nested and long hand. This is primarily due to this inlining.</td></tr>
+<tr><td class="h"><a name="904"></a>904</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="905"></a>905</td><td></td><td></td><td></td><td></td><td class="s">At around this time, some statistics code that existed in the early</td></tr>
+<tr><td class="h"><a name="906"></a>906</td><td></td><td></td><td></td><td></td><td class="s">versions of the parser was also removed, as it was determined that</td></tr>
+<tr><td class="h"><a name="907"></a>907</td><td></td><td></td><td></td><td></td><td class="s">it was consuming around 15% of the CPU for the entire parser, while</td></tr>
+<tr><td class="h"><a name="908"></a>908</td><td></td><td></td><td></td><td></td><td class="s">making the core more complicated.</td></tr>
+<tr><td class="h"><a name="909"></a>909</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="910"></a>910</td><td></td><td></td><td></td><td></td><td class="s">A judgment call was made that with the difficulties likely to be</td></tr>
+<tr><td class="h"><a name="911"></a>911</td><td></td><td></td><td></td><td></td><td class="s">encountered with future planned enhancements, and given the relatively</td></tr>
+<tr><td class="h"><a name="912"></a>912</td><td></td><td></td><td></td><td></td><td class="s">high cost involved, the statistics features would be removed from the</td></tr>
+<tr><td class="h"><a name="913"></a>913</td><td></td><td></td><td></td><td></td><td class="s">Tokenizer.</td></tr>
+<tr><td class="h"><a name="914"></a>914</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="915"></a>915</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster - Quote Engine</td></tr>
+<tr><td class="h"><a name="916"></a>916</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="917"></a>917</td><td></td><td></td><td></td><td></td><td class="s">Once inlining had reached diminishing returns, it became obvious from</td></tr>
+<tr><td class="h"><a name="918"></a>918</td><td></td><td></td><td></td><td></td><td class="s">the profiling results that a huge amount of time was being spent</td></tr>
+<tr><td class="h"><a name="919"></a>919</td><td></td><td></td><td></td><td></td><td class="s">stepping a char at a time though long, simple and "syntactically boring"</td></tr>
+<tr><td class="h"><a name="920"></a>920</td><td></td><td></td><td></td><td></td><td class="s">code such as comments and strings.</td></tr>
+<tr><td class="h"><a name="921"></a>921</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="922"></a>922</td><td></td><td></td><td></td><td></td><td class="s">The existing regex engine was expanded to also encompass quotes and</td></tr>
+<tr><td class="h"><a name="923"></a>923</td><td></td><td></td><td></td><td></td><td class="s">other quote-like things, and a special abstract base class was added</td></tr>
+<tr><td class="h"><a name="924"></a>924</td><td></td><td></td><td></td><td></td><td class="s">that provided a number of specialised parsing methods that would "scan</td></tr>
+<tr><td class="h"><a name="925"></a>925</td><td></td><td></td><td></td><td></td><td class="s">ahead", looking out ahead to find the end of a string, and updating</td></tr>
+<tr><td class="h"><a name="926"></a>926</td><td></td><td></td><td></td><td></td><td class="s">the cursor to leave it in a valid position for the next call.</td></tr>
+<tr><td class="h"><a name="927"></a>927</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="928"></a>928</td><td></td><td></td><td></td><td></td><td class="s">This is also the point at which the number of character handler calls began</td></tr>
+<tr><td class="h"><a name="929"></a>929</td><td></td><td></td><td></td><td></td><td class="s">to greatly differ from the number of characters. But it has been done</td></tr>
+<tr><td class="h"><a name="930"></a>930</td><td></td><td></td><td></td><td></td><td class="s">in a way that allows the parser to retain the power of the original</td></tr>
+<tr><td class="h"><a name="931"></a>931</td><td></td><td></td><td></td><td></td><td class="s">version at the critical points, while skipping through the "boring bits"</td></tr>
+<tr><td class="h"><a name="932"></a>932</td><td></td><td></td><td></td><td></td><td class="s">as needed for additional speed.</td></tr>
+<tr><td class="h"><a name="933"></a>933</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="934"></a>934</td><td></td><td></td><td></td><td></td><td class="s">The addition of this feature allowed the tokenizer to exceed 1000 LPGC</td></tr>
+<tr><td class="h"><a name="935"></a>935</td><td></td><td></td><td></td><td></td><td class="s">for the first time.</td></tr>
+<tr><td class="h"><a name="936"></a>936</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="937"></a>937</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster - The "Complete" Mechanism</td></tr>
+<tr><td class="h"><a name="938"></a>938</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="939"></a>939</td><td></td><td></td><td></td><td></td><td class="s">As it became evident that great speed increases were available by using</td></tr>
+<tr><td class="h"><a name="940"></a>940</td><td></td><td></td><td></td><td></td><td class="s">this "skipping ahead" mechanism, a new handler method was added that</td></tr>
+<tr><td class="h"><a name="941"></a>941</td><td></td><td></td><td></td><td></td><td class="s">explicitly handles the parsing of an entire token, where the structure</td></tr>
+<tr><td class="h"><a name="942"></a>942</td><td></td><td></td><td></td><td></td><td class="s">of the token is relatively simple. Tokens such as symbols fit this case,</td></tr>
+<tr><td class="h"><a name="943"></a>943</td><td></td><td></td><td></td><td></td><td class="s">as once we are passed the initial sigil and word char, we know that we</td></tr>
+<tr><td class="h"><a name="944"></a>944</td><td></td><td></td><td></td><td></td><td class="s">can skip ahead and "complete" the rest of the token much more easily.</td></tr>
+<tr><td class="h"><a name="945"></a>945</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="946"></a>946</td><td></td><td></td><td></td><td></td><td class="s">A number of these have been added for most or possibly all of the common</td></tr>
+<tr><td class="h"><a name="947"></a>947</td><td></td><td></td><td></td><td></td><td class="s">cases, with most of these "complete" handlers implemented using regular</td></tr>
+<tr><td class="h"><a name="948"></a>948</td><td></td><td></td><td></td><td></td><td class="s">expressions.</td></tr>
+<tr><td class="h"><a name="949"></a>949</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="950"></a>950</td><td></td><td></td><td></td><td></td><td class="s">In fact, so many have been added that at this point, you could arguably</td></tr>
+<tr><td class="h"><a name="951"></a>951</td><td></td><td></td><td></td><td></td><td class="s">reclassify the tokenizer as a "hybrid regex, char-by=char heuristic</td></tr>
+<tr><td class="h"><a name="952"></a>952</td><td></td><td></td><td></td><td></td><td class="s">tokenizer". More tokens are now consumed in "complete" methods in a</td></tr>
+<tr><td class="h"><a name="953"></a>953</td><td></td><td></td><td></td><td></td><td class="s">typical program than are handled by the normal char-by-char methods.</td></tr>
+<tr><td class="h"><a name="954"></a>954</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="955"></a>955</td><td></td><td></td><td></td><td></td><td class="s">Many of the these complete-handlers were implemented during the writing</td></tr>
+<tr><td class="h"><a name="956"></a>956</td><td></td><td></td><td></td><td></td><td class="s">of the Lexer, and this has allowed the full parser to maintain around</td></tr>
+<tr><td class="h"><a name="957"></a>957</td><td></td><td></td><td></td><td></td><td class="s">1000 LPGC despite the increasing weight of the Lexer.</td></tr>
+<tr><td class="h"><a name="958"></a>958</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="959"></a>959</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster - Porting To C (In Progress)</td></tr>
+<tr><td class="h"><a name="960"></a>960</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="961"></a>961</td><td></td><td></td><td></td><td></td><td class="s">While it would be extraordinarily difficult to port all of the Tokenizer</td></tr>
+<tr><td class="h"><a name="962"></a>962</td><td></td><td></td><td></td><td></td><td class="s">to C, work has started on a L<PPI::XS> "accelerator" package which acts as</td></tr>
+<tr><td class="h"><a name="963"></a>963</td><td></td><td></td><td></td><td></td><td class="s">a separate and automatically-detected add-on to the main PPI package.</td></tr>
+<tr><td class="h"><a name="964"></a>964</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="965"></a>965</td><td></td><td></td><td></td><td></td><td class="s">L<PPI::XS> implements faster versions of a variety of functions scattered</td></tr>
+<tr><td class="h"><a name="966"></a>966</td><td></td><td></td><td></td><td></td><td class="s">over the entire PPI codebase, from the Tokenizer Core, Quote Engine, and</td></tr>
+<tr><td class="h"><a name="967"></a>967</td><td></td><td></td><td></td><td></td><td class="s">various other places, and implements them identically in XS/C.</td></tr>
+<tr><td class="h"><a name="968"></a>968</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="969"></a>969</td><td></td><td></td><td></td><td></td><td class="s">In particular, the skip-ahead methods from the Quote Engine would appear</td></tr>
+<tr><td class="h"><a name="970"></a>970</td><td></td><td></td><td></td><td></td><td class="s">to be extremely amenable to being done in C, and a number of other</td></tr>
+<tr><td class="h"><a name="971"></a>971</td><td></td><td></td><td></td><td></td><td class="s">functions could be cherry-picked one at a time and implemented in C.</td></tr>
+<tr><td class="h"><a name="972"></a>972</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="973"></a>973</td><td></td><td></td><td></td><td></td><td class="s">Each method is heavily tested to ensure that the functionality is</td></tr>
+<tr><td class="h"><a name="974"></a>974</td><td></td><td></td><td></td><td></td><td class="s">identical, and a versioning mechanism is included to ensure that if a</td></tr>
+<tr><td class="h"><a name="975"></a>975</td><td></td><td></td><td></td><td></td><td class="s">function gets out of sync, L<PPI::XS> will degrade gracefully and just</td></tr>
+<tr><td class="h"><a name="976"></a>976</td><td></td><td></td><td></td><td></td><td class="s">not replace that single method.</td></tr>
+<tr><td class="h"><a name="977"></a>977</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="978"></a>978</td><td></td><td></td><td></td><td></td><td class="s">=head1 TO DO</td></tr>
+<tr><td class="h"><a name="979"></a>979</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="980"></a>980</td><td></td><td></td><td></td><td></td><td class="s">- Add an option to reset or seek the token stream...</td></tr>
+<tr><td class="h"><a name="981"></a>981</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="982"></a>982</td><td></td><td></td><td></td><td></td><td class="s">- Implement more Tokenizer functions in L<PPI::XS></td></tr>
+<tr><td class="h"><a name="983"></a>983</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="984"></a>984</td><td></td><td></td><td></td><td></td><td class="s">=head1 SUPPORT</td></tr>
+<tr><td class="h"><a name="985"></a>985</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="986"></a>986</td><td></td><td></td><td></td><td></td><td class="s">See the L<support section|PPI/SUPPORT> in the main module.</td></tr>
+<tr><td class="h"><a name="987"></a>987</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="988"></a>988</td><td></td><td></td><td></td><td></td><td class="s">=head1 AUTHOR</td></tr>
+<tr><td class="h"><a name="989"></a>989</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="990"></a>990</td><td></td><td></td><td></td><td></td><td class="s">Adam Kennedy E<lt>adamk@cpan.orgE<gt></td></tr>
+<tr><td class="h"><a name="991"></a>991</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="992"></a>992</td><td></td><td></td><td></td><td></td><td class="s">=head1 COPYRIGHT</td></tr>
+<tr><td class="h"><a name="993"></a>993</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="994"></a>994</td><td></td><td></td><td></td><td></td><td class="s">Copyright 2001 - 2011 Adam Kennedy.</td></tr>
+<tr><td class="h"><a name="995"></a>995</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="996"></a>996</td><td></td><td></td><td></td><td></td><td class="s">This program is free software; you can redistribute</td></tr>
+<tr><td class="h"><a name="997"></a>997</td><td></td><td></td><td></td><td></td><td class="s">it and/or modify it under the same terms as Perl itself.</td></tr>
+<tr><td class="h"><a name="998"></a>998</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="999"></a>999</td><td></td><td></td><td></td><td></td><td class="s">The full text of the license can be found in the</td></tr>
+<tr><td class="h"><a name="1000"></a>1000</td><td></td><td></td><td></td><td></td><td class="s">LICENSE file included with this module.</td></tr>
+<tr><td class="h"><a name="1001"></a>1001</td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="1002"></a>1002</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr>
+<tr><td class="s"><a name=""></a> </td><td></td><td></td><td></td><td></td><td class="s"></td></tr>
+<tr><td class="h"><a name="PPI__Tokenizer__CORE_match"></a></td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 22µs within PPI::Tokenizer::CORE:match which was called 16 times, avg 1µs/call:
+# 16 times (22µs+0s) by List::MoreUtils::any or PPI::Tokenizer::new at <a href="PPI-Tokenizer-pm-1003-block.html#211">line 211</a>, avg 1µs/call</div></div>sub PPI::Tokenizer::CORE:match; # opcode<br /> </td></tr>
+<tr><td class="h"><a name="PPI__Tokenizer__CORE_subst"></a></td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 70µs within PPI::Tokenizer::CORE:subst which was called 8 times, avg 9µs/call:
+# 8 times (70µs+0s) by PPI::Tokenizer::new at <a href="PPI-Tokenizer-pm-1003-block.html#186">line 186</a>, avg 9µs/call</div></div>sub PPI::Tokenizer::CORE:subst; # opcode<br /> </td></tr>
+</tbody></table></div>
+
+ <script type="text/javascript"> $(document).ready(function() {
+
+ $("#subs_table").tablesorter({
+ sortList: [[3,1]],
+ headers: {
+ 3: { sorter: 'fmt_time' },
+ 4: { sorter: 'fmt_time' }
+ }
+ });
+
+ } ); </script>
+
+ <div class="footer">Report produced by the
+ <a href="http://search.cpan.org/dist/Devel-NYTProf/">NYTProf 4.06</a>
+ Perl profiler, developed by
+ <a href="http://www.linkedin.com/in/timbunce">Tim Bunce</a> and
+ <a href="http://code.nytimes.com">Adam Kaplan</a>.
+ </div>
+ <br /><br /><br /><br /><br /><br /><br /><br /><br /><br />
+ </body></html>
\ No newline at end of file
projects / scpubgit/stemmatology.git / blobdiff