Commit | Line | Data |
8e30e889 |
1 | <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd"> |
2 | <html xmlns="http://www.w3.org/1999/xhtml"> |
3 | <!-- |
4 | This file was generated by Devel::NYTProf version 4.06 |
5 | --> |
6 | <head> |
7 | <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> |
8 | <meta http-equiv="Content-Language" content="en-us" /> |
9 | <title>Profile of PPI/Tokenizer.pm</title> |
10 | <link rel="stylesheet" type="text/css" href="style.css" /> |
11 | <script type="text/javascript" src="js/jquery-min.js"></script> |
12 | |
13 | <script type="text/javascript" src="js/jquery-tablesorter-min.js"></script> |
14 | <link rel="stylesheet" type="text/css" href="js/style-tablesorter.css" /> |
15 | <script type="text/javascript"> |
16 | // when a column is first clicked on to sort it, use descending order |
17 | // XXX doesn't seem to work (and not just because the tablesorter formatSortingOrder() is broken) |
18 | $.tablesorter.defaults.sortInitialOrder = "desc"; |
19 | // add parser through the tablesorter addParser method |
20 | $.tablesorter.addParser({ |
21 | id: 'fmt_time', // name of this parser |
22 | is: function(s) { |
23 | return false; // return false so this parser is not auto detected |
24 | }, |
25 | format: function(orig) { // format data for normalization |
26 | // console.log(orig); |
27 | val = orig.replace(/ns/,''); |
28 | if (val != orig) { return val / (1000*1000*1000); } |
29 | val = orig.replace(/µs/,''); /* XXX use µ ? */ |
30 | if (val != orig) { return val / (1000*1000); } |
31 | var val = orig.replace(/ms/,''); |
32 | if (val != orig) { return val / (1000); } |
33 | var val = orig.replace(/s/,''); |
34 | if (val != orig) { return val; } |
35 | if (orig == '0') { return orig; } |
36 | console.log('no match for fmt_time of '.concat(orig)); |
37 | return orig; |
38 | }, |
39 | type: 'numeric' // set type, either numeric or text |
40 | }); |
41 | </script> |
42 | </head> |
43 | |
44 | <body > |
45 | <div class="header" style="position: relative; overflow-x: hidden; overflow-y: hidden; z-index: 0; "> |
46 | <div class="header_back"> |
47 | <a href="index.html">← Index</a> |
48 | </div> |
49 | <div class="headerForeground" style="float: left"> |
50 | <span class="siteTitle">NYTProf Performance Profile</span> |
51 | <span class="siteSubtitle">  <span>« <span class="mode_btn mode_btn_selected">block view</span> • <span class="mode_btn"><a href="PPI-Tokenizer-pm-1003-line.html">line view</a></span> • <span class="mode_btn"><a href="PPI-Tokenizer-pm-1003-sub.html">sub view</a></span> »</span><br /> |
52 | For script/nytprof.pl |
53 | </span> |
54 | </div> |
55 | <div class="headerForeground" style="float: right; text-align: right"> |
56 | <span class="siteTitle"> </span> |
57 | <span class="siteSubtitle">Run on Thu May 31 16:49:15 2012<br />Reported on Thu May 31 16:52:33 2012</span> |
58 | </div> |
59 | <div style="position: absolute; left: 0px; top: 0%; width: 100%; height: 101%; z-index: -1; background-color: rgb(17, 136, 255); "></div> |
60 | <div style="position: absolute; left: 0px; top: 2%; width: 100%; height: 99%; z-index: -1; background-color: rgb(16, 134, 253); "></div> |
61 | <div style="position: absolute; left: 0px; top: 4%; width: 100%; height: 97%; z-index: -1; background-color: rgb(16, 133, 252); "></div> |
62 | <div style="position: absolute; left: 0px; top: 6%; width: 100%; height: 95%; z-index: -1; background-color: rgb(15, 131, 250); "></div> |
63 | <div style="position: absolute; left: 0px; top: 8%; width: 100%; height: 93%; z-index: -1; background-color: rgb(15, 130, 249); "></div> |
64 | <div style="position: absolute; left: 0px; top: 10%; width: 100%; height: 91%; z-index: -1; background-color: rgb(15, 129, 248); "></div> |
65 | <div style="position: absolute; left: 0px; top: 12%; width: 100%; height: 89%; z-index: -1; background-color: rgb(14, 127, 246); "></div> |
66 | <div style="position: absolute; left: 0px; top: 14%; width: 100%; height: 87%; z-index: -1; background-color: rgb(14, 126, 245); "></div> |
67 | <div style="position: absolute; left: 0px; top: 16%; width: 100%; height: 85%; z-index: -1; background-color: rgb(14, 125, 244); "></div> |
68 | <div style="position: absolute; left: 0px; top: 18%; width: 100%; height: 83%; z-index: -1; background-color: rgb(13, 123, 242); "></div> |
69 | <div style="position: absolute; left: 0px; top: 20%; width: 100%; height: 81%; z-index: -1; background-color: rgb(13, 122, 241); "></div> |
70 | <div style="position: absolute; left: 0px; top: 22%; width: 100%; height: 79%; z-index: -1; background-color: rgb(13, 121, 240); "></div> |
71 | <div style="position: absolute; left: 0px; top: 24%; width: 100%; height: 77%; z-index: -1; background-color: rgb(12, 119, 238); "></div> |
72 | <div style="position: absolute; left: 0px; top: 26%; width: 100%; height: 75%; z-index: -1; background-color: rgb(12, 118, 237); "></div> |
73 | <div style="position: absolute; left: 0px; top: 28%; width: 100%; height: 73%; z-index: -1; background-color: rgb(12, 116, 235); "></div> |
74 | <div style="position: absolute; left: 0px; top: 30%; width: 100%; height: 71%; z-index: -1; background-color: rgb(11, 115, 234); "></div> |
75 | <div style="position: absolute; left: 0px; top: 32%; width: 100%; height: 69%; z-index: -1; background-color: rgb(11, 114, 233); "></div> |
76 | <div style="position: absolute; left: 0px; top: 34%; width: 100%; height: 67%; z-index: -1; background-color: rgb(11, 112, 231); "></div> |
77 | <div style="position: absolute; left: 0px; top: 36%; width: 100%; height: 65%; z-index: -1; background-color: rgb(10, 111, 230); "></div> |
78 | <div style="position: absolute; left: 0px; top: 38%; width: 100%; height: 63%; z-index: -1; background-color: rgb(10, 110, 229); "></div> |
79 | <div style="position: absolute; left: 0px; top: 40%; width: 100%; height: 61%; z-index: -1; background-color: rgb(10, 108, 227); "></div> |
80 | <div style="position: absolute; left: 0px; top: 42%; width: 100%; height: 59%; z-index: -1; background-color: rgb(9, 107, 226); "></div> |
81 | <div style="position: absolute; left: 0px; top: 44%; width: 100%; height: 57%; z-index: -1; background-color: rgb(9, 106, 225); "></div> |
82 | <div style="position: absolute; left: 0px; top: 46%; width: 100%; height: 55%; z-index: -1; background-color: rgb(9, 104, 223); "></div> |
83 | <div style="position: absolute; left: 0px; top: 48%; width: 100%; height: 53%; z-index: -1; background-color: rgb(8, 103, 222); "></div> |
84 | <div style="position: absolute; left: 0px; top: 50%; width: 100%; height: 51%; z-index: -1; background-color: rgb(8, 102, 221); "></div> |
85 | <div style="position: absolute; left: 0px; top: 52%; width: 100%; height: 49%; z-index: -1; background-color: rgb(8, 100, 219); "></div> |
86 | <div style="position: absolute; left: 0px; top: 54%; width: 100%; height: 47%; z-index: -1; background-color: rgb(7, 99, 218); "></div> |
87 | <div style="position: absolute; left: 0px; top: 56%; width: 100%; height: 45%; z-index: -1; background-color: rgb(7, 97, 216); "></div> |
88 | <div style="position: absolute; left: 0px; top: 58%; width: 100%; height: 43%; z-index: -1; background-color: rgb(7, 96, 215); "></div> |
89 | <div style="position: absolute; left: 0px; top: 60%; width: 100%; height: 41%; z-index: -1; background-color: rgb(6, 95, 214); "></div> |
90 | <div style="position: absolute; left: 0px; top: 62%; width: 100%; height: 39%; z-index: -1; background-color: rgb(6, 93, 212); "></div> |
91 | <div style="position: absolute; left: 0px; top: 64%; width: 100%; height: 37%; z-index: -1; background-color: rgb(6, 92, 211); "></div> |
92 | <div style="position: absolute; left: 0px; top: 66%; width: 100%; height: 35%; z-index: -1; background-color: rgb(5, 91, 210); "></div> |
93 | <div style="position: absolute; left: 0px; top: 68%; width: 100%; height: 33%; z-index: -1; background-color: rgb(5, 89, 208); "></div> |
94 | <div style="position: absolute; left: 0px; top: 70%; width: 100%; height: 31%; z-index: -1; background-color: rgb(5, 88, 207); "></div> |
95 | <div style="position: absolute; left: 0px; top: 72%; width: 100%; height: 29%; z-index: -1; background-color: rgb(4, 87, 206); "></div> |
96 | <div style="position: absolute; left: 0px; top: 74%; width: 100%; height: 27%; z-index: -1; background-color: rgb(4, 85, 204); "></div> |
97 | <div style="position: absolute; left: 0px; top: 76%; width: 100%; height: 25%; z-index: -1; background-color: rgb(4, 84, 203); "></div> |
98 | <div style="position: absolute; left: 0px; top: 78%; width: 100%; height: 23%; z-index: -1; background-color: rgb(3, 82, 201); "></div> |
99 | <div style="position: absolute; left: 0px; top: 80%; width: 100%; height: 21%; z-index: -1; background-color: rgb(3, 81, 200); "></div> |
100 | <div style="position: absolute; left: 0px; top: 82%; width: 100%; height: 19%; z-index: -1; background-color: rgb(3, 80, 199); "></div> |
101 | <div style="position: absolute; left: 0px; top: 84%; width: 100%; height: 17%; z-index: -1; background-color: rgb(2, 78, 197); "></div> |
102 | <div style="position: absolute; left: 0px; top: 86%; width: 100%; height: 15%; z-index: -1; background-color: rgb(2, 77, 196); "></div> |
103 | <div style="position: absolute; left: 0px; top: 88%; width: 100%; height: 13%; z-index: -1; background-color: rgb(2, 76, 195); "></div> |
104 | <div style="position: absolute; left: 0px; top: 90%; width: 100%; height: 11%; z-index: -1; background-color: rgb(1, 74, 193); "></div> |
105 | <div style="position: absolute; left: 0px; top: 92%; width: 100%; height: 9%; z-index: -1; background-color: rgb(1, 73, 192); "></div> |
106 | <div style="position: absolute; left: 0px; top: 94%; width: 100%; height: 7%; z-index: -1; background-color: rgb(1, 72, 191); "></div> |
107 | <div style="position: absolute; left: 0px; top: 96%; width: 100%; height: 5%; z-index: -1; background-color: rgb(0, 70, 189); "></div> |
108 | <div style="position: absolute; left: 0px; top: 98%; width: 100%; height: 3%; z-index: -1; background-color: rgb(0, 69, 188); "></div> |
109 | <div style="position: absolute; left: 0px; top: 100%; width: 100%; height: 1%; z-index: -1; background-color: rgb(0, 68, 187); "></div> |
110 | </div> |
111 | |
112 | <div class="body_content"><br /> |
113 | <table class="file_summary"><tr><td class="h">Filename</td><td align="left"><a href="file:///Users/edenc/perl5/lib/perl5/PPI/Tokenizer.pm">/Users/edenc/perl5/lib/perl5/PPI/Tokenizer.pm</a></td></tr> |
114 | <tr><td class="h">Statements</td><td align="left">Executed 1862 statements in 4.57ms</td></tr></table> |
115 | |
116 | <table id="subs_table" border="1" cellpadding="0" class="tablesorter"> |
117 | <caption>Subroutines</caption> |
118 | <thead> |
119 | <tr> |
120 | <th>Calls</th> |
121 | <th><span title="Number of Places sub is called from">P</span></th> |
122 | <th><span title="Number of Files sub is called from">F</span></th> |
123 | <th>Exclusive<br />Time</th> |
124 | <th>Inclusive<br />Time</th> |
125 | <th>Subroutine</th> |
126 | </tr> |
127 | </thead> |
128 | <tbody> |
129 | <tr><td class="c0">48</td><td class="c3">1</td><td class="c3">1</td><td class="c0"><span title="0.0%">452µs</span></td><td class="c0"><span title="0.0%">2.66ms</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_process_next_char</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#539">_process_next_char</a></span></td></tr> |
130 | <tr><td class="c0">24</td><td class="c3">1</td><td class="c3">1</td><td class="c0"><span title="0.0%">369µs</span></td><td class="c0"><span title="0.0%">3.63ms</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_process_next_line</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#485">_process_next_line</a></span></td></tr> |
131 | <tr><td class="c0">56</td><td class="c3">1</td><td class="c3">1</td><td class="c0"><span title="0.0%">364µs</span></td><td class="c0"><span title="0.0%">4.02ms</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::get_token</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#255">get_token</a></span></td></tr> |
132 | <tr><td class="c3">8</td><td class="c3">1</td><td class="c3">1</td><td class="c0"><span title="0.0%">340µs</span></td><td class="c0"><span title="0.0%">475µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::new</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#119">new</a></span></td></tr> |
133 | <tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c0"><span title="0.0%">241µs</span></td><td class="c0"><span title="0.0%">334µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@88</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#88">BEGIN@88</a></span></td></tr> |
134 | <tr><td class="c0">24</td><td class="c3">1</td><td class="c3">1</td><td class="c1"><span title="0.0%">161µs</span></td><td class="c0"><span title="0.0%">219µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_fill_line</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#438">_fill_line</a></span></td></tr> |
135 | <tr><td class="c1">16</td><td class="c1">2</td><td class="c1">2</td><td class="c1"><span title="0.0%">159µs</span></td><td class="c0"><span title="0.0%">547µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_new_token</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#613">_new_token</a></span></td></tr> |
136 | <tr><td class="c0">56</td><td class="c0">5</td><td class="c0">4</td><td class="c1"><span title="0.0%">150µs</span></td><td class="c1"><span title="0.0%">150µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_finalize_token</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#599">_finalize_token</a></span></td></tr> |
137 | <tr><td class="c3">8</td><td class="c3">1</td><td class="c3">1</td><td class="c2"><span title="0.0%">102µs</span></td><td class="c0"><span title="0.0%">243µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_previous_significant_tokens</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#691">_previous_significant_tokens</a></span></td></tr> |
138 | <tr><td class="c3">8</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">63µs</span></td><td class="c3"><span title="0.0%">63µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::CORE:subst</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#PPI__Tokenizer__CORE_subst">CORE:subst</a> (opcode)</span></td></tr> |
139 | <tr><td class="c0">24</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">58µs</span></td><td class="c3"><span title="0.0%">58µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_get_line</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#421">_get_line</a></span></td></tr> |
140 | <tr><td class="c3">8</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">38µs</span></td><td class="c3"><span title="0.0%">38µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_clean_eof</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#631">_clean_eof</a></span></td></tr> |
141 | <tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">20µs</span></td><td class="c3"><span title="0.0%">25µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@81</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#81">BEGIN@81</a></span></td></tr> |
142 | <tr><td class="c1">16</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">20µs</span></td><td class="c3"><span title="0.0%">20µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::CORE:match</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#PPI__Tokenizer__CORE_match">CORE:match</a> (opcode)</span></td></tr> |
143 | <tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">13µs</span></td><td class="c3"><span title="0.0%">72µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@82</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#82">BEGIN@82</a></span></td></tr> |
144 | <tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">7µs</span></td><td class="c3"><span title="0.0%">27µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@90</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#90">BEGIN@90</a></span></td></tr> |
145 | <tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">5µs</span></td><td class="c3"><span title="0.0%">5µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@85</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#85">BEGIN@85</a></span></td></tr> |
146 | <tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">5µs</span></td><td class="c3"><span title="0.0%">5µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@87</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#87">BEGIN@87</a></span></td></tr> |
147 | <tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">5µs</span></td><td class="c3"><span title="0.0%">5µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@83</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#83">BEGIN@83</a></span></td></tr> |
148 | <tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">5µs</span></td><td class="c3"><span title="0.0%">5µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@84</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#84">BEGIN@84</a></span></td></tr> |
149 | <tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">5µs</span></td><td class="c3"><span title="0.0%">5µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@86</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#86">BEGIN@86</a></span></td></tr> |
150 | <tr><td class="c3">1</td><td class="c3">1</td><td class="c3">1</td><td class="c3"><span title="0.0%">4µs</span></td><td class="c3"><span title="0.0%">4µs</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::BEGIN@91</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#91">BEGIN@91</a></span></td></tr> |
151 | <tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::__ANON__[:211]</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#211">__ANON__[:211]</a></span></td></tr> |
152 | <tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_char</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#469">_char</a></span></td></tr> |
153 | <tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_last_significant_token</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#675">_last_significant_token</a></span></td></tr> |
154 | <tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_last_token</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#671">_last_token</a></span></td></tr> |
155 | <tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::_opcontext</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#739">_opcontext</a></span></td></tr> |
156 | <tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::all_tokens</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#332">all_tokens</a></span></td></tr> |
157 | <tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::decrement_cursor</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#401">decrement_cursor</a></span></td></tr> |
158 | <tr><td class="c3">0</td><td class="c3">0</td><td class="c3">0</td><td class="c3"><span title="0.0%">0s</span></td><td class="c3"><span title="0.0%">0s</span></td><td class="sub_name"><span style="display: none;">PPI::Tokenizer::::increment_cursor</span>PPI::Tokenizer::<a href="PPI-Tokenizer-pm-1003-block.html#376">increment_cursor</a></span></td></tr> |
159 | </tbody></table> |
160 | Call graph for these subroutines as a |
161 | <a href="http://en.wikipedia.org/wiki/Graphviz">Graphviz</a> |
162 | <a href="Users-edenc-perl5-lib-perl5-PPI-Tokenizer-pm.dot">dot language file</a>. |
163 | |
164 | <table border="1" cellpadding="0"> |
165 | <thead> |
166 | <tr><th>Line</th> |
167 | <th><span title="Number of statements executed">State<br />ments</span></th> |
168 | <th><span title="Time spend executing statements on the line, |
169 | excluding time spent executing statements in any called subroutines">Time<br />on line</span></th> |
170 | <th><span title="Number of subroutines calls">Calls</span></th> |
171 | <th><span title="Time spent in subroutines called (inclusive)">Time<br />in subs</span></th> |
172 | <th class="left_indent_header">Code</th> |
173 | </tr> |
174 | |
175 | </thead> |
176 | <tbody> |
177 | <tr><td class="h"><a name="1"></a>1</td><td></td><td></td><td></td><td></td><td class="s">package PPI::Tokenizer;</td></tr> |
178 | <tr><td class="h"><a name="2"></a>2</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
179 | <tr><td class="h"><a name="3"></a>3</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr> |
180 | <tr><td class="h"><a name="4"></a>4</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
181 | <tr><td class="h"><a name="5"></a>5</td><td></td><td></td><td></td><td></td><td class="s">=head1 NAME</td></tr> |
182 | <tr><td class="h"><a name="6"></a>6</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
183 | <tr><td class="h"><a name="7"></a>7</td><td></td><td></td><td></td><td></td><td class="s">PPI::Tokenizer - The Perl Document Tokenizer</td></tr> |
184 | <tr><td class="h"><a name="8"></a>8</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
185 | <tr><td class="h"><a name="9"></a>9</td><td></td><td></td><td></td><td></td><td class="s">=head1 SYNOPSIS</td></tr> |
186 | <tr><td class="h"><a name="10"></a>10</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
187 | <tr><td class="h"><a name="11"></a>11</td><td></td><td></td><td></td><td></td><td class="s"> # Create a tokenizer for a file, array or string</td></tr> |
188 | <tr><td class="h"><a name="12"></a>12</td><td></td><td></td><td></td><td></td><td class="s"> $Tokenizer = PPI::Tokenizer->new( 'filename.pl' );</td></tr> |
189 | <tr><td class="h"><a name="13"></a>13</td><td></td><td></td><td></td><td></td><td class="s"> $Tokenizer = PPI::Tokenizer->new( \@lines );</td></tr> |
190 | <tr><td class="h"><a name="14"></a>14</td><td></td><td></td><td></td><td></td><td class="s"> $Tokenizer = PPI::Tokenizer->new( \$source );</td></tr> |
191 | <tr><td class="h"><a name="15"></a>15</td><td></td><td></td><td></td><td></td><td class="s"> </td></tr> |
192 | <tr><td class="h"><a name="16"></a>16</td><td></td><td></td><td></td><td></td><td class="s"> # Return all the tokens for the document</td></tr> |
193 | <tr><td class="h"><a name="17"></a>17</td><td></td><td></td><td></td><td></td><td class="s"> my $tokens = $Tokenizer->all_tokens;</td></tr> |
194 | <tr><td class="h"><a name="18"></a>18</td><td></td><td></td><td></td><td></td><td class="s"> </td></tr> |
195 | <tr><td class="h"><a name="19"></a>19</td><td></td><td></td><td></td><td></td><td class="s"> # Or we can use it as an iterator</td></tr> |
196 | <tr><td class="h"><a name="20"></a>20</td><td></td><td></td><td></td><td></td><td class="s"> while ( my $Token = $Tokenizer->get_token ) {</td></tr> |
197 | <tr><td class="h"><a name="21"></a>21</td><td></td><td></td><td></td><td></td><td class="s"> print "Found token '$Token'\n";</td></tr> |
198 | <tr><td class="h"><a name="22"></a>22</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
199 | <tr><td class="h"><a name="23"></a>23</td><td></td><td></td><td></td><td></td><td class="s"> </td></tr> |
200 | <tr><td class="h"><a name="24"></a>24</td><td></td><td></td><td></td><td></td><td class="s"> # If we REALLY need to manually nudge the cursor, you</td></tr> |
201 | <tr><td class="h"><a name="25"></a>25</td><td></td><td></td><td></td><td></td><td class="s"> # can do that to (The lexer needs this ability to do rollbacks)</td></tr> |
202 | <tr><td class="h"><a name="26"></a>26</td><td></td><td></td><td></td><td></td><td class="s"> $is_incremented = $Tokenizer->increment_cursor;</td></tr> |
203 | <tr><td class="h"><a name="27"></a>27</td><td></td><td></td><td></td><td></td><td class="s"> $is_decremented = $Tokenizer->decrement_cursor;</td></tr> |
204 | <tr><td class="h"><a name="28"></a>28</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
205 | <tr><td class="h"><a name="29"></a>29</td><td></td><td></td><td></td><td></td><td class="s">=head1 DESCRIPTION</td></tr> |
206 | <tr><td class="h"><a name="30"></a>30</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
207 | <tr><td class="h"><a name="31"></a>31</td><td></td><td></td><td></td><td></td><td class="s">PPI::Tokenizer is the class that provides Tokenizer objects for use in</td></tr> |
208 | <tr><td class="h"><a name="32"></a>32</td><td></td><td></td><td></td><td></td><td class="s">breaking strings of Perl source code into Tokens.</td></tr> |
209 | <tr><td class="h"><a name="33"></a>33</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
210 | <tr><td class="h"><a name="34"></a>34</td><td></td><td></td><td></td><td></td><td class="s">By the time you are reading this, you probably need to know a little</td></tr> |
211 | <tr><td class="h"><a name="35"></a>35</td><td></td><td></td><td></td><td></td><td class="s">about the difference between how perl parses Perl "code" and how PPI</td></tr> |
212 | <tr><td class="h"><a name="36"></a>36</td><td></td><td></td><td></td><td></td><td class="s">parsers Perl "documents".</td></tr> |
213 | <tr><td class="h"><a name="37"></a>37</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
214 | <tr><td class="h"><a name="38"></a>38</td><td></td><td></td><td></td><td></td><td class="s">"perl" itself (the interpreter) uses a heavily modified lex specification</td></tr> |
215 | <tr><td class="h"><a name="39"></a>39</td><td></td><td></td><td></td><td></td><td class="s">to specify its parsing logic, maintains several types of state as it</td></tr> |
216 | <tr><td class="h"><a name="40"></a>40</td><td></td><td></td><td></td><td></td><td class="s">goes, and incrementally tokenizes, lexes AND EXECUTES at the same time.</td></tr> |
217 | <tr><td class="h"><a name="41"></a>41</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
218 | <tr><td class="h"><a name="42"></a>42</td><td></td><td></td><td></td><td></td><td class="s">In fact, it is provably impossible to use perl's parsing method without</td></tr> |
219 | <tr><td class="h"><a name="43"></a>43</td><td></td><td></td><td></td><td></td><td class="s">simultaneously executing code. A formal mathematical proof has been</td></tr> |
220 | <tr><td class="h"><a name="44"></a>44</td><td></td><td></td><td></td><td></td><td class="s">published demonstrating the method.</td></tr> |
221 | <tr><td class="h"><a name="45"></a>45</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
222 | <tr><td class="h"><a name="46"></a>46</td><td></td><td></td><td></td><td></td><td class="s">This is where the truism "Only perl can parse Perl" comes from.</td></tr> |
223 | <tr><td class="h"><a name="47"></a>47</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
224 | <tr><td class="h"><a name="48"></a>48</td><td></td><td></td><td></td><td></td><td class="s">PPI uses a completely different approach by abandoning the (impossible)</td></tr> |
225 | <tr><td class="h"><a name="49"></a>49</td><td></td><td></td><td></td><td></td><td class="s">ability to parse Perl the same way that the interpreter does, and instead</td></tr> |
226 | <tr><td class="h"><a name="50"></a>50</td><td></td><td></td><td></td><td></td><td class="s">parsing the source as a document, using a document structure independantly</td></tr> |
227 | <tr><td class="h"><a name="51"></a>51</td><td></td><td></td><td></td><td></td><td class="s">derived from the Perl documentation and approximating the perl interpreter</td></tr> |
228 | <tr><td class="h"><a name="52"></a>52</td><td></td><td></td><td></td><td></td><td class="s">interpretation as closely as possible.</td></tr> |
229 | <tr><td class="h"><a name="53"></a>53</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
230 | <tr><td class="h"><a name="54"></a>54</td><td></td><td></td><td></td><td></td><td class="s">It was touch and go for a long time whether we could get it close enough,</td></tr> |
231 | <tr><td class="h"><a name="55"></a>55</td><td></td><td></td><td></td><td></td><td class="s">but in the end it turned out that it could be done.</td></tr> |
232 | <tr><td class="h"><a name="56"></a>56</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
233 | <tr><td class="h"><a name="57"></a>57</td><td></td><td></td><td></td><td></td><td class="s">In this approach, the tokenizer C<PPI::Tokenizer> is implemented separately</td></tr> |
234 | <tr><td class="h"><a name="58"></a>58</td><td></td><td></td><td></td><td></td><td class="s">from the lexer L<PPI::Lexer>.</td></tr> |
235 | <tr><td class="h"><a name="59"></a>59</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
236 | <tr><td class="h"><a name="60"></a>60</td><td></td><td></td><td></td><td></td><td class="s">The job of C<PPI::Tokenizer> is to take pure source as a string and break it</td></tr> |
237 | <tr><td class="h"><a name="61"></a>61</td><td></td><td></td><td></td><td></td><td class="s">up into a stream/set of tokens, and contains most of the "black magic" used</td></tr> |
238 | <tr><td class="h"><a name="62"></a>62</td><td></td><td></td><td></td><td></td><td class="s">in PPI. By comparison, the lexer implements a relatively straight forward</td></tr> |
239 | <tr><td class="h"><a name="63"></a>63</td><td></td><td></td><td></td><td></td><td class="s">tree structure, and has an implementation that is uncomplicated (compared</td></tr> |
240 | <tr><td class="h"><a name="64"></a>64</td><td></td><td></td><td></td><td></td><td class="s">to the insanity in the tokenizer at least).</td></tr> |
241 | <tr><td class="h"><a name="65"></a>65</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
242 | <tr><td class="h"><a name="66"></a>66</td><td></td><td></td><td></td><td></td><td class="s">The Tokenizer uses an immense amount of heuristics, guessing and cruft,</td></tr> |
243 | <tr><td class="h"><a name="67"></a>67</td><td></td><td></td><td></td><td></td><td class="s">supported by a very B<VERY> flexible internal API, but fortunately it was</td></tr> |
244 | <tr><td class="h"><a name="68"></a>68</td><td></td><td></td><td></td><td></td><td class="s">possible to largely encapsulate the black magic, so there is not a lot that</td></tr> |
245 | <tr><td class="h"><a name="69"></a>69</td><td></td><td></td><td></td><td></td><td class="s">gets exposed to people using the C<PPI::Tokenizer> itself.</td></tr> |
246 | <tr><td class="h"><a name="70"></a>70</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
247 | <tr><td class="h"><a name="71"></a>71</td><td></td><td></td><td></td><td></td><td class="s">=head1 METHODS</td></tr> |
248 | <tr><td class="h"><a name="72"></a>72</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
249 | <tr><td class="h"><a name="73"></a>73</td><td></td><td></td><td></td><td></td><td class="s">Despite the incredible complexity, the Tokenizer itself only exposes a</td></tr> |
250 | <tr><td class="h"><a name="74"></a>74</td><td></td><td></td><td></td><td></td><td class="s">relatively small number of methods, with most of the complexity implemented</td></tr> |
251 | <tr><td class="h"><a name="75"></a>75</td><td></td><td></td><td></td><td></td><td class="s">in private methods.</td></tr> |
252 | <tr><td class="h"><a name="76"></a>76</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
253 | <tr><td class="h"><a name="77"></a>77</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr> |
254 | <tr><td class="h"><a name="78"></a>78</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
255 | <tr><td class="h"><a name="79"></a>79</td><td></td><td></td><td></td><td></td><td class="s"># Make sure everything we need is loaded so</td></tr> |
256 | <tr><td class="h"><a name="80"></a>80</td><td></td><td></td><td></td><td></td><td class="s"># we don't have to go and load all of PPI.</td></tr> |
257 | <tr><td class="h"><a name="81"></a>81</td><td class="c3">2</td><td class="c3"><span title="Avg 22µs">44µs</span></td><td class="c3">2</td><td class="c3">29µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 25µs (20+4) within PPI::Tokenizer::BEGIN@81 which was called: |
258 | # once (20µs+4µs) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#81">line 81</a></div></div>use strict;<div class="calls"><div class="calls_out"># spent 25µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#81">PPI::Tokenizer::BEGIN@81</a> |
259 | # spent 4µs making 1 call to <a href="strict-pm-3-block.html#34">strict::import</a></div></div></td></tr> |
260 | <tr><td class="h"><a name="82"></a>82</td><td class="c3">2</td><td class="c3"><span title="Avg 19µs">38µs</span></td><td class="c3">2</td><td class="c1">131µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 72µs (13+59) within PPI::Tokenizer::BEGIN@82 which was called: |
261 | # once (13µs+59µs) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#82">line 82</a></div></div>use Params::Util qw{_INSTANCE _SCALAR0 _ARRAY0};<div class="calls"><div class="calls_out"># spent 72µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#82">PPI::Tokenizer::BEGIN@82</a> |
262 | # spent 59µs making 1 call to <a href="Exporter-pm-8-block.html#28">Exporter::import</a></div></div></td></tr> |
263 | <tr><td class="h"><a name="83"></a>83</td><td class="c3">2</td><td class="c3"><span title="Avg 14µs">28µs</span></td><td class="c3">1</td><td class="c3">5µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 5µs within PPI::Tokenizer::BEGIN@83 which was called: |
264 | # once (5µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#83">line 83</a></div></div>use List::MoreUtils ();<div class="calls"><div class="calls_out"># spent 5µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#83">PPI::Tokenizer::BEGIN@83</a></div></div></td></tr> |
265 | <tr><td class="h"><a name="84"></a>84</td><td class="c3">2</td><td class="c3"><span title="Avg 14µs">29µs</span></td><td class="c3">1</td><td class="c3">5µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 5µs within PPI::Tokenizer::BEGIN@84 which was called: |
266 | # once (5µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#84">line 84</a></div></div>use PPI::Util ();<div class="calls"><div class="calls_out"># spent 5µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#84">PPI::Tokenizer::BEGIN@84</a></div></div></td></tr> |
267 | <tr><td class="h"><a name="85"></a>85</td><td class="c3">2</td><td class="c3"><span title="Avg 17µs">35µs</span></td><td class="c3">1</td><td class="c3">5µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 5µs within PPI::Tokenizer::BEGIN@85 which was called: |
268 | # once (5µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#85">line 85</a></div></div>use PPI::Element ();<div class="calls"><div class="calls_out"># spent 5µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#85">PPI::Tokenizer::BEGIN@85</a></div></div></td></tr> |
269 | <tr><td class="h"><a name="86"></a>86</td><td class="c3">2</td><td class="c3"><span title="Avg 15µs">30µs</span></td><td class="c3">1</td><td class="c3">5µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 5µs within PPI::Tokenizer::BEGIN@86 which was called: |
270 | # once (5µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#86">line 86</a></div></div>use PPI::Token ();<div class="calls"><div class="calls_out"># spent 5µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#86">PPI::Tokenizer::BEGIN@86</a></div></div></td></tr> |
271 | <tr><td class="h"><a name="87"></a>87</td><td class="c3">2</td><td class="c3"><span title="Avg 16µs">32µs</span></td><td class="c3">1</td><td class="c3">5µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 5µs within PPI::Tokenizer::BEGIN@87 which was called: |
272 | # once (5µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#87">line 87</a></div></div>use PPI::Exception ();<div class="calls"><div class="calls_out"># spent 5µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#87">PPI::Tokenizer::BEGIN@87</a></div></div></td></tr> |
273 | <tr><td class="h"><a name="88"></a>88</td><td class="c3">2</td><td class="c0"><span title="Avg 68µs">137µs</span></td><td class="c3">1</td><td class="c0">334µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 334µs (241+93) within PPI::Tokenizer::BEGIN@88 which was called: |
274 | # once (241µs+93µs) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#88">line 88</a></div></div>use PPI::Exception::ParserRejection ();<div class="calls"><div class="calls_out"># spent 334µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#88">PPI::Tokenizer::BEGIN@88</a></div></div></td></tr> |
275 | <tr><td class="h"><a name="89"></a>89</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
276 | <tr><td class="h"><a name="90"></a>90</td><td class="c3">2</td><td class="c3"><span title="Avg 15µs">30µs</span></td><td class="c3">2</td><td class="c3">46µs</td><td class="s"><div class="calls"><div class="calls_in"># spent 27µs (7+19) within PPI::Tokenizer::BEGIN@90 which was called: |
277 | # once (7µs+19µs) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#90">line 90</a></div></div>use vars qw{$VERSION};<div class="calls"><div class="calls_out"># spent 27µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#90">PPI::Tokenizer::BEGIN@90</a> |
278 | # spent 19µs making 1 call to <a href="vars-pm-6-block.html#10">vars::import</a></div></div></td></tr> |
279 | <tr><td class="h"><a name="91"></a>91</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 4µs within PPI::Tokenizer::BEGIN@91 which was called: |
280 | # once (4µs+0s) by PPI::BEGIN@28 at <a href="PPI-Tokenizer-pm-1003-block.html#93">line 93</a></div></div>BEGIN {</td></tr> |
281 | <tr><td class="h"><a name="92"></a>92</td><td class="c3">1</td><td class="c3"><span title="Avg 4µs">4µs</span></td><td></td><td></td><td class="s"> $VERSION = '1.215';</td></tr> |
282 | <tr><td class="h"><a name="93"></a>93</td><td class="c3">1</td><td class="c0"><span title="Avg 2.00ms">2.00ms</span></td><td class="c3">1</td><td class="c3">4µs</td><td class="s">}<div class="calls"><div class="calls_out"># spent 4µs making 1 call to <a href="PPI-Tokenizer-pm-1003-block.html#91">PPI::Tokenizer::BEGIN@91</a></div></div></td></tr> |
283 | <tr><td class="h"><a name="94"></a>94</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
284 | <tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
285 | <tr><td class="h"><a name="99"></a>99</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr> |
286 | <tr><td class="h"><a name="100"></a>100</td><td></td><td></td><td></td><td></td><td class="s"># Creation and Initialization</td></tr> |
287 | <tr><td class="h"><a name="101"></a>101</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
288 | <tr><td class="h"><a name="102"></a>102</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr> |
289 | <tr><td class="h"><a name="103"></a>103</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
290 | <tr><td class="h"><a name="104"></a>104</td><td></td><td></td><td></td><td></td><td class="s">=head2 new $file | \@lines | \$source</td></tr> |
291 | <tr><td class="h"><a name="105"></a>105</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
292 | <tr><td class="h"><a name="106"></a>106</td><td></td><td></td><td></td><td></td><td class="s">The main C<new> constructor creates a new Tokenizer object. These</td></tr> |
293 | <tr><td class="h"><a name="107"></a>107</td><td></td><td></td><td></td><td></td><td class="s">objects have no configuration parameters, and can only be used once,</td></tr> |
294 | <tr><td class="h"><a name="108"></a>108</td><td></td><td></td><td></td><td></td><td class="s">to tokenize a single perl source file.</td></tr> |
295 | <tr><td class="h"><a name="109"></a>109</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
296 | <tr><td class="h"><a name="110"></a>110</td><td></td><td></td><td></td><td></td><td class="s">It takes as argument either a normal scalar containing source code,</td></tr> |
297 | <tr><td class="h"><a name="111"></a>111</td><td></td><td></td><td></td><td></td><td class="s">a reference to a scalar containing source code, or a reference to an</td></tr> |
298 | <tr><td class="h"><a name="112"></a>112</td><td></td><td></td><td></td><td></td><td class="s">ARRAY containing newline-terminated lines of source code.</td></tr> |
299 | <tr><td class="h"><a name="113"></a>113</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
300 | <tr><td class="h"><a name="114"></a>114</td><td></td><td></td><td></td><td></td><td class="s">Returns a new C<PPI::Tokenizer> object on success, or throws a</td></tr> |
301 | <tr><td class="h"><a name="115"></a>115</td><td></td><td></td><td></td><td></td><td class="s">L<PPI::Exception> exception on error.</td></tr> |
302 | <tr><td class="h"><a name="116"></a>116</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
303 | <tr><td class="h"><a name="117"></a>117</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr> |
304 | <tr><td class="h"><a name="118"></a>118</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
305 | <tr><td class="h"><a name="119"></a>119</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 475µs (340+134) within PPI::Tokenizer::new which was called 8 times, avg 59µs/call: |
306 | # 8 times (340µs+134µs) by PPI::Lexer::lex_source at <a href="PPI-Lexer-pm-1005-block.html#191">line 191 of PPI/Lexer.pm</a>, avg 59µs/call</div></div>sub new {</td></tr> |
307 | <tr><td class="h"><a name="120"></a>120</td><td class="c0">56</td><td class="c0"><span title="Avg 4µs">222µs</span></td><td></td><td></td><td class="s"> my $class = ref($_[0]) || $_[0];</td></tr> |
308 | <tr><td class="h"><a name="121"></a>121</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
309 | <tr><td class="h"><a name="122"></a>122</td><td></td><td></td><td></td><td></td><td class="s"> # Create the empty tokenizer struct</td></tr> |
310 | <tr><td class="h"><a name="123"></a>123</td><td></td><td></td><td></td><td></td><td class="s"> my $self = bless {</td></tr> |
311 | <tr><td class="h"><a name="124"></a>124</td><td></td><td></td><td></td><td></td><td class="s"> # Source code</td></tr> |
312 | <tr><td class="h"><a name="125"></a>125</td><td></td><td></td><td></td><td></td><td class="s"> source => undef,</td></tr> |
313 | <tr><td class="h"><a name="126"></a>126</td><td></td><td></td><td></td><td></td><td class="s"> source_bytes => undef,</td></tr> |
314 | <tr><td class="h"><a name="127"></a>127</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
315 | <tr><td class="h"><a name="128"></a>128</td><td></td><td></td><td></td><td></td><td class="s"> # Line buffer</td></tr> |
316 | <tr><td class="h"><a name="129"></a>129</td><td></td><td></td><td></td><td></td><td class="s"> line => undef,</td></tr> |
317 | <tr><td class="h"><a name="130"></a>130</td><td></td><td></td><td></td><td></td><td class="s"> line_length => undef,</td></tr> |
318 | <tr><td class="h"><a name="131"></a>131</td><td></td><td></td><td></td><td></td><td class="s"> line_cursor => undef,</td></tr> |
319 | <tr><td class="h"><a name="132"></a>132</td><td></td><td></td><td></td><td></td><td class="s"> line_count => 0,</td></tr> |
320 | <tr><td class="h"><a name="133"></a>133</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
321 | <tr><td class="h"><a name="134"></a>134</td><td></td><td></td><td></td><td></td><td class="s"> # Parse state</td></tr> |
322 | <tr><td class="h"><a name="135"></a>135</td><td></td><td></td><td></td><td></td><td class="s"> token => undef,</td></tr> |
323 | <tr><td class="h"><a name="136"></a>136</td><td></td><td></td><td></td><td></td><td class="s"> class => 'PPI::Token::BOM',</td></tr> |
324 | <tr><td class="h"><a name="137"></a>137</td><td></td><td></td><td></td><td></td><td class="s"> zone => 'PPI::Token::Whitespace',</td></tr> |
325 | <tr><td class="h"><a name="138"></a>138</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
326 | <tr><td class="h"><a name="139"></a>139</td><td></td><td></td><td></td><td></td><td class="s"> # Output token buffer</td></tr> |
327 | <tr><td class="h"><a name="140"></a>140</td><td></td><td></td><td></td><td></td><td class="s"> tokens => [],</td></tr> |
328 | <tr><td class="h"><a name="141"></a>141</td><td></td><td></td><td></td><td></td><td class="s"> token_cursor => 0,</td></tr> |
329 | <tr><td class="h"><a name="142"></a>142</td><td></td><td></td><td></td><td></td><td class="s"> token_eof => 0,</td></tr> |
330 | <tr><td class="h"><a name="143"></a>143</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
331 | <tr><td class="h"><a name="144"></a>144</td><td></td><td></td><td></td><td></td><td class="s"> # Perl 6 blocks</td></tr> |
332 | <tr><td class="h"><a name="145"></a>145</td><td></td><td></td><td></td><td></td><td class="s"> perl6 => [],</td></tr> |
333 | <tr><td class="h"><a name="146"></a>146</td><td></td><td></td><td></td><td></td><td class="s"> }, $class;</td></tr> |
334 | <tr><td class="h"><a name="147"></a>147</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
335 | <tr><td class="h"><a name="148"></a>148</td><td></td><td></td><td class="c3">8</td><td class="c3">7µs</td><td class="s"> if ( ! defined $_[1] ) {<div class="calls"><div class="calls_out"> # spent 7µs making 8 calls to <a href="Params-Util-pm-70-block.html#Params__Util___SCALAR0">Params::Util::_SCALAR0</a>, avg 900ns/call</div></div></td></tr> |
336 | <tr><td class="h"><a name="149"></a>149</td><td></td><td></td><td></td><td></td><td class="s"> # We weren't given anything</td></tr> |
337 | <tr><td class="h"><a name="150"></a>150</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw("No source provided to Tokenizer");</td></tr> |
338 | <tr><td class="h"><a name="151"></a>151</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
339 | <tr><td class="h"><a name="152"></a>152</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( ! ref $_[1] ) {</td></tr> |
340 | <tr><td class="h"><a name="153"></a>153</td><td></td><td></td><td></td><td></td><td class="s"> my $source = PPI::Util::_slurp($_[1]);</td></tr> |
341 | <tr><td class="h"><a name="154"></a>154</td><td></td><td></td><td></td><td></td><td class="s"> if ( ref $source ) {</td></tr> |
342 | <tr><td class="h"><a name="155"></a>155</td><td></td><td></td><td></td><td></td><td class="s"> # Content returned by reference</td></tr> |
343 | <tr><td class="h"><a name="156"></a>156</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = $$source;</td></tr> |
344 | <tr><td class="h"><a name="157"></a>157</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr> |
345 | <tr><td class="h"><a name="158"></a>158</td><td></td><td></td><td></td><td></td><td class="s"> # Errors returned as a string</td></tr> |
346 | <tr><td class="h"><a name="159"></a>159</td><td></td><td></td><td></td><td></td><td class="s"> return( $source );</td></tr> |
347 | <tr><td class="h"><a name="160"></a>160</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
348 | <tr><td class="h"><a name="161"></a>161</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
349 | <tr><td class="h"><a name="162"></a>162</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( _SCALAR0($_[1]) ) {</td></tr> |
350 | <tr><td class="h"><a name="163"></a>163</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = ${$_[1]};</td></tr> |
351 | <tr><td class="h"><a name="164"></a>164</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
352 | <tr><td class="h"><a name="165"></a>165</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( _ARRAY0($_[1]) ) {</td></tr> |
353 | <tr><td class="h"><a name="166"></a>166</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = join '', map { "\n" } @{$_[1]};</td></tr> |
354 | <tr><td class="h"><a name="167"></a>167</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
355 | <tr><td class="h"><a name="168"></a>168</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr> |
356 | <tr><td class="h"><a name="169"></a>169</td><td></td><td></td><td></td><td></td><td class="s"> # We don't support whatever this is</td></tr> |
357 | <tr><td class="h"><a name="170"></a>170</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw(ref($_[1]) . " is not supported as a source provider");</td></tr> |
358 | <tr><td class="h"><a name="171"></a>171</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
359 | <tr><td class="h"><a name="172"></a>172</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
360 | <tr><td class="h"><a name="173"></a>173</td><td></td><td></td><td></td><td></td><td class="s"> # We can't handle a null string</td></tr> |
361 | <tr><td class="h"><a name="174"></a>174</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_bytes} = length $self->{source};</td></tr> |
362 | <tr><td class="h"><a name="175"></a>175</td><td class="c3">16</td><td class="c0"><span title="Avg 9µs">150µs</span></td><td></td><td></td><td class="s"> if ( $self->{source_bytes} > 1048576 ) {</td></tr> |
363 | <tr><td class="h"><a name="176"></a>176</td><td></td><td></td><td></td><td></td><td class="s"> # Dammit! It's ALWAYS the "Perl" modules larger than a</td></tr> |
364 | <tr><td class="h"><a name="177"></a>177</td><td></td><td></td><td></td><td></td><td class="s"> # meg that seems to blow up the Tokenizer/Lexer.</td></tr> |
365 | <tr><td class="h"><a name="178"></a>178</td><td></td><td></td><td></td><td></td><td class="s"> # Nobody actually writes real programs larger than a meg</td></tr> |
366 | <tr><td class="h"><a name="179"></a>179</td><td></td><td></td><td></td><td></td><td class="s"> # Perl::Tidy (the largest) is only 800k.</td></tr> |
367 | <tr><td class="h"><a name="180"></a>180</td><td></td><td></td><td></td><td></td><td class="s"> # It is always these idiots with massive Data::Dumper</td></tr> |
368 | <tr><td class="h"><a name="181"></a>181</td><td></td><td></td><td></td><td></td><td class="s"> # structs or huge RecDescent parser.</td></tr> |
369 | <tr><td class="h"><a name="182"></a>182</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception::ParserRejection->throw("File is too large");</td></tr> |
370 | <tr><td class="h"><a name="183"></a>183</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
371 | <tr><td class="h"><a name="184"></a>184</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( $self->{source_bytes} ) {</td></tr> |
372 | <tr><td class="h"><a name="185"></a>185</td><td></td><td></td><td></td><td></td><td class="s"> # Split on local newlines</td></tr> |
373 | <tr><td class="h"><a name="186"></a>186</td><td></td><td></td><td class="c3">8</td><td class="c3">63µs</td><td class="s"> $self->{source} =~ s/(?:\015{1,2}\012|\015|\012)/\n/g;<div class="calls"><div class="calls_out"> # spent 63µs making 8 calls to <a href="PPI-Tokenizer-pm-1003-block.html#PPI__Tokenizer__CORE_subst">PPI::Tokenizer::CORE:subst</a>, avg 8µs/call</div></div></td></tr> |
374 | <tr><td class="h"><a name="187"></a>187</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = [ split /(?<=\n)/, $self->{source} ];</td></tr> |
375 | <tr><td class="h"><a name="188"></a>188</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
376 | <tr><td class="h"><a name="189"></a>189</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr> |
377 | <tr><td class="h"><a name="190"></a>190</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = [ ];</td></tr> |
378 | <tr><td class="h"><a name="191"></a>191</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
379 | <tr><td class="h"><a name="192"></a>192</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
380 | <tr><td class="h"><a name="193"></a>193</td><td></td><td></td><td></td><td></td><td class="s"> ### EVIL</td></tr> |
381 | <tr><td class="h"><a name="194"></a>194</td><td></td><td></td><td></td><td></td><td class="s"> # I'm explaining this earlier than I should so you can understand</td></tr> |
382 | <tr><td class="h"><a name="195"></a>195</td><td></td><td></td><td></td><td></td><td class="s"> # why I'm about to do something that looks very strange. There's</td></tr> |
383 | <tr><td class="h"><a name="196"></a>196</td><td></td><td></td><td></td><td></td><td class="s"> # a problem with the Tokenizer, in that tokens tend to change</td></tr> |
384 | <tr><td class="h"><a name="197"></a>197</td><td></td><td></td><td></td><td></td><td class="s"> # classes as each letter is added, but they don't get allocated</td></tr> |
385 | <tr><td class="h"><a name="198"></a>198</td><td></td><td></td><td></td><td></td><td class="s"> # their definite final class until the "end" of the token, the</td></tr> |
386 | <tr><td class="h"><a name="199"></a>199</td><td></td><td></td><td></td><td></td><td class="s"> # detection of which occurs in about a hundred different places,</td></tr> |
387 | <tr><td class="h"><a name="200"></a>200</td><td></td><td></td><td></td><td></td><td class="s"> # all through various crufty code (that triples the speed).</td></tr> |
388 | <tr><td class="h"><a name="201"></a>201</td><td></td><td></td><td></td><td></td><td class="s"> #</td></tr> |
389 | <tr><td class="h"><a name="202"></a>202</td><td></td><td></td><td></td><td></td><td class="s"> # However, in general, this does not apply to tokens in which a</td></tr> |
390 | <tr><td class="h"><a name="203"></a>203</td><td></td><td></td><td></td><td></td><td class="s"> # whitespace character is valid, such as comments, whitespace and</td></tr> |
391 | <tr><td class="h"><a name="204"></a>204</td><td></td><td></td><td></td><td></td><td class="s"> # big strings.</td></tr> |
392 | <tr><td class="h"><a name="205"></a>205</td><td></td><td></td><td></td><td></td><td class="s"> #</td></tr> |
393 | <tr><td class="h"><a name="206"></a>206</td><td></td><td></td><td></td><td></td><td class="s"> # So what we do is add a space to the end of the source. This</td></tr> |
394 | <tr><td class="h"><a name="207"></a>207</td><td></td><td></td><td></td><td></td><td class="s"> # triggers normal "end of token" functionality for all cases. Then,</td></tr> |
395 | <tr><td class="h"><a name="208"></a>208</td><td></td><td></td><td></td><td></td><td class="s"> # once the tokenizer hits end of file, it examines the last token to</td></tr> |
396 | <tr><td class="h"><a name="209"></a>209</td><td></td><td></td><td></td><td></td><td class="s"> # manually either remove the ' ' token, or chop it off the end of</td></tr> |
397 | <tr><td class="h"><a name="210"></a>210</td><td></td><td></td><td></td><td></td><td class="s"> # a longer one in which the space would be valid.</td></tr> |
398 | <tr><td class="h"><a name="211"></a>211</td><td class="c3">8</td><td class="c0"><span title="Avg 12µs">98µs</span></td><td class="c0">24</td><td class="c3">72µs</td><td class="s"> if ( List::MoreUtils::any { /^__(?:DATA|END)__\s*$/ } @{$self->{source}} ) {<div class="calls"><div class="calls_out"> # spent 52µs making 8 calls to <a href="List-MoreUtils-pm-81-block.html#List__MoreUtils__any">List::MoreUtils::any</a>, avg 6µs/call |
399 | # spent 20µs making 16 calls to <a href="PPI-Tokenizer-pm-1003-block.html#PPI__Tokenizer__CORE_match">PPI::Tokenizer::CORE:match</a>, avg 1µs/call</div></div></td></tr> |
400 | <tr><td class="h"><a name="212"></a>212</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_eof_chop} = '';</td></tr> |
401 | <tr><td class="h"><a name="213"></a>213</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( ! defined $self->{source}->[0] ) {</td></tr> |
402 | <tr><td class="h"><a name="214"></a>214</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_eof_chop} = '';</td></tr> |
403 | <tr><td class="h"><a name="215"></a>215</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( $self->{source}->[-1] =~ /\s$/ ) {</td></tr> |
404 | <tr><td class="h"><a name="216"></a>216</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_eof_chop} = '';</td></tr> |
405 | <tr><td class="h"><a name="217"></a>217</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr> |
406 | <tr><td class="h"><a name="218"></a>218</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_eof_chop} = 1;</td></tr> |
407 | <tr><td class="h"><a name="219"></a>219</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source}->[-1] .= ' ';</td></tr> |
408 | <tr><td class="h"><a name="220"></a>220</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
409 | <tr><td class="h"><a name="221"></a>221</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
410 | <tr><td class="h"><a name="222"></a>222</td><td></td><td></td><td></td><td></td><td class="s"> $self;</td></tr> |
411 | <tr><td class="h"><a name="223"></a>223</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
412 | <tr><td class="h"><a name="224"></a>224</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
413 | <tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
414 | <tr><td class="h"><a name="229"></a>229</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr> |
415 | <tr><td class="h"><a name="230"></a>230</td><td></td><td></td><td></td><td></td><td class="s"># Main Public Methods</td></tr> |
416 | <tr><td class="h"><a name="231"></a>231</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
417 | <tr><td class="h"><a name="232"></a>232</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr> |
418 | <tr><td class="h"><a name="233"></a>233</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
419 | <tr><td class="h"><a name="234"></a>234</td><td></td><td></td><td></td><td></td><td class="s">=head2 get_token</td></tr> |
420 | <tr><td class="h"><a name="235"></a>235</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
421 | <tr><td class="h"><a name="236"></a>236</td><td></td><td></td><td></td><td></td><td class="s">When using the PPI::Tokenizer object as an iterator, the C<get_token></td></tr> |
422 | <tr><td class="h"><a name="237"></a>237</td><td></td><td></td><td></td><td></td><td class="s">method is the primary method that is used. It increments the cursor</td></tr> |
423 | <tr><td class="h"><a name="238"></a>238</td><td></td><td></td><td></td><td></td><td class="s">and returns the next Token in the output array.</td></tr> |
424 | <tr><td class="h"><a name="239"></a>239</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
425 | <tr><td class="h"><a name="240"></a>240</td><td></td><td></td><td></td><td></td><td class="s">The actual parsing of the file is done only as-needed, and a line at</td></tr> |
426 | <tr><td class="h"><a name="241"></a>241</td><td></td><td></td><td></td><td></td><td class="s">a time. When C<get_token> hits the end of the token array, it will</td></tr> |
427 | <tr><td class="h"><a name="242"></a>242</td><td></td><td></td><td></td><td></td><td class="s">cause the parser to pull in the next line and parse it, continuing</td></tr> |
428 | <tr><td class="h"><a name="243"></a>243</td><td></td><td></td><td></td><td></td><td class="s">as needed until there are more tokens on the output array that</td></tr> |
429 | <tr><td class="h"><a name="244"></a>244</td><td></td><td></td><td></td><td></td><td class="s">get_token can then return.</td></tr> |
430 | <tr><td class="h"><a name="245"></a>245</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
431 | <tr><td class="h"><a name="246"></a>246</td><td></td><td></td><td></td><td></td><td class="s">This means that a number of Tokenizer objects can be created, and</td></tr> |
432 | <tr><td class="h"><a name="247"></a>247</td><td></td><td></td><td></td><td></td><td class="s">won't consume significant CPU until you actually begin to pull tokens</td></tr> |
433 | <tr><td class="h"><a name="248"></a>248</td><td></td><td></td><td></td><td></td><td class="s">from it.</td></tr> |
434 | <tr><td class="h"><a name="249"></a>249</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
435 | <tr><td class="h"><a name="250"></a>250</td><td></td><td></td><td></td><td></td><td class="s">Return a L<PPI::Token> object on success, C<0> if the Tokenizer had</td></tr> |
436 | <tr><td class="h"><a name="251"></a>251</td><td></td><td></td><td></td><td></td><td class="s">reached the end of the file, or C<undef> on error.</td></tr> |
437 | <tr><td class="h"><a name="252"></a>252</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
438 | <tr><td class="h"><a name="253"></a>253</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr> |
439 | <tr><td class="h"><a name="254"></a>254</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
440 | <tr><td class="h"><a name="255"></a>255</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 4.02ms (364µs+3.65) within PPI::Tokenizer::get_token which was called 56 times, avg 72µs/call: |
441 | # 56 times (364µs+3.65ms) by PPI::Lexer::_get_token at <a href="PPI-Lexer-pm-1005-block.html#1413">line 1413 of PPI/Lexer.pm</a>, avg 72µs/call</div></div>sub get_token {</td></tr> |
442 | <tr><td class="h"><a name="256"></a>256</td><td class="c0">256</td><td class="c0"><span title="Avg 786ns">201µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
443 | <tr><td class="h"><a name="257"></a>257</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
444 | <tr><td class="h"><a name="258"></a>258</td><td></td><td></td><td></td><td></td><td class="s"> # Shortcut for EOF</td></tr> |
445 | <tr><td class="h"><a name="259"></a>259</td><td></td><td></td><td></td><td></td><td class="s"> if ( $self->{token_eof}</td></tr> |
446 | <tr><td class="h"><a name="260"></a>260</td><td></td><td></td><td></td><td></td><td class="s"> and $self->{token_cursor} > scalar @{$self->{tokens}}</td></tr> |
447 | <tr><td class="h"><a name="261"></a>261</td><td></td><td></td><td></td><td></td><td class="s"> ) {</td></tr> |
448 | <tr><td class="h"><a name="262"></a>262</td><td></td><td></td><td></td><td></td><td class="s"> return 0;</td></tr> |
449 | <tr><td class="h"><a name="263"></a>263</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
450 | <tr><td class="h"><a name="264"></a>264</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
451 | <tr><td class="h"><a name="265"></a>265</td><td></td><td></td><td></td><td></td><td class="s"> # Return the next token if we can</td></tr> |
452 | <tr><td class="h"><a name="266"></a>266</td><td class="c0">64</td><td class="c1"><span title="Avg 1µs">67µs</span></td><td class="c0">32</td><td class="c3">16µs</td><td class="s"> if ( my $token = $self->{tokens}->[ $self->{token_cursor} ] ) {<div class="calls"><div class="calls_out"> # spent 16µs making 32 calls to <a href="PPI-Util-pm-915-block.html#PPI__Util__TRUE">PPI::Util::TRUE</a>, avg 487ns/call</div></div></td></tr> |
453 | <tr><td class="h"><a name="267"></a>267</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token_cursor}++;</td></tr> |
454 | <tr><td class="h"><a name="268"></a>268</td><td></td><td></td><td></td><td></td><td class="s"> return $token;</td></tr> |
455 | <tr><td class="h"><a name="269"></a>269</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
456 | <tr><td class="h"><a name="270"></a>270</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
457 | <tr><td class="h"><a name="271"></a>271</td><td></td><td></td><td></td><td></td><td class="s"> my $line_rv;</td></tr> |
458 | <tr><td class="h"><a name="272"></a>272</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
459 | <tr><td class="h"><a name="273"></a>273</td><td></td><td></td><td></td><td></td><td class="s"> # Catch exceptions and return undef, so that we</td></tr> |
460 | <tr><td class="h"><a name="274"></a>274</td><td></td><td></td><td></td><td></td><td class="s"> # can start to convert code to exception-based code.</td></tr> |
461 | <tr><td class="h"><a name="275"></a>275</td><td class="c1">40</td><td class="c3"><span title="Avg 853ns">34µs</span></td><td></td><td></td><td class="s"> my $rv = eval {</td></tr> |
462 | <tr><td class="h"><a name="276"></a>276</td><td></td><td></td><td></td><td></td><td class="s"> # No token, we need to get some more</td></tr> |
463 | <tr><td class="h"><a name="277"></a>277</td><td></td><td></td><td class="c0">24</td><td class="c0">3.63ms</td><td class="s"> while ( $line_rv = $self->_process_next_line ) {<div class="calls"><div class="calls_out"> # spent 3.63ms making 24 calls to <a href="PPI-Tokenizer-pm-1003-block.html#485">PPI::Tokenizer::_process_next_line</a>, avg 151µs/call</div></div></td></tr> |
464 | <tr><td class="h"><a name="278"></a>278</td><td></td><td></td><td></td><td></td><td class="s"> # If there is something in the buffer, return it</td></tr> |
465 | <tr><td class="h"><a name="279"></a>279</td><td></td><td></td><td></td><td></td><td class="s"> # The defined() prevents a ton of calls to PPI::Util::TRUE</td></tr> |
466 | <tr><td class="h"><a name="280"></a>280</td><td class="c2">24</td><td class="c3"><span title="Avg 1µs">25µs</span></td><td></td><td></td><td class="s"> if ( defined( my $token = $self->{tokens}->[ $self->{token_cursor} ] ) ) {</td></tr> |
467 | <tr><td class="h"><a name="281"></a>281</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token_cursor}++;</td></tr> |
468 | <tr><td class="h"><a name="282"></a>282</td><td></td><td></td><td></td><td></td><td class="s"> return $token;</td></tr> |
469 | <tr><td class="h"><a name="283"></a>283</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
470 | <tr><td class="h"><a name="284"></a>284</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
471 | <tr><td class="h"><a name="285"></a>285</td><td></td><td></td><td></td><td></td><td class="s"> return undef;</td></tr> |
472 | <tr><td class="h"><a name="286"></a>286</td><td></td><td></td><td></td><td></td><td class="s"> };</td></tr> |
473 | <tr><td class="h"><a name="287"></a>287</td><td></td><td></td><td class="c3">8</td><td class="c3">6µs</td><td class="s"> if ( $@ ) {<div class="calls"><div class="calls_out"> # spent 6µs making 8 calls to <a href="PPI-Util-pm-915-block.html#PPI__Util__TRUE">PPI::Util::TRUE</a>, avg 688ns/call</div></div></td></tr> |
474 | <tr><td class="h"><a name="288"></a>288</td><td></td><td></td><td></td><td></td><td class="s"> if ( _INSTANCE($@, 'PPI::Exception') ) {</td></tr> |
475 | <tr><td class="h"><a name="289"></a>289</td><td></td><td></td><td></td><td></td><td class="s"> $@->throw;</td></tr> |
476 | <tr><td class="h"><a name="290"></a>290</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr> |
477 | <tr><td class="h"><a name="291"></a>291</td><td></td><td></td><td></td><td></td><td class="s"> my $errstr = $@;</td></tr> |
478 | <tr><td class="h"><a name="292"></a>292</td><td></td><td></td><td></td><td></td><td class="s"> $errstr =~ s/^(.*) at line .+$/$1/;</td></tr> |
479 | <tr><td class="h"><a name="293"></a>293</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw( $errstr );</td></tr> |
480 | <tr><td class="h"><a name="294"></a>294</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
481 | <tr><td class="h"><a name="295"></a>295</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( $rv ) {</td></tr> |
482 | <tr><td class="h"><a name="296"></a>296</td><td></td><td></td><td></td><td></td><td class="s"> return $rv;</td></tr> |
483 | <tr><td class="h"><a name="297"></a>297</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
484 | <tr><td class="h"><a name="298"></a>298</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
485 | <tr><td class="h"><a name="299"></a>299</td><td class="c0">48</td><td class="c3"><span title="Avg 883ns">42µs</span></td><td></td><td></td><td class="s"> if ( defined $line_rv ) {</td></tr> |
486 | <tr><td class="h"><a name="300"></a>300</td><td></td><td></td><td></td><td></td><td class="s"> # End of file, but we can still return things from the buffer</td></tr> |
487 | <tr><td class="h"><a name="301"></a>301</td><td></td><td></td><td></td><td></td><td class="s"> if ( my $token = $self->{tokens}->[ $self->{token_cursor} ] ) {</td></tr> |
488 | <tr><td class="h"><a name="302"></a>302</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token_cursor}++;</td></tr> |
489 | <tr><td class="h"><a name="303"></a>303</td><td></td><td></td><td></td><td></td><td class="s"> return $token;</td></tr> |
490 | <tr><td class="h"><a name="304"></a>304</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
491 | <tr><td class="h"><a name="305"></a>305</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
492 | <tr><td class="h"><a name="306"></a>306</td><td></td><td></td><td></td><td></td><td class="s"> # Set our token end of file flag</td></tr> |
493 | <tr><td class="h"><a name="307"></a>307</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token_eof} = 1;</td></tr> |
494 | <tr><td class="h"><a name="308"></a>308</td><td></td><td></td><td></td><td></td><td class="s"> return 0;</td></tr> |
495 | <tr><td class="h"><a name="309"></a>309</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
496 | <tr><td class="h"><a name="310"></a>310</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
497 | <tr><td class="h"><a name="311"></a>311</td><td></td><td></td><td></td><td></td><td class="s"> # Error, pass it up to our caller</td></tr> |
498 | <tr><td class="h"><a name="312"></a>312</td><td></td><td></td><td></td><td></td><td class="s"> undef;</td></tr> |
499 | <tr><td class="h"><a name="313"></a>313</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
500 | <tr><td class="h"><a name="314"></a>314</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
501 | <tr><td class="h"><a name="315"></a>315</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr> |
502 | <tr><td class="h"><a name="316"></a>316</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
503 | <tr><td class="h"><a name="317"></a>317</td><td></td><td></td><td></td><td></td><td class="s">=head2 all_tokens</td></tr> |
504 | <tr><td class="h"><a name="318"></a>318</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
505 | <tr><td class="h"><a name="319"></a>319</td><td></td><td></td><td></td><td></td><td class="s">When not being used as an iterator, the C<all_tokens> method tells</td></tr> |
506 | <tr><td class="h"><a name="320"></a>320</td><td></td><td></td><td></td><td></td><td class="s">the Tokenizer to parse the entire file and return all of the tokens</td></tr> |
507 | <tr><td class="h"><a name="321"></a>321</td><td></td><td></td><td></td><td></td><td class="s">in a single ARRAY reference.</td></tr> |
508 | <tr><td class="h"><a name="322"></a>322</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
509 | <tr><td class="h"><a name="323"></a>323</td><td></td><td></td><td></td><td></td><td class="s">It should be noted that C<all_tokens> does B<NOT> interfere with the</td></tr> |
510 | <tr><td class="h"><a name="324"></a>324</td><td></td><td></td><td></td><td></td><td class="s">use of the Tokenizer object as an iterator (does not modify the token</td></tr> |
511 | <tr><td class="h"><a name="325"></a>325</td><td></td><td></td><td></td><td></td><td class="s">cursor) and use of the two different mechanisms can be mixed safely.</td></tr> |
512 | <tr><td class="h"><a name="326"></a>326</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
513 | <tr><td class="h"><a name="327"></a>327</td><td></td><td></td><td></td><td></td><td class="s">Returns a reference to an ARRAY of L<PPI::Token> objects on success</td></tr> |
514 | <tr><td class="h"><a name="328"></a>328</td><td></td><td></td><td></td><td></td><td class="s">or throws an exception on error.</td></tr> |
515 | <tr><td class="h"><a name="329"></a>329</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
516 | <tr><td class="h"><a name="330"></a>330</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr> |
517 | <tr><td class="h"><a name="331"></a>331</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
518 | <tr><td class="h"><a name="332"></a>332</td><td></td><td></td><td></td><td></td><td class="s">sub all_tokens {</td></tr> |
519 | <tr><td class="h"><a name="333"></a>333</td><td></td><td></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
520 | <tr><td class="h"><a name="334"></a>334</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
521 | <tr><td class="h"><a name="335"></a>335</td><td></td><td></td><td></td><td></td><td class="s"> # Catch exceptions and return undef, so that we</td></tr> |
522 | <tr><td class="h"><a name="336"></a>336</td><td></td><td></td><td></td><td></td><td class="s"> # can start to convert code to exception-based code.</td></tr> |
523 | <tr><td class="h"><a name="337"></a>337</td><td></td><td></td><td></td><td></td><td class="s"> eval {</td></tr> |
524 | <tr><td class="h"><a name="338"></a>338</td><td></td><td></td><td></td><td></td><td class="s"> # Process lines until we get EOF</td></tr> |
525 | <tr><td class="h"><a name="339"></a>339</td><td></td><td></td><td></td><td></td><td class="s"> unless ( $self->{token_eof} ) {</td></tr> |
526 | <tr><td class="h"><a name="340"></a>340</td><td></td><td></td><td></td><td></td><td class="s"> my $rv;</td></tr> |
527 | <tr><td class="h"><a name="341"></a>341</td><td></td><td></td><td></td><td></td><td class="s"> while ( $rv = $self->_process_next_line ) {}</td></tr> |
528 | <tr><td class="h"><a name="342"></a>342</td><td></td><td></td><td></td><td></td><td class="s"> unless ( defined $rv ) {</td></tr> |
529 | <tr><td class="h"><a name="343"></a>343</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw("Error while processing source");</td></tr> |
530 | <tr><td class="h"><a name="344"></a>344</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
531 | <tr><td class="h"><a name="345"></a>345</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
532 | <tr><td class="h"><a name="346"></a>346</td><td></td><td></td><td></td><td></td><td class="s"> # Clean up the end of the tokenizer</td></tr> |
533 | <tr><td class="h"><a name="347"></a>347</td><td></td><td></td><td></td><td></td><td class="s"> $self->_clean_eof;</td></tr> |
534 | <tr><td class="h"><a name="348"></a>348</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
535 | <tr><td class="h"><a name="349"></a>349</td><td></td><td></td><td></td><td></td><td class="s"> };</td></tr> |
536 | <tr><td class="h"><a name="350"></a>350</td><td></td><td></td><td></td><td></td><td class="s"> if ( $@ ) {</td></tr> |
537 | <tr><td class="h"><a name="351"></a>351</td><td></td><td></td><td></td><td></td><td class="s"> my $errstr = $@;</td></tr> |
538 | <tr><td class="h"><a name="352"></a>352</td><td></td><td></td><td></td><td></td><td class="s"> $errstr =~ s/^(.*) at line .+$/$1/;</td></tr> |
539 | <tr><td class="h"><a name="353"></a>353</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw( $errstr );</td></tr> |
540 | <tr><td class="h"><a name="354"></a>354</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
541 | <tr><td class="h"><a name="355"></a>355</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
542 | <tr><td class="h"><a name="356"></a>356</td><td></td><td></td><td></td><td></td><td class="s"> # End of file, return a copy of the token array.</td></tr> |
543 | <tr><td class="h"><a name="357"></a>357</td><td></td><td></td><td></td><td></td><td class="s"> return [ @{$self->{tokens}} ];</td></tr> |
544 | <tr><td class="h"><a name="358"></a>358</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
545 | <tr><td class="h"><a name="359"></a>359</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
546 | <tr><td class="h"><a name="360"></a>360</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr> |
547 | <tr><td class="h"><a name="361"></a>361</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
548 | <tr><td class="h"><a name="362"></a>362</td><td></td><td></td><td></td><td></td><td class="s">=head2 increment_cursor</td></tr> |
549 | <tr><td class="h"><a name="363"></a>363</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
550 | <tr><td class="h"><a name="364"></a>364</td><td></td><td></td><td></td><td></td><td class="s">Although exposed as a public method, C<increment_method> is implemented</td></tr> |
551 | <tr><td class="h"><a name="365"></a>365</td><td></td><td></td><td></td><td></td><td class="s">for expert use only, when writing lexers or other components that work</td></tr> |
552 | <tr><td class="h"><a name="366"></a>366</td><td></td><td></td><td></td><td></td><td class="s">directly on token streams.</td></tr> |
553 | <tr><td class="h"><a name="367"></a>367</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
554 | <tr><td class="h"><a name="368"></a>368</td><td></td><td></td><td></td><td></td><td class="s">It manually increments the token cursor forward through the file, in effect</td></tr> |
555 | <tr><td class="h"><a name="369"></a>369</td><td></td><td></td><td></td><td></td><td class="s">"skipping" the next token.</td></tr> |
556 | <tr><td class="h"><a name="370"></a>370</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
557 | <tr><td class="h"><a name="371"></a>371</td><td></td><td></td><td></td><td></td><td class="s">Return true if the cursor is incremented, C<0> if already at the end of</td></tr> |
558 | <tr><td class="h"><a name="372"></a>372</td><td></td><td></td><td></td><td></td><td class="s">the file, or C<undef> on error.</td></tr> |
559 | <tr><td class="h"><a name="373"></a>373</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
560 | <tr><td class="h"><a name="374"></a>374</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr> |
561 | <tr><td class="h"><a name="375"></a>375</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
562 | <tr><td class="h"><a name="376"></a>376</td><td></td><td></td><td></td><td></td><td class="s">sub increment_cursor {</td></tr> |
563 | <tr><td class="h"><a name="377"></a>377</td><td></td><td></td><td></td><td></td><td class="s"> # Do this via the get_token method, which makes sure there</td></tr> |
564 | <tr><td class="h"><a name="378"></a>378</td><td></td><td></td><td></td><td></td><td class="s"> # is actually a token there to move to.</td></tr> |
565 | <tr><td class="h"><a name="379"></a>379</td><td></td><td></td><td></td><td></td><td class="s"> $_[0]->get_token and 1;</td></tr> |
566 | <tr><td class="h"><a name="380"></a>380</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
567 | <tr><td class="h"><a name="381"></a>381</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
568 | <tr><td class="h"><a name="382"></a>382</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr> |
569 | <tr><td class="h"><a name="383"></a>383</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
570 | <tr><td class="h"><a name="384"></a>384</td><td></td><td></td><td></td><td></td><td class="s">=head2 decrement_cursor</td></tr> |
571 | <tr><td class="h"><a name="385"></a>385</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
572 | <tr><td class="h"><a name="386"></a>386</td><td></td><td></td><td></td><td></td><td class="s">Although exposed as a public method, C<decrement_method> is implemented</td></tr> |
573 | <tr><td class="h"><a name="387"></a>387</td><td></td><td></td><td></td><td></td><td class="s">for expert use only, when writing lexers or other components that work</td></tr> |
574 | <tr><td class="h"><a name="388"></a>388</td><td></td><td></td><td></td><td></td><td class="s">directly on token streams.</td></tr> |
575 | <tr><td class="h"><a name="389"></a>389</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
576 | <tr><td class="h"><a name="390"></a>390</td><td></td><td></td><td></td><td></td><td class="s">It manually decrements the token cursor backwards through the file, in</td></tr> |
577 | <tr><td class="h"><a name="391"></a>391</td><td></td><td></td><td></td><td></td><td class="s">effect "rolling back" the token stream. And indeed that is what it is</td></tr> |
578 | <tr><td class="h"><a name="392"></a>392</td><td></td><td></td><td></td><td></td><td class="s">primarily intended for, when the component that is consuming the token</td></tr> |
579 | <tr><td class="h"><a name="393"></a>393</td><td></td><td></td><td></td><td></td><td class="s">stream needs to implement some sort of "roll back" feature in its use</td></tr> |
580 | <tr><td class="h"><a name="394"></a>394</td><td></td><td></td><td></td><td></td><td class="s">of the token stream.</td></tr> |
581 | <tr><td class="h"><a name="395"></a>395</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
582 | <tr><td class="h"><a name="396"></a>396</td><td></td><td></td><td></td><td></td><td class="s">Return true if the cursor is decremented, C<0> if already at the</td></tr> |
583 | <tr><td class="h"><a name="397"></a>397</td><td></td><td></td><td></td><td></td><td class="s">beginning of the file, or C<undef> on error.</td></tr> |
584 | <tr><td class="h"><a name="398"></a>398</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
585 | <tr><td class="h"><a name="399"></a>399</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr> |
586 | <tr><td class="h"><a name="400"></a>400</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
587 | <tr><td class="h"><a name="401"></a>401</td><td></td><td></td><td></td><td></td><td class="s">sub decrement_cursor {</td></tr> |
588 | <tr><td class="h"><a name="402"></a>402</td><td></td><td></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
589 | <tr><td class="h"><a name="403"></a>403</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
590 | <tr><td class="h"><a name="404"></a>404</td><td></td><td></td><td></td><td></td><td class="s"> # Check for the beginning of the file</td></tr> |
591 | <tr><td class="h"><a name="405"></a>405</td><td></td><td></td><td></td><td></td><td class="s"> return 0 unless $self->{token_cursor};</td></tr> |
592 | <tr><td class="h"><a name="406"></a>406</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
593 | <tr><td class="h"><a name="407"></a>407</td><td></td><td></td><td></td><td></td><td class="s"> # Decrement the token cursor</td></tr> |
594 | <tr><td class="h"><a name="408"></a>408</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token_eof} = 0;</td></tr> |
595 | <tr><td class="h"><a name="409"></a>409</td><td></td><td></td><td></td><td></td><td class="s"> --$self->{token_cursor};</td></tr> |
596 | <tr><td class="h"><a name="410"></a>410</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
597 | <tr><td class="h"><a name="411"></a>411</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
598 | <tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
599 | <tr><td class="h"><a name="416"></a>416</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr> |
600 | <tr><td class="h"><a name="417"></a>417</td><td></td><td></td><td></td><td></td><td class="s"># Working With Source</td></tr> |
601 | <tr><td class="h"><a name="418"></a>418</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
602 | <tr><td class="h"><a name="419"></a>419</td><td></td><td></td><td></td><td></td><td class="s"># Fetches the next line from the input line buffer</td></tr> |
603 | <tr><td class="h"><a name="420"></a>420</td><td></td><td></td><td></td><td></td><td class="s"># Returns undef at EOF.</td></tr> |
604 | <tr><td class="h"><a name="421"></a>421</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 58µs within PPI::Tokenizer::_get_line which was called 24 times, avg 2µs/call: |
605 | # 24 times (58µs+0s) by PPI::Tokenizer::_fill_line at <a href="PPI-Tokenizer-pm-1003-block.html#443">line 443</a>, avg 2µs/call</div></div>sub _get_line {</td></tr> |
606 | <tr><td class="h"><a name="422"></a>422</td><td class="c0">96</td><td class="c1"><span title="Avg 831ns">80µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
607 | <tr><td class="h"><a name="423"></a>423</td><td></td><td></td><td></td><td></td><td class="s"> return undef unless $self->{source}; # EOF hit previously</td></tr> |
608 | <tr><td class="h"><a name="424"></a>424</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
609 | <tr><td class="h"><a name="425"></a>425</td><td></td><td></td><td></td><td></td><td class="s"> # Pull off the next line</td></tr> |
610 | <tr><td class="h"><a name="426"></a>426</td><td></td><td></td><td></td><td></td><td class="s"> my $line = shift @{$self->{source}};</td></tr> |
611 | <tr><td class="h"><a name="427"></a>427</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
612 | <tr><td class="h"><a name="428"></a>428</td><td></td><td></td><td></td><td></td><td class="s"> # Flag EOF if we hit it</td></tr> |
613 | <tr><td class="h"><a name="429"></a>429</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source} = undef unless defined $line;</td></tr> |
614 | <tr><td class="h"><a name="430"></a>430</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
615 | <tr><td class="h"><a name="431"></a>431</td><td></td><td></td><td></td><td></td><td class="s"> # Return the line (or EOF flag)</td></tr> |
616 | <tr><td class="h"><a name="432"></a>432</td><td></td><td></td><td></td><td></td><td class="s"> return $line; # string or undef</td></tr> |
617 | <tr><td class="h"><a name="433"></a>433</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
618 | <tr><td class="h"><a name="434"></a>434</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
619 | <tr><td class="h"><a name="435"></a>435</td><td></td><td></td><td></td><td></td><td class="s"># Fetches the next line, ready to process</td></tr> |
620 | <tr><td class="h"><a name="436"></a>436</td><td></td><td></td><td></td><td></td><td class="s"># Returns 1 on success</td></tr> |
621 | <tr><td class="h"><a name="437"></a>437</td><td></td><td></td><td></td><td></td><td class="s"># Returns 0 on EOF</td></tr> |
622 | <tr><td class="h"><a name="438"></a>438</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 219µs (161+58) within PPI::Tokenizer::_fill_line which was called 24 times, avg 9µs/call: |
623 | # 24 times (161µs+58µs) by PPI::Tokenizer::_process_next_line at <a href="PPI-Tokenizer-pm-1003-block.html#490">line 490</a>, avg 9µs/call</div></div>sub _fill_line {</td></tr> |
624 | <tr><td class="h"><a name="439"></a>439</td><td class="c0">136</td><td class="c1"><span title="Avg 596ns">81µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
625 | <tr><td class="h"><a name="440"></a>440</td><td></td><td></td><td></td><td></td><td class="s"> my $inscan = shift;</td></tr> |
626 | <tr><td class="h"><a name="441"></a>441</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
627 | <tr><td class="h"><a name="442"></a>442</td><td></td><td></td><td></td><td></td><td class="s"> # Get the next line</td></tr> |
628 | <tr><td class="h"><a name="443"></a>443</td><td></td><td></td><td class="c0">24</td><td class="c3">58µs</td><td class="s"> my $line = $self->_get_line;<div class="calls"><div class="calls_out"> # spent 58µs making 24 calls to <a href="PPI-Tokenizer-pm-1003-block.html#421">PPI::Tokenizer::_get_line</a>, avg 2µs/call</div></div></td></tr> |
629 | <tr><td class="h"><a name="444"></a>444</td><td class="c3">16</td><td class="c3"><span title="Avg 200ns">3µs</span></td><td></td><td></td><td class="s"> unless ( defined $line ) {</td></tr> |
630 | <tr><td class="h"><a name="445"></a>445</td><td></td><td></td><td></td><td></td><td class="s"> # End of file</td></tr> |
631 | <tr><td class="h"><a name="446"></a>446</td><td class="c0">64</td><td class="c3"><span title="Avg 678ns">43µs</span></td><td></td><td></td><td class="s"> unless ( $inscan ) {</td></tr> |
632 | <tr><td class="h"><a name="447"></a>447</td><td></td><td></td><td></td><td></td><td class="s"> delete $self->{line};</td></tr> |
633 | <tr><td class="h"><a name="448"></a>448</td><td></td><td></td><td></td><td></td><td class="s"> delete $self->{line_cursor};</td></tr> |
634 | <tr><td class="h"><a name="449"></a>449</td><td></td><td></td><td></td><td></td><td class="s"> delete $self->{line_length};</td></tr> |
635 | <tr><td class="h"><a name="450"></a>450</td><td></td><td></td><td></td><td></td><td class="s"> return 0;</td></tr> |
636 | <tr><td class="h"><a name="451"></a>451</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
637 | <tr><td class="h"><a name="452"></a>452</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
638 | <tr><td class="h"><a name="453"></a>453</td><td></td><td></td><td></td><td></td><td class="s"> # In the scan version, just set the cursor to the end</td></tr> |
639 | <tr><td class="h"><a name="454"></a>454</td><td></td><td></td><td></td><td></td><td class="s"> # of the line, and the rest should just cascade out.</td></tr> |
640 | <tr><td class="h"><a name="455"></a>455</td><td></td><td></td><td></td><td></td><td class="s"> $self->{line_cursor} = $self->{line_length};</td></tr> |
641 | <tr><td class="h"><a name="456"></a>456</td><td></td><td></td><td></td><td></td><td class="s"> return 0;</td></tr> |
642 | <tr><td class="h"><a name="457"></a>457</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
643 | <tr><td class="h"><a name="458"></a>458</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
644 | <tr><td class="h"><a name="459"></a>459</td><td></td><td></td><td></td><td></td><td class="s"> # Populate the appropriate variables</td></tr> |
645 | <tr><td class="h"><a name="460"></a>460</td><td></td><td></td><td></td><td></td><td class="s"> $self->{line} = $line;</td></tr> |
646 | <tr><td class="h"><a name="461"></a>461</td><td></td><td></td><td></td><td></td><td class="s"> $self->{line_cursor} = -1;</td></tr> |
647 | <tr><td class="h"><a name="462"></a>462</td><td></td><td></td><td></td><td></td><td class="s"> $self->{line_length} = length $line;</td></tr> |
648 | <tr><td class="h"><a name="463"></a>463</td><td></td><td></td><td></td><td></td><td class="s"> $self->{line_count}++;</td></tr> |
649 | <tr><td class="h"><a name="464"></a>464</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
650 | <tr><td class="h"><a name="465"></a>465</td><td></td><td></td><td></td><td></td><td class="s"> 1;</td></tr> |
651 | <tr><td class="h"><a name="466"></a>466</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
652 | <tr><td class="h"><a name="467"></a>467</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
653 | <tr><td class="h"><a name="468"></a>468</td><td></td><td></td><td></td><td></td><td class="s"># Get the current character</td></tr> |
654 | <tr><td class="h"><a name="469"></a>469</td><td></td><td></td><td></td><td></td><td class="s">sub _char {</td></tr> |
655 | <tr><td class="h"><a name="470"></a>470</td><td></td><td></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
656 | <tr><td class="h"><a name="471"></a>471</td><td></td><td></td><td></td><td></td><td class="s"> substr( $self->{line}, $self->{line_cursor}, 1 );</td></tr> |
657 | <tr><td class="h"><a name="472"></a>472</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
658 | <tr><td class="h"><a name="473"></a>473</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
659 | <tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
660 | <tr><td class="h"><a name="478"></a>478</td><td></td><td></td><td></td><td></td><td class="s">####################################################################</td></tr> |
661 | <tr><td class="h"><a name="479"></a>479</td><td></td><td></td><td></td><td></td><td class="s"># Per line processing methods</td></tr> |
662 | <tr><td class="h"><a name="480"></a>480</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
663 | <tr><td class="h"><a name="481"></a>481</td><td></td><td></td><td></td><td></td><td class="s"># Processes the next line</td></tr> |
664 | <tr><td class="h"><a name="482"></a>482</td><td></td><td></td><td></td><td></td><td class="s"># Returns 1 on success completion</td></tr> |
665 | <tr><td class="h"><a name="483"></a>483</td><td></td><td></td><td></td><td></td><td class="s"># Returns 0 if EOF</td></tr> |
666 | <tr><td class="h"><a name="484"></a>484</td><td></td><td></td><td></td><td></td><td class="s"># Returns undef on error</td></tr> |
667 | <tr><td class="h"><a name="485"></a>485</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 3.63ms (369µs+3.26) within PPI::Tokenizer::_process_next_line which was called 24 times, avg 151µs/call: |
668 | # 24 times (369µs+3.26ms) by PPI::Tokenizer::get_token at <a href="PPI-Tokenizer-pm-1003-block.html#277">line 277</a>, avg 151µs/call</div></div>sub _process_next_line {</td></tr> |
669 | <tr><td class="h"><a name="486"></a>486</td><td class="c0">120</td><td class="c0"><span title="Avg 2µs">220µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
670 | <tr><td class="h"><a name="487"></a>487</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
671 | <tr><td class="h"><a name="488"></a>488</td><td></td><td></td><td></td><td></td><td class="s"> # Fill the line buffer</td></tr> |
672 | <tr><td class="h"><a name="489"></a>489</td><td></td><td></td><td></td><td></td><td class="s"> my $rv;</td></tr> |
673 | <tr><td class="h"><a name="490"></a>490</td><td class="c0">48</td><td class="c3"><span title="Avg 908ns">44µs</span></td><td class="c0">24</td><td class="c0">219µs</td><td class="s"> unless ( $rv = $self->_fill_line ) {<div class="calls"><div class="calls_out"> # spent 219µs making 24 calls to <a href="PPI-Tokenizer-pm-1003-block.html#438">PPI::Tokenizer::_fill_line</a>, avg 9µs/call</div></div></td></tr> |
674 | <tr><td class="h"><a name="491"></a>491</td><td></td><td></td><td></td><td></td><td class="s"> return undef unless defined $rv;</td></tr> |
675 | <tr><td class="h"><a name="492"></a>492</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
676 | <tr><td class="h"><a name="493"></a>493</td><td></td><td></td><td></td><td></td><td class="s"> # End of file, finalize last token</td></tr> |
677 | <tr><td class="h"><a name="494"></a>494</td><td></td><td></td><td class="c1">16</td><td class="c3">23µs</td><td class="s"> $self->_finalize_token;<div class="calls"><div class="calls_out"> # spent 23µs making 16 calls to <a href="PPI-Tokenizer-pm-1003-block.html#599">PPI::Tokenizer::_finalize_token</a>, avg 1µs/call</div></div></td></tr> |
678 | <tr><td class="h"><a name="495"></a>495</td><td></td><td></td><td></td><td></td><td class="s"> return 0;</td></tr> |
679 | <tr><td class="h"><a name="496"></a>496</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
680 | <tr><td class="h"><a name="497"></a>497</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
681 | <tr><td class="h"><a name="498"></a>498</td><td></td><td></td><td></td><td></td><td class="s"> # Run the __TOKENIZER__on_line_start</td></tr> |
682 | <tr><td class="h"><a name="499"></a>499</td><td></td><td></td><td class="c3">8</td><td class="c0">228µs</td><td class="s"> $rv = $self->{class}->__TOKENIZER__on_line_start( $self );<div class="calls"><div class="calls_out"> # spent 228µs making 8 calls to <a href="PPI-Token-BOM-pm-923-block.html#68">PPI::Token::BOM::__TOKENIZER__on_line_start</a>, avg 28µs/call</div></div></td></tr> |
683 | <tr><td class="h"><a name="500"></a>500</td><td></td><td></td><td></td><td></td><td class="s"> unless ( $rv ) {</td></tr> |
684 | <tr><td class="h"><a name="501"></a>501</td><td></td><td></td><td></td><td></td><td class="s"> # If there are no more source lines, then clean up</td></tr> |
685 | <tr><td class="h"><a name="502"></a>502</td><td></td><td></td><td></td><td></td><td class="s"> if ( ref $self->{source} eq 'ARRAY' and ! @{$self->{source}} ) {</td></tr> |
686 | <tr><td class="h"><a name="503"></a>503</td><td></td><td></td><td></td><td></td><td class="s"> $self->_clean_eof;</td></tr> |
687 | <tr><td class="h"><a name="504"></a>504</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
688 | <tr><td class="h"><a name="505"></a>505</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
689 | <tr><td class="h"><a name="506"></a>506</td><td></td><td></td><td></td><td></td><td class="s"> # Defined but false means next line</td></tr> |
690 | <tr><td class="h"><a name="507"></a>507</td><td></td><td></td><td></td><td></td><td class="s"> return 1 if defined $rv;</td></tr> |
691 | <tr><td class="h"><a name="508"></a>508</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw("Error at line $self->{line_count}");</td></tr> |
692 | <tr><td class="h"><a name="509"></a>509</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
693 | <tr><td class="h"><a name="510"></a>510</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
694 | <tr><td class="h"><a name="511"></a>511</td><td></td><td></td><td></td><td></td><td class="s"> # If we can't deal with the entire line, process char by char</td></tr> |
695 | <tr><td class="h"><a name="512"></a>512</td><td></td><td></td><td class="c0">48</td><td class="c0">2.66ms</td><td class="s"> while ( $rv = $self->_process_next_char ) {}<div class="calls"><div class="calls_out"> # spent 2.66ms making 48 calls to <a href="PPI-Tokenizer-pm-1003-block.html#539">PPI::Tokenizer::_process_next_char</a>, avg 55µs/call</div></div></td></tr> |
696 | <tr><td class="h"><a name="513"></a>513</td><td></td><td></td><td></td><td></td><td class="s"> unless ( defined $rv ) {</td></tr> |
697 | <tr><td class="h"><a name="514"></a>514</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Exception->throw("Error at line $self->{line_count}, character $self->{line_cursor}");</td></tr> |
698 | <tr><td class="h"><a name="515"></a>515</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
699 | <tr><td class="h"><a name="516"></a>516</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
700 | <tr><td class="h"><a name="517"></a>517</td><td></td><td></td><td></td><td></td><td class="s"> # Trigger any action that needs to happen at the end of a line</td></tr> |
701 | <tr><td class="h"><a name="518"></a>518</td><td></td><td></td><td class="c3">8</td><td class="c2">95µs</td><td class="s"> $self->{class}->__TOKENIZER__on_line_end( $self );<div class="calls"><div class="calls_out"> # spent 95µs making 8 calls to <a href="PPI-Token-Whitespace-pm-924-block.html#416">PPI::Token::Whitespace::__TOKENIZER__on_line_end</a>, avg 12µs/call</div></div></td></tr> |
702 | <tr><td class="h"><a name="519"></a>519</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
703 | <tr><td class="h"><a name="520"></a>520</td><td></td><td></td><td></td><td></td><td class="s"> # If there are no more source lines, then clean up</td></tr> |
704 | <tr><td class="h"><a name="521"></a>521</td><td></td><td></td><td class="c3">8</td><td class="c3">38µs</td><td class="s"> unless ( ref($self->{source}) eq 'ARRAY' and @{$self->{source}} ) {<div class="calls"><div class="calls_out"> # spent 38µs making 8 calls to <a href="PPI-Tokenizer-pm-1003-block.html#631">PPI::Tokenizer::_clean_eof</a>, avg 5µs/call</div></div></td></tr> |
705 | <tr><td class="h"><a name="522"></a>522</td><td></td><td></td><td></td><td></td><td class="s"> return $self->_clean_eof;</td></tr> |
706 | <tr><td class="h"><a name="523"></a>523</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
707 | <tr><td class="h"><a name="524"></a>524</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
708 | <tr><td class="h"><a name="525"></a>525</td><td></td><td></td><td></td><td></td><td class="s"> return 1;</td></tr> |
709 | <tr><td class="h"><a name="526"></a>526</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
710 | <tr><td class="h"><a name="527"></a>527</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
711 | <tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
712 | <tr><td class="h"><a name="532"></a>532</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr> |
713 | <tr><td class="h"><a name="533"></a>533</td><td></td><td></td><td></td><td></td><td class="s"># Per-character processing methods</td></tr> |
714 | <tr><td class="h"><a name="534"></a>534</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
715 | <tr><td class="h"><a name="535"></a>535</td><td></td><td></td><td></td><td></td><td class="s"># Process on a per-character basis.</td></tr> |
716 | <tr><td class="h"><a name="536"></a>536</td><td></td><td></td><td></td><td></td><td class="s"># Note that due the the high number of times this gets</td></tr> |
717 | <tr><td class="h"><a name="537"></a>537</td><td></td><td></td><td></td><td></td><td class="s"># called, it has been fairly heavily in-lined, so the code</td></tr> |
718 | <tr><td class="h"><a name="538"></a>538</td><td></td><td></td><td></td><td></td><td class="s"># might look a bit ugly and duplicated.</td></tr> |
719 | <tr><td class="h"><a name="539"></a>539</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 2.66ms (452µs+2.21) within PPI::Tokenizer::_process_next_char which was called 48 times, avg 55µs/call: |
720 | # 48 times (452µs+2.21ms) by PPI::Tokenizer::_process_next_line at <a href="PPI-Tokenizer-pm-1003-block.html#512">line 512</a>, avg 55µs/call</div></div>sub _process_next_char {</td></tr> |
721 | <tr><td class="h"><a name="540"></a>540</td><td class="c0">368</td><td class="c0"><span title="Avg 804ns">296µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
722 | <tr><td class="h"><a name="541"></a>541</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
723 | <tr><td class="h"><a name="542"></a>542</td><td></td><td></td><td></td><td></td><td class="s"> ### FIXME - This checks for a screwed up condition that triggers</td></tr> |
724 | <tr><td class="h"><a name="543"></a>543</td><td></td><td></td><td></td><td></td><td class="s"> ### several warnings, amoungst other things.</td></tr> |
725 | <tr><td class="h"><a name="544"></a>544</td><td></td><td></td><td></td><td></td><td class="s"> if ( ! defined $self->{line_cursor} or ! defined $self->{line_length} ) {</td></tr> |
726 | <tr><td class="h"><a name="545"></a>545</td><td></td><td></td><td></td><td></td><td class="s"> # $DB::single = 1;</td></tr> |
727 | <tr><td class="h"><a name="546"></a>546</td><td></td><td></td><td></td><td></td><td class="s"> return undef;</td></tr> |
728 | <tr><td class="h"><a name="547"></a>547</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
729 | <tr><td class="h"><a name="548"></a>548</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
730 | <tr><td class="h"><a name="549"></a>549</td><td></td><td></td><td></td><td></td><td class="s"> # Increment the counter and check for end of line</td></tr> |
731 | <tr><td class="h"><a name="550"></a>550</td><td></td><td></td><td></td><td></td><td class="s"> return 0 if ++$self->{line_cursor} >= $self->{line_length};</td></tr> |
732 | <tr><td class="h"><a name="551"></a>551</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
733 | <tr><td class="h"><a name="552"></a>552</td><td></td><td></td><td></td><td></td><td class="s"> # Pass control to the token class</td></tr> |
734 | <tr><td class="h"><a name="553"></a>553</td><td></td><td></td><td></td><td></td><td class="s"> my $result;</td></tr> |
735 | <tr><td class="h"><a name="554"></a>554</td><td></td><td></td><td class="c0">40</td><td class="c0">1.71ms</td><td class="s"> unless ( $result = $self->{class}->__TOKENIZER__on_char( $self ) ) {<div class="calls"><div class="calls_out"> # spent 1.05ms making 24 calls to <a href="PPI-Token-Whitespace-pm-924-block.html#201">PPI::Token::Whitespace::__TOKENIZER__on_char</a>, avg 44µs/call |
736 | # spent 385µs making 8 calls to <a href="PPI-Token-Unknown-pm-939-block.html#48">PPI::Token::Unknown::__TOKENIZER__on_char</a>, avg 48µs/call |
737 | # spent 267µs making 8 calls to <a href="PPI-Token-Symbol-pm-936-block.html#160">PPI::Token::Symbol::__TOKENIZER__on_char</a>, avg 33µs/call</div></div></td></tr> |
738 | <tr><td class="h"><a name="555"></a>555</td><td></td><td></td><td></td><td></td><td class="s"> # undef is error. 0 is "Did stuff ourself, you don't have to do anything"</td></tr> |
739 | <tr><td class="h"><a name="556"></a>556</td><td></td><td></td><td></td><td></td><td class="s"> return defined $result ? 1 : undef;</td></tr> |
740 | <tr><td class="h"><a name="557"></a>557</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
741 | <tr><td class="h"><a name="558"></a>558</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
742 | <tr><td class="h"><a name="559"></a>559</td><td></td><td></td><td></td><td></td><td class="s"> # We will need the value of the current character</td></tr> |
743 | <tr><td class="h"><a name="560"></a>560</td><td></td><td></td><td></td><td></td><td class="s"> my $char = substr( $self->{line}, $self->{line_cursor}, 1 );</td></tr> |
744 | <tr><td class="h"><a name="561"></a>561</td><td class="c3">16</td><td class="c3"><span title="Avg 1µs">23µs</span></td><td></td><td></td><td class="s"> if ( $result eq '1' ) {</td></tr> |
745 | <tr><td class="h"><a name="562"></a>562</td><td></td><td></td><td></td><td></td><td class="s"> # If __TOKENIZER__on_char returns 1, it is signaling that it thinks that</td></tr> |
746 | <tr><td class="h"><a name="563"></a>563</td><td></td><td></td><td></td><td></td><td class="s"> # the character is part of it.</td></tr> |
747 | <tr><td class="h"><a name="564"></a>564</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
748 | <tr><td class="h"><a name="565"></a>565</td><td></td><td></td><td></td><td></td><td class="s"> # Add the character</td></tr> |
749 | <tr><td class="h"><a name="566"></a>566</td><td></td><td></td><td></td><td></td><td class="s"> if ( defined $self->{token} ) {</td></tr> |
750 | <tr><td class="h"><a name="567"></a>567</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token}->{content} .= $char;</td></tr> |
751 | <tr><td class="h"><a name="568"></a>568</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr> |
752 | <tr><td class="h"><a name="569"></a>569</td><td></td><td></td><td></td><td></td><td class="s"> defined($self->{token} = $self->{class}->new($char)) or return undef;</td></tr> |
753 | <tr><td class="h"><a name="570"></a>570</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
754 | <tr><td class="h"><a name="571"></a>571</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
755 | <tr><td class="h"><a name="572"></a>572</td><td></td><td></td><td></td><td></td><td class="s"> return 1;</td></tr> |
756 | <tr><td class="h"><a name="573"></a>573</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
757 | <tr><td class="h"><a name="574"></a>574</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
758 | <tr><td class="h"><a name="575"></a>575</td><td></td><td></td><td></td><td></td><td class="s"> # We have been provided with the name of a class</td></tr> |
759 | <tr><td class="h"><a name="576"></a>576</td><td class="c2">24</td><td class="c2"><span title="Avg 2µs">59µs</span></td><td class="c3">8</td><td class="c0">259µs</td><td class="s"> if ( $self->{class} ne "PPI::Token::$result" ) {<div class="calls"><div class="calls_out"> # spent 259µs making 8 calls to <a href="PPI-Tokenizer-pm-1003-block.html#613">PPI::Tokenizer::_new_token</a>, avg 32µs/call</div></div></td></tr> |
760 | <tr><td class="h"><a name="577"></a>577</td><td></td><td></td><td></td><td></td><td class="s"> # New class</td></tr> |
761 | <tr><td class="h"><a name="578"></a>578</td><td></td><td></td><td></td><td></td><td class="s"> $self->_new_token( $result, $char );</td></tr> |
762 | <tr><td class="h"><a name="579"></a>579</td><td></td><td></td><td></td><td></td><td class="s"> } elsif ( defined $self->{token} ) {</td></tr> |
763 | <tr><td class="h"><a name="580"></a>580</td><td></td><td></td><td></td><td></td><td class="s"> # Same class as current</td></tr> |
764 | <tr><td class="h"><a name="581"></a>581</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token}->{content} .= $char;</td></tr> |
765 | <tr><td class="h"><a name="582"></a>582</td><td></td><td></td><td></td><td></td><td class="s"> } else {</td></tr> |
766 | <tr><td class="h"><a name="583"></a>583</td><td></td><td></td><td></td><td></td><td class="s"> # Same class, but no current</td></tr> |
767 | <tr><td class="h"><a name="584"></a>584</td><td></td><td></td><td class="c0">24</td><td class="c0">245µs</td><td class="s"> defined($self->{token} = $self->{class}->new($char)) or return undef;<div class="calls"><div class="calls_out"> # spent 245µs making 24 calls to <a href="PPI-Token-pm-922-block.html#85">PPI::Token::new</a>, avg 10µs/call</div></div></td></tr> |
768 | <tr><td class="h"><a name="585"></a>585</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
769 | <tr><td class="h"><a name="586"></a>586</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
770 | <tr><td class="h"><a name="587"></a>587</td><td></td><td></td><td></td><td></td><td class="s"> 1;</td></tr> |
771 | <tr><td class="h"><a name="588"></a>588</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
772 | <tr><td class="h"><a name="589"></a>589</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
773 | <tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
774 | <tr><td class="h"><a name="594"></a>594</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr> |
775 | <tr><td class="h"><a name="595"></a>595</td><td></td><td></td><td></td><td></td><td class="s"># Altering Tokens in Tokenizer</td></tr> |
776 | <tr><td class="h"><a name="596"></a>596</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
777 | <tr><td class="h"><a name="597"></a>597</td><td></td><td></td><td></td><td></td><td class="s"># Finish the end of a token.</td></tr> |
778 | <tr><td class="h"><a name="598"></a>598</td><td></td><td></td><td></td><td></td><td class="s"># Returns the resulting parse class as a convenience.</td></tr> |
779 | <tr><td class="h"><a name="599"></a>599</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 150µs within PPI::Tokenizer::_finalize_token which was called 56 times, avg 3µs/call: |
780 | # 16 times (50µs+0s) by PPI::Tokenizer::_new_token at <a href="PPI-Tokenizer-pm-1003-block.html#620">line 620</a>, avg 3µs/call |
781 | # 16 times (23µs+0s) by PPI::Tokenizer::_process_next_line at <a href="PPI-Tokenizer-pm-1003-block.html#494">line 494</a>, avg 1µs/call |
782 | # 8 times (28µs+0s) by PPI::Token::Symbol::__TOKENIZER__on_char at <a href="PPI-Token-Symbol-pm-936-block.html#216">line 216 of PPI/Token/Symbol.pm</a>, avg 4µs/call |
783 | # 8 times (26µs+0s) by PPI::Token::Whitespace::__TOKENIZER__on_line_end at <a href="PPI-Token-Whitespace-pm-924-block.html#417">line 417 of PPI/Token/Whitespace.pm</a>, avg 3µs/call |
784 | # 8 times (22µs+0s) by PPI::Token::Word::__TOKENIZER__commit at <a href="PPI-Token-Word-pm-934-block.html#539">line 539 of PPI/Token/Word.pm</a>, avg 3µs/call</div></div>sub _finalize_token {</td></tr> |
785 | <tr><td class="h"><a name="600"></a>600</td><td class="c0">232</td><td class="c0"><span title="Avg 813ns">189µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
786 | <tr><td class="h"><a name="601"></a>601</td><td></td><td></td><td></td><td></td><td class="s"> return $self->{class} unless defined $self->{token};</td></tr> |
787 | <tr><td class="h"><a name="602"></a>602</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
788 | <tr><td class="h"><a name="603"></a>603</td><td></td><td></td><td></td><td></td><td class="s"> # Add the token to the token buffer</td></tr> |
789 | <tr><td class="h"><a name="604"></a>604</td><td></td><td></td><td></td><td></td><td class="s"> push @{ $self->{tokens} }, $self->{token};</td></tr> |
790 | <tr><td class="h"><a name="605"></a>605</td><td></td><td></td><td></td><td></td><td class="s"> $self->{token} = undef;</td></tr> |
791 | <tr><td class="h"><a name="606"></a>606</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
792 | <tr><td class="h"><a name="607"></a>607</td><td></td><td></td><td></td><td></td><td class="s"> # Return the parse class to that of the zone we are in</td></tr> |
793 | <tr><td class="h"><a name="608"></a>608</td><td></td><td></td><td></td><td></td><td class="s"> $self->{class} = $self->{zone};</td></tr> |
794 | <tr><td class="h"><a name="609"></a>609</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
795 | <tr><td class="h"><a name="610"></a>610</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
796 | <tr><td class="h"><a name="611"></a>611</td><td></td><td></td><td></td><td></td><td class="s"># Creates a new token and sets it in the tokenizer</td></tr> |
797 | <tr><td class="h"><a name="612"></a>612</td><td></td><td></td><td></td><td></td><td class="s"># The defined() in here prevent a ton of calls to PPI::Util::TRUE</td></tr> |
798 | <tr><td class="h"><a name="613"></a>613</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 547µs (159+388) within PPI::Tokenizer::_new_token which was called 16 times, avg 34µs/call: |
799 | # 8 times (88µs+199µs) by PPI::Token::Word::__TOKENIZER__commit at <a href="PPI-Token-Word-pm-934-block.html#533">line 533 of PPI/Token/Word.pm</a>, avg 36µs/call |
800 | # 8 times (70µs+189µs) by PPI::Tokenizer::_process_next_char at <a href="PPI-Tokenizer-pm-1003-block.html#576">line 576</a>, avg 32µs/call</div></div>sub _new_token {</td></tr> |
801 | <tr><td class="h"><a name="614"></a>614</td><td class="c0">96</td><td class="c0"><span title="Avg 1µs">128µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
802 | <tr><td class="h"><a name="615"></a>615</td><td></td><td></td><td></td><td></td><td class="s"> # throw PPI::Exception() unless @_;</td></tr> |
803 | <tr><td class="h"><a name="616"></a>616</td><td></td><td></td><td></td><td></td><td class="s"> my $class = substr( $_[0], 0, 12 ) eq 'PPI::Token::'</td></tr> |
804 | <tr><td class="h"><a name="617"></a>617</td><td></td><td></td><td></td><td></td><td class="s"> ? shift : 'PPI::Token::' . shift;</td></tr> |
805 | <tr><td class="h"><a name="618"></a>618</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
806 | <tr><td class="h"><a name="619"></a>619</td><td></td><td></td><td></td><td></td><td class="s"> # Finalize any existing token</td></tr> |
807 | <tr><td class="h"><a name="620"></a>620</td><td></td><td></td><td class="c1">16</td><td class="c3">50µs</td><td class="s"> $self->_finalize_token if defined $self->{token};<div class="calls"><div class="calls_out"> # spent 50µs making 16 calls to <a href="PPI-Tokenizer-pm-1003-block.html#599">PPI::Tokenizer::_finalize_token</a>, avg 3µs/call</div></div></td></tr> |
808 | <tr><td class="h"><a name="621"></a>621</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
809 | <tr><td class="h"><a name="622"></a>622</td><td></td><td></td><td></td><td></td><td class="s"> # Create the new token and update the parse class</td></tr> |
810 | <tr><td class="h"><a name="623"></a>623</td><td></td><td></td><td class="c1">16</td><td class="c0">338µs</td><td class="s"> defined($self->{token} = $class->new($_[0])) or PPI::Exception->throw;<div class="calls"><div class="calls_out"> # spent 338µs making 16 calls to <a href="PPI-Token-pm-922-block.html#85">PPI::Token::new</a>, avg 21µs/call</div></div></td></tr> |
811 | <tr><td class="h"><a name="624"></a>624</td><td></td><td></td><td></td><td></td><td class="s"> $self->{class} = $class;</td></tr> |
812 | <tr><td class="h"><a name="625"></a>625</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
813 | <tr><td class="h"><a name="626"></a>626</td><td></td><td></td><td></td><td></td><td class="s"> 1;</td></tr> |
814 | <tr><td class="h"><a name="627"></a>627</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
815 | <tr><td class="h"><a name="628"></a>628</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
816 | <tr><td class="h"><a name="629"></a>629</td><td></td><td></td><td></td><td></td><td class="s"># At the end of the file, we need to clean up the results of the erroneous</td></tr> |
817 | <tr><td class="h"><a name="630"></a>630</td><td></td><td></td><td></td><td></td><td class="s"># space that we inserted at the beginning of the process.</td></tr> |
818 | <tr><td class="h"><a name="631"></a>631</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 38µs within PPI::Tokenizer::_clean_eof which was called 8 times, avg 5µs/call: |
819 | # 8 times (38µs+0s) by PPI::Tokenizer::_process_next_line at <a href="PPI-Tokenizer-pm-1003-block.html#521">line 521</a>, avg 5µs/call</div></div>sub _clean_eof {</td></tr> |
820 | <tr><td class="h"><a name="632"></a>632</td><td class="c0">48</td><td class="c3"><span title="Avg 883ns">42µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
821 | <tr><td class="h"><a name="633"></a>633</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
822 | <tr><td class="h"><a name="634"></a>634</td><td></td><td></td><td></td><td></td><td class="s"> # Finish any partially completed token</td></tr> |
823 | <tr><td class="h"><a name="635"></a>635</td><td></td><td></td><td></td><td></td><td class="s"> $self->_finalize_token if $self->{token};</td></tr> |
824 | <tr><td class="h"><a name="636"></a>636</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
825 | <tr><td class="h"><a name="637"></a>637</td><td></td><td></td><td></td><td></td><td class="s"> # Find the last token, and if it has no content, kill it.</td></tr> |
826 | <tr><td class="h"><a name="638"></a>638</td><td></td><td></td><td></td><td></td><td class="s"> # There appears to be some evidence that such "null tokens" are</td></tr> |
827 | <tr><td class="h"><a name="639"></a>639</td><td></td><td></td><td></td><td></td><td class="s"> # somehow getting created accidentally.</td></tr> |
828 | <tr><td class="h"><a name="640"></a>640</td><td></td><td></td><td></td><td></td><td class="s"> my $last_token = $self->{tokens}->[ -1 ];</td></tr> |
829 | <tr><td class="h"><a name="641"></a>641</td><td></td><td></td><td></td><td></td><td class="s"> unless ( length $last_token->{content} ) {</td></tr> |
830 | <tr><td class="h"><a name="642"></a>642</td><td></td><td></td><td></td><td></td><td class="s"> pop @{$self->{tokens}};</td></tr> |
831 | <tr><td class="h"><a name="643"></a>643</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
832 | <tr><td class="h"><a name="644"></a>644</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
833 | <tr><td class="h"><a name="645"></a>645</td><td></td><td></td><td></td><td></td><td class="s"> # Now, if the last character of the last token is a space we added,</td></tr> |
834 | <tr><td class="h"><a name="646"></a>646</td><td></td><td></td><td></td><td></td><td class="s"> # chop it off, deleting the token if there's nothing else left.</td></tr> |
835 | <tr><td class="h"><a name="647"></a>647</td><td></td><td></td><td></td><td></td><td class="s"> if ( $self->{source_eof_chop} ) {</td></tr> |
836 | <tr><td class="h"><a name="648"></a>648</td><td></td><td></td><td></td><td></td><td class="s"> $last_token = $self->{tokens}->[ -1 ];</td></tr> |
837 | <tr><td class="h"><a name="649"></a>649</td><td></td><td></td><td></td><td></td><td class="s"> $last_token->{content} =~ s/ $//;</td></tr> |
838 | <tr><td class="h"><a name="650"></a>650</td><td></td><td></td><td></td><td></td><td class="s"> unless ( length $last_token->{content} ) {</td></tr> |
839 | <tr><td class="h"><a name="651"></a>651</td><td></td><td></td><td></td><td></td><td class="s"> # Popping token</td></tr> |
840 | <tr><td class="h"><a name="652"></a>652</td><td></td><td></td><td></td><td></td><td class="s"> pop @{$self->{tokens}};</td></tr> |
841 | <tr><td class="h"><a name="653"></a>653</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
842 | <tr><td class="h"><a name="654"></a>654</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
843 | <tr><td class="h"><a name="655"></a>655</td><td></td><td></td><td></td><td></td><td class="s"> # The hack involving adding an extra space is now reversed, and</td></tr> |
844 | <tr><td class="h"><a name="656"></a>656</td><td></td><td></td><td></td><td></td><td class="s"> # now nobody will ever know. The perfect crime!</td></tr> |
845 | <tr><td class="h"><a name="657"></a>657</td><td></td><td></td><td></td><td></td><td class="s"> $self->{source_eof_chop} = '';</td></tr> |
846 | <tr><td class="h"><a name="658"></a>658</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
847 | <tr><td class="h"><a name="659"></a>659</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
848 | <tr><td class="h"><a name="660"></a>660</td><td></td><td></td><td></td><td></td><td class="s"> 1;</td></tr> |
849 | <tr><td class="h"><a name="661"></a>661</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
850 | <tr><td class="h"><a name="662"></a>662</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
851 | <tr><td class="h"><a name="- -"></a>- -</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
852 | <tr><td class="h"><a name="667"></a>667</td><td></td><td></td><td></td><td></td><td class="s">#####################################################################</td></tr> |
853 | <tr><td class="h"><a name="668"></a>668</td><td></td><td></td><td></td><td></td><td class="s"># Utility Methods</td></tr> |
854 | <tr><td class="h"><a name="669"></a>669</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
855 | <tr><td class="h"><a name="670"></a>670</td><td></td><td></td><td></td><td></td><td class="s"># Context</td></tr> |
856 | <tr><td class="h"><a name="671"></a>671</td><td></td><td></td><td></td><td></td><td class="s">sub _last_token {</td></tr> |
857 | <tr><td class="h"><a name="672"></a>672</td><td></td><td></td><td></td><td></td><td class="s"> $_[0]->{tokens}->[-1];</td></tr> |
858 | <tr><td class="h"><a name="673"></a>673</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
859 | <tr><td class="h"><a name="674"></a>674</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
860 | <tr><td class="h"><a name="675"></a>675</td><td></td><td></td><td></td><td></td><td class="s">sub _last_significant_token {</td></tr> |
861 | <tr><td class="h"><a name="676"></a>676</td><td></td><td></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
862 | <tr><td class="h"><a name="677"></a>677</td><td></td><td></td><td></td><td></td><td class="s"> my $cursor = $#{ $self->{tokens} };</td></tr> |
863 | <tr><td class="h"><a name="678"></a>678</td><td></td><td></td><td></td><td></td><td class="s"> while ( $cursor >= 0 ) {</td></tr> |
864 | <tr><td class="h"><a name="679"></a>679</td><td></td><td></td><td></td><td></td><td class="s"> my $token = $self->{tokens}->[$cursor--];</td></tr> |
865 | <tr><td class="h"><a name="680"></a>680</td><td></td><td></td><td></td><td></td><td class="s"> return $token if $token->significant;</td></tr> |
866 | <tr><td class="h"><a name="681"></a>681</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
867 | <tr><td class="h"><a name="682"></a>682</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
868 | <tr><td class="h"><a name="683"></a>683</td><td></td><td></td><td></td><td></td><td class="s"> # Nothing...</td></tr> |
869 | <tr><td class="h"><a name="684"></a>684</td><td></td><td></td><td></td><td></td><td class="s"> PPI::Token::Whitespace->null;</td></tr> |
870 | <tr><td class="h"><a name="685"></a>685</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
871 | <tr><td class="h"><a name="686"></a>686</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
872 | <tr><td class="h"><a name="687"></a>687</td><td></td><td></td><td></td><td></td><td class="s"># Get an array ref of previous significant tokens.</td></tr> |
873 | <tr><td class="h"><a name="688"></a>688</td><td></td><td></td><td></td><td></td><td class="s"># Like _last_significant_token except it gets more than just one token</td></tr> |
874 | <tr><td class="h"><a name="689"></a>689</td><td></td><td></td><td></td><td></td><td class="s"># Returns array ref on success.</td></tr> |
875 | <tr><td class="h"><a name="690"></a>690</td><td></td><td></td><td></td><td></td><td class="s"># Returns 0 on not enough tokens</td></tr> |
876 | <tr><td class="h"><a name="691"></a>691</td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 243µs (102+141) within PPI::Tokenizer::_previous_significant_tokens which was called 8 times, avg 30µs/call: |
877 | # 8 times (102µs+141µs) by PPI::Token::Word::__TOKENIZER__commit at <a href="PPI-Token-Word-pm-934-block.html#430">line 430 of PPI/Token/Word.pm</a>, avg 30µs/call</div></div>sub _previous_significant_tokens {</td></tr> |
878 | <tr><td class="h"><a name="692"></a>692</td><td class="c0">56</td><td class="c2"><span title="Avg 1µs">61µs</span></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
879 | <tr><td class="h"><a name="693"></a>693</td><td></td><td></td><td></td><td></td><td class="s"> my $count = shift || 1;</td></tr> |
880 | <tr><td class="h"><a name="694"></a>694</td><td></td><td></td><td></td><td></td><td class="s"> my $cursor = $#{ $self->{tokens} };</td></tr> |
881 | <tr><td class="h"><a name="695"></a>695</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
882 | <tr><td class="h"><a name="696"></a>696</td><td></td><td></td><td></td><td></td><td class="s"> my ($token, @tokens);</td></tr> |
883 | <tr><td class="h"><a name="697"></a>697</td><td></td><td></td><td></td><td></td><td class="s"> while ( $cursor >= 0 ) {</td></tr> |
884 | <tr><td class="h"><a name="698"></a>698</td><td></td><td></td><td></td><td></td><td class="s"> $token = $self->{tokens}->[$cursor--];</td></tr> |
885 | <tr><td class="h"><a name="699"></a>699</td><td></td><td></td><td></td><td></td><td class="s"> if ( $token->significant ) {</td></tr> |
886 | <tr><td class="h"><a name="700"></a>700</td><td></td><td></td><td></td><td></td><td class="s"> push @tokens, $token;</td></tr> |
887 | <tr><td class="h"><a name="701"></a>701</td><td></td><td></td><td></td><td></td><td class="s"> return \@tokens if scalar @tokens >= $count;</td></tr> |
888 | <tr><td class="h"><a name="702"></a>702</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
889 | <tr><td class="h"><a name="703"></a>703</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
890 | <tr><td class="h"><a name="704"></a>704</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
891 | <tr><td class="h"><a name="705"></a>705</td><td></td><td></td><td></td><td></td><td class="s"> # Pad with empties</td></tr> |
892 | <tr><td class="h"><a name="706"></a>706</td><td></td><td></td><td></td><td></td><td class="s"> foreach ( 1 .. ($count - scalar @tokens) ) {</td></tr> |
893 | <tr><td class="h"><a name="707"></a>707</td><td class="c3">8</td><td class="c3"><span title="Avg 4µs">31µs</span></td><td class="c3">8</td><td class="c1">141µs</td><td class="s"> push @tokens, PPI::Token::Whitespace->null;<div class="calls"><div class="calls_out"> # spent 141µs making 8 calls to <a href="PPI-Token-Whitespace-pm-924-block.html#78">PPI::Token::Whitespace::null</a>, avg 18µs/call</div></div></td></tr> |
894 | <tr><td class="h"><a name="708"></a>708</td><td></td><td></td><td></td><td></td><td class="s"> }</td></tr> |
895 | <tr><td class="h"><a name="709"></a>709</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
896 | <tr><td class="h"><a name="710"></a>710</td><td></td><td></td><td></td><td></td><td class="s"> \@tokens;</td></tr> |
897 | <tr><td class="h"><a name="711"></a>711</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
898 | <tr><td class="h"><a name="712"></a>712</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
899 | <tr><td class="h"><a name="713"></a>713</td><td class="c3">1</td><td class="c3"><span title="Avg 11µs">11µs</span></td><td></td><td></td><td class="s">my %OBVIOUS_CLASS = (</td></tr> |
900 | <tr><td class="h"><a name="714"></a>714</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Symbol' => 'operator',</td></tr> |
901 | <tr><td class="h"><a name="715"></a>715</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Magic' => 'operator',</td></tr> |
902 | <tr><td class="h"><a name="716"></a>716</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Number' => 'operator',</td></tr> |
903 | <tr><td class="h"><a name="717"></a>717</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::ArrayIndex' => 'operator',</td></tr> |
904 | <tr><td class="h"><a name="718"></a>718</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Quote::Double' => 'operator',</td></tr> |
905 | <tr><td class="h"><a name="719"></a>719</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Quote::Interpolate' => 'operator',</td></tr> |
906 | <tr><td class="h"><a name="720"></a>720</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Quote::Literal' => 'operator',</td></tr> |
907 | <tr><td class="h"><a name="721"></a>721</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::Quote::Single' => 'operator',</td></tr> |
908 | <tr><td class="h"><a name="722"></a>722</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::QuoteLike::Backtick' => 'operator',</td></tr> |
909 | <tr><td class="h"><a name="723"></a>723</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::QuoteLike::Command' => 'operator',</td></tr> |
910 | <tr><td class="h"><a name="724"></a>724</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::QuoteLike::Readline' => 'operator',</td></tr> |
911 | <tr><td class="h"><a name="725"></a>725</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::QuoteLike::Regexp' => 'operator',</td></tr> |
912 | <tr><td class="h"><a name="726"></a>726</td><td></td><td></td><td></td><td></td><td class="s"> 'PPI::Token::QuoteLike::Words' => 'operator',</td></tr> |
913 | <tr><td class="h"><a name="727"></a>727</td><td></td><td></td><td></td><td></td><td class="s">);</td></tr> |
914 | <tr><td class="h"><a name="728"></a>728</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
915 | <tr><td class="h"><a name="729"></a>729</td><td class="c3">1</td><td class="c3"><span title="Avg 3µs">3µs</span></td><td></td><td></td><td class="s">my %OBVIOUS_CONTENT = (</td></tr> |
916 | <tr><td class="h"><a name="730"></a>730</td><td></td><td></td><td></td><td></td><td class="s"> '(' => 'operand',</td></tr> |
917 | <tr><td class="h"><a name="731"></a>731</td><td></td><td></td><td></td><td></td><td class="s"> '{' => 'operand',</td></tr> |
918 | <tr><td class="h"><a name="732"></a>732</td><td></td><td></td><td></td><td></td><td class="s"> '[' => 'operand',</td></tr> |
919 | <tr><td class="h"><a name="733"></a>733</td><td></td><td></td><td></td><td></td><td class="s"> ';' => 'operand',</td></tr> |
920 | <tr><td class="h"><a name="734"></a>734</td><td></td><td></td><td></td><td></td><td class="s"> '}' => 'operator',</td></tr> |
921 | <tr><td class="h"><a name="735"></a>735</td><td></td><td></td><td></td><td></td><td class="s">);</td></tr> |
922 | <tr><td class="h"><a name="736"></a>736</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
923 | <tr><td class="h"><a name="737"></a>737</td><td></td><td></td><td></td><td></td><td class="s"># Try to determine operator/operand context, is possible.</td></tr> |
924 | <tr><td class="h"><a name="738"></a>738</td><td></td><td></td><td></td><td></td><td class="s"># Returns "operator", "operand", or "" if unknown.</td></tr> |
925 | <tr><td class="h"><a name="739"></a>739</td><td></td><td></td><td></td><td></td><td class="s">sub _opcontext {</td></tr> |
926 | <tr><td class="h"><a name="740"></a>740</td><td></td><td></td><td></td><td></td><td class="s"> my $self = shift;</td></tr> |
927 | <tr><td class="h"><a name="741"></a>741</td><td></td><td></td><td></td><td></td><td class="s"> my $tokens = $self->_previous_significant_tokens(1);</td></tr> |
928 | <tr><td class="h"><a name="742"></a>742</td><td></td><td></td><td></td><td></td><td class="s"> my $p0 = $tokens->[0];</td></tr> |
929 | <tr><td class="h"><a name="743"></a>743</td><td></td><td></td><td></td><td></td><td class="s"> my $c0 = ref $p0;</td></tr> |
930 | <tr><td class="h"><a name="744"></a>744</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
931 | <tr><td class="h"><a name="745"></a>745</td><td></td><td></td><td></td><td></td><td class="s"> # Map the obvious cases</td></tr> |
932 | <tr><td class="h"><a name="746"></a>746</td><td></td><td></td><td></td><td></td><td class="s"> return $OBVIOUS_CLASS{$c0} if defined $OBVIOUS_CLASS{$c0};</td></tr> |
933 | <tr><td class="h"><a name="747"></a>747</td><td></td><td></td><td></td><td></td><td class="s"> return $OBVIOUS_CONTENT{$p0} if defined $OBVIOUS_CONTENT{$p0};</td></tr> |
934 | <tr><td class="h"><a name="748"></a>748</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
935 | <tr><td class="h"><a name="749"></a>749</td><td></td><td></td><td></td><td></td><td class="s"> # Most of the time after an operator, we are an operand</td></tr> |
936 | <tr><td class="h"><a name="750"></a>750</td><td></td><td></td><td></td><td></td><td class="s"> return 'operand' if $p0->isa('PPI::Token::Operator');</td></tr> |
937 | <tr><td class="h"><a name="751"></a>751</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
938 | <tr><td class="h"><a name="752"></a>752</td><td></td><td></td><td></td><td></td><td class="s"> # If there's NOTHING, it's operand</td></tr> |
939 | <tr><td class="h"><a name="753"></a>753</td><td></td><td></td><td></td><td></td><td class="s"> return 'operand' if $p0->content eq '';</td></tr> |
940 | <tr><td class="h"><a name="754"></a>754</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
941 | <tr><td class="h"><a name="755"></a>755</td><td></td><td></td><td></td><td></td><td class="s"> # Otherwise, we don't know</td></tr> |
942 | <tr><td class="h"><a name="756"></a>756</td><td></td><td></td><td></td><td></td><td class="s"> return ''</td></tr> |
943 | <tr><td class="h"><a name="757"></a>757</td><td></td><td></td><td></td><td></td><td class="s">}</td></tr> |
944 | <tr><td class="h"><a name="758"></a>758</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
945 | <tr><td class="h"><a name="759"></a>759</td><td class="c3">1</td><td class="c3"><span title="Avg 13µs">13µs</span></td><td></td><td></td><td class="s">1;</td></tr> |
946 | <tr><td class="h"><a name="760"></a>760</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
947 | <tr><td class="h"><a name="761"></a>761</td><td></td><td></td><td></td><td></td><td class="s">=pod</td></tr> |
948 | <tr><td class="h"><a name="762"></a>762</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
949 | <tr><td class="h"><a name="763"></a>763</td><td></td><td></td><td></td><td></td><td class="s">=head1 NOTES</td></tr> |
950 | <tr><td class="h"><a name="764"></a>764</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
951 | <tr><td class="h"><a name="765"></a>765</td><td></td><td></td><td></td><td></td><td class="s">=head2 How the Tokenizer Works</td></tr> |
952 | <tr><td class="h"><a name="766"></a>766</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
953 | <tr><td class="h"><a name="767"></a>767</td><td></td><td></td><td></td><td></td><td class="s">Understanding the Tokenizer is not for the feint-hearted. It is by far</td></tr> |
954 | <tr><td class="h"><a name="768"></a>768</td><td></td><td></td><td></td><td></td><td class="s">the most complex and twisty piece of perl I've ever written that is actually</td></tr> |
955 | <tr><td class="h"><a name="769"></a>769</td><td></td><td></td><td></td><td></td><td class="s">still built properly and isn't a terrible spaghetti-like mess. In fact, you</td></tr> |
956 | <tr><td class="h"><a name="770"></a>770</td><td></td><td></td><td></td><td></td><td class="s">probably want to skip this section.</td></tr> |
957 | <tr><td class="h"><a name="771"></a>771</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
958 | <tr><td class="h"><a name="772"></a>772</td><td></td><td></td><td></td><td></td><td class="s">But if you really want to understand, well then here goes.</td></tr> |
959 | <tr><td class="h"><a name="773"></a>773</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
960 | <tr><td class="h"><a name="774"></a>774</td><td></td><td></td><td></td><td></td><td class="s">=head2 Source Input and Clean Up</td></tr> |
961 | <tr><td class="h"><a name="775"></a>775</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
962 | <tr><td class="h"><a name="776"></a>776</td><td></td><td></td><td></td><td></td><td class="s">The Tokenizer starts by taking source in a variety of forms, sucking it</td></tr> |
963 | <tr><td class="h"><a name="777"></a>777</td><td></td><td></td><td></td><td></td><td class="s">all in and merging into one big string, and doing our own internal line</td></tr> |
964 | <tr><td class="h"><a name="778"></a>778</td><td></td><td></td><td></td><td></td><td class="s">split, using a "universal line separator" which allows the Tokenizer to</td></tr> |
965 | <tr><td class="h"><a name="779"></a>779</td><td></td><td></td><td></td><td></td><td class="s">take source for any platform (and even supports a few known types of</td></tr> |
966 | <tr><td class="h"><a name="780"></a>780</td><td></td><td></td><td></td><td></td><td class="s">broken newlines caused by mixed mac/pc/*nix editor screw ups).</td></tr> |
967 | <tr><td class="h"><a name="781"></a>781</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
968 | <tr><td class="h"><a name="782"></a>782</td><td></td><td></td><td></td><td></td><td class="s">The resulting array of lines is used to feed the tokenizer, and is also</td></tr> |
969 | <tr><td class="h"><a name="783"></a>783</td><td></td><td></td><td></td><td></td><td class="s">accessed directly by the heredoc-logic to do the line-oriented part of</td></tr> |
970 | <tr><td class="h"><a name="784"></a>784</td><td></td><td></td><td></td><td></td><td class="s">here-doc support.</td></tr> |
971 | <tr><td class="h"><a name="785"></a>785</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
972 | <tr><td class="h"><a name="786"></a>786</td><td></td><td></td><td></td><td></td><td class="s">=head2 Doing Things the Old Fashioned Way</td></tr> |
973 | <tr><td class="h"><a name="787"></a>787</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
974 | <tr><td class="h"><a name="788"></a>788</td><td></td><td></td><td></td><td></td><td class="s">Due to the complexity of perl, and after 2 previously aborted parser</td></tr> |
975 | <tr><td class="h"><a name="789"></a>789</td><td></td><td></td><td></td><td></td><td class="s">attempts, in the end the tokenizer was fashioned around a line-buffered</td></tr> |
976 | <tr><td class="h"><a name="790"></a>790</td><td></td><td></td><td></td><td></td><td class="s">character-by-character method.</td></tr> |
977 | <tr><td class="h"><a name="791"></a>791</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
978 | <tr><td class="h"><a name="792"></a>792</td><td></td><td></td><td></td><td></td><td class="s">That is, the Tokenizer pulls and holds a line at a time into a line buffer,</td></tr> |
979 | <tr><td class="h"><a name="793"></a>793</td><td></td><td></td><td></td><td></td><td class="s">and then iterates a cursor along it. At each cursor position, a method is</td></tr> |
980 | <tr><td class="h"><a name="794"></a>794</td><td></td><td></td><td></td><td></td><td class="s">called in whatever token class we are currently in, which will examine the</td></tr> |
981 | <tr><td class="h"><a name="795"></a>795</td><td></td><td></td><td></td><td></td><td class="s">character at the current position, and handle it.</td></tr> |
982 | <tr><td class="h"><a name="796"></a>796</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
983 | <tr><td class="h"><a name="797"></a>797</td><td></td><td></td><td></td><td></td><td class="s">As the handler methods in the various token classes are called, they</td></tr> |
984 | <tr><td class="h"><a name="798"></a>798</td><td></td><td></td><td></td><td></td><td class="s">build up a output token array for the source code.</td></tr> |
985 | <tr><td class="h"><a name="799"></a>799</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
986 | <tr><td class="h"><a name="800"></a>800</td><td></td><td></td><td></td><td></td><td class="s">Various parts of the Tokenizer use look-ahead, arbitrary-distance</td></tr> |
987 | <tr><td class="h"><a name="801"></a>801</td><td></td><td></td><td></td><td></td><td class="s">look-behind (although currently the maximum is three significant tokens),</td></tr> |
988 | <tr><td class="h"><a name="802"></a>802</td><td></td><td></td><td></td><td></td><td class="s">or both, and various other heuristic guesses.</td></tr> |
989 | <tr><td class="h"><a name="803"></a>803</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
990 | <tr><td class="h"><a name="804"></a>804</td><td></td><td></td><td></td><td></td><td class="s">I've been told it is officially termed a I<"backtracking parser</td></tr> |
991 | <tr><td class="h"><a name="805"></a>805</td><td></td><td></td><td></td><td></td><td class="s">with infinite lookaheads">.</td></tr> |
992 | <tr><td class="h"><a name="806"></a>806</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
993 | <tr><td class="h"><a name="807"></a>807</td><td></td><td></td><td></td><td></td><td class="s">=head2 State Variables</td></tr> |
994 | <tr><td class="h"><a name="808"></a>808</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
995 | <tr><td class="h"><a name="809"></a>809</td><td></td><td></td><td></td><td></td><td class="s">Aside from the current line and the character cursor, the Tokenizer</td></tr> |
996 | <tr><td class="h"><a name="810"></a>810</td><td></td><td></td><td></td><td></td><td class="s">maintains a number of different state variables.</td></tr> |
997 | <tr><td class="h"><a name="811"></a>811</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
998 | <tr><td class="h"><a name="812"></a>812</td><td></td><td></td><td></td><td></td><td class="s">=over</td></tr> |
999 | <tr><td class="h"><a name="813"></a>813</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1000 | <tr><td class="h"><a name="814"></a>814</td><td></td><td></td><td></td><td></td><td class="s">=item Current Class</td></tr> |
1001 | <tr><td class="h"><a name="815"></a>815</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1002 | <tr><td class="h"><a name="816"></a>816</td><td></td><td></td><td></td><td></td><td class="s">The Tokenizer maintains the current token class at all times. Much of the</td></tr> |
1003 | <tr><td class="h"><a name="817"></a>817</td><td></td><td></td><td></td><td></td><td class="s">time is just going to be the "Whitespace" class, which is what the base of</td></tr> |
1004 | <tr><td class="h"><a name="818"></a>818</td><td></td><td></td><td></td><td></td><td class="s">a document is. As the tokenizer executes the various character handlers,</td></tr> |
1005 | <tr><td class="h"><a name="819"></a>819</td><td></td><td></td><td></td><td></td><td class="s">the class changes a lot as it moves a long. In fact, in some instances,</td></tr> |
1006 | <tr><td class="h"><a name="820"></a>820</td><td></td><td></td><td></td><td></td><td class="s">the character handler may not handle the character directly itself, but</td></tr> |
1007 | <tr><td class="h"><a name="821"></a>821</td><td></td><td></td><td></td><td></td><td class="s">rather change the "current class" and then hand off to the character</td></tr> |
1008 | <tr><td class="h"><a name="822"></a>822</td><td></td><td></td><td></td><td></td><td class="s">handler for the new class.</td></tr> |
1009 | <tr><td class="h"><a name="823"></a>823</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1010 | <tr><td class="h"><a name="824"></a>824</td><td></td><td></td><td></td><td></td><td class="s">Because of this, and some other things I'll deal with later, the number of</td></tr> |
1011 | <tr><td class="h"><a name="825"></a>825</td><td></td><td></td><td></td><td></td><td class="s">times the character handlers are called does not in fact have a direct</td></tr> |
1012 | <tr><td class="h"><a name="826"></a>826</td><td></td><td></td><td></td><td></td><td class="s">relationship to the number of actual characters in the document.</td></tr> |
1013 | <tr><td class="h"><a name="827"></a>827</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1014 | <tr><td class="h"><a name="828"></a>828</td><td></td><td></td><td></td><td></td><td class="s">=item Current Zone</td></tr> |
1015 | <tr><td class="h"><a name="829"></a>829</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1016 | <tr><td class="h"><a name="830"></a>830</td><td></td><td></td><td></td><td></td><td class="s">Rather than create a class stack to allow for infinitely nested layers of</td></tr> |
1017 | <tr><td class="h"><a name="831"></a>831</td><td></td><td></td><td></td><td></td><td class="s">classes, the Tokenizer recognises just a single layer.</td></tr> |
1018 | <tr><td class="h"><a name="832"></a>832</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1019 | <tr><td class="h"><a name="833"></a>833</td><td></td><td></td><td></td><td></td><td class="s">To put it a different way, in various parts of the file, the Tokenizer will</td></tr> |
1020 | <tr><td class="h"><a name="834"></a>834</td><td></td><td></td><td></td><td></td><td class="s">recognise different "base" or "substrate" classes. When a Token such as a</td></tr> |
1021 | <tr><td class="h"><a name="835"></a>835</td><td></td><td></td><td></td><td></td><td class="s">comment or a number is finalised by the tokenizer, it "falls back" to the</td></tr> |
1022 | <tr><td class="h"><a name="836"></a>836</td><td></td><td></td><td></td><td></td><td class="s">base state.</td></tr> |
1023 | <tr><td class="h"><a name="837"></a>837</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1024 | <tr><td class="h"><a name="838"></a>838</td><td></td><td></td><td></td><td></td><td class="s">This allows proper tokenization of special areas such as __DATA__</td></tr> |
1025 | <tr><td class="h"><a name="839"></a>839</td><td></td><td></td><td></td><td></td><td class="s">and __END__ blocks, which also contain things like comments and POD,</td></tr> |
1026 | <tr><td class="h"><a name="840"></a>840</td><td></td><td></td><td></td><td></td><td class="s">without allowing the creation of any significant Tokens inside these areas.</td></tr> |
1027 | <tr><td class="h"><a name="841"></a>841</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1028 | <tr><td class="h"><a name="842"></a>842</td><td></td><td></td><td></td><td></td><td class="s">For the main part of a document we use L<PPI::Token::Whitespace> for this,</td></tr> |
1029 | <tr><td class="h"><a name="843"></a>843</td><td></td><td></td><td></td><td></td><td class="s">with the idea being that code is "floating in a sea of whitespace".</td></tr> |
1030 | <tr><td class="h"><a name="844"></a>844</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1031 | <tr><td class="h"><a name="845"></a>845</td><td></td><td></td><td></td><td></td><td class="s">=item Current Token</td></tr> |
1032 | <tr><td class="h"><a name="846"></a>846</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1033 | <tr><td class="h"><a name="847"></a>847</td><td></td><td></td><td></td><td></td><td class="s">The final main state variable is the "current token". This is the Token</td></tr> |
1034 | <tr><td class="h"><a name="848"></a>848</td><td></td><td></td><td></td><td></td><td class="s">that is currently being built by the Tokenizer. For certain types, it</td></tr> |
1035 | <tr><td class="h"><a name="849"></a>849</td><td></td><td></td><td></td><td></td><td class="s">can be manipulated and morphed and change class quite a bit while being</td></tr> |
1036 | <tr><td class="h"><a name="850"></a>850</td><td></td><td></td><td></td><td></td><td class="s">assembled, as the Tokenizer's understanding of the token content changes.</td></tr> |
1037 | <tr><td class="h"><a name="851"></a>851</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1038 | <tr><td class="h"><a name="852"></a>852</td><td></td><td></td><td></td><td></td><td class="s">When the Tokenizer is confident that it has seen the end of the Token, it</td></tr> |
1039 | <tr><td class="h"><a name="853"></a>853</td><td></td><td></td><td></td><td></td><td class="s">will be "finalized", which adds it to the output token array and resets</td></tr> |
1040 | <tr><td class="h"><a name="854"></a>854</td><td></td><td></td><td></td><td></td><td class="s">the current class to that of the zone that we are currently in.</td></tr> |
1041 | <tr><td class="h"><a name="855"></a>855</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1042 | <tr><td class="h"><a name="856"></a>856</td><td></td><td></td><td></td><td></td><td class="s">I should also note at this point that the "current token" variable is</td></tr> |
1043 | <tr><td class="h"><a name="857"></a>857</td><td></td><td></td><td></td><td></td><td class="s">optional. The Tokenizer is capable of knowing what class it is currently</td></tr> |
1044 | <tr><td class="h"><a name="858"></a>858</td><td></td><td></td><td></td><td></td><td class="s">set to, without actually having accumulated any characters in the Token.</td></tr> |
1045 | <tr><td class="h"><a name="859"></a>859</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1046 | <tr><td class="h"><a name="860"></a>860</td><td></td><td></td><td></td><td></td><td class="s">=back</td></tr> |
1047 | <tr><td class="h"><a name="861"></a>861</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1048 | <tr><td class="h"><a name="862"></a>862</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster</td></tr> |
1049 | <tr><td class="h"><a name="863"></a>863</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1050 | <tr><td class="h"><a name="864"></a>864</td><td></td><td></td><td></td><td></td><td class="s">As I'm sure you can imagine, calling several different methods for each</td></tr> |
1051 | <tr><td class="h"><a name="865"></a>865</td><td></td><td></td><td></td><td></td><td class="s">character and running regexes and other complex heuristics made the first</td></tr> |
1052 | <tr><td class="h"><a name="866"></a>866</td><td></td><td></td><td></td><td></td><td class="s">fully working version of the tokenizer extremely slow.</td></tr> |
1053 | <tr><td class="h"><a name="867"></a>867</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1054 | <tr><td class="h"><a name="868"></a>868</td><td></td><td></td><td></td><td></td><td class="s">During testing, I created a metric to measure parsing speed called</td></tr> |
1055 | <tr><td class="h"><a name="869"></a>869</td><td></td><td></td><td></td><td></td><td class="s">LPGC, or "lines per gigacycle" . A gigacycle is simple a billion CPU</td></tr> |
1056 | <tr><td class="h"><a name="870"></a>870</td><td></td><td></td><td></td><td></td><td class="s">cycles on a typical single-core CPU, and so a Tokenizer running at</td></tr> |
1057 | <tr><td class="h"><a name="871"></a>871</td><td></td><td></td><td></td><td></td><td class="s">"1000 lines per gigacycle" should generate around 1200 lines of tokenized</td></tr> |
1058 | <tr><td class="h"><a name="872"></a>872</td><td></td><td></td><td></td><td></td><td class="s">code when running on a 1200 MHz processor.</td></tr> |
1059 | <tr><td class="h"><a name="873"></a>873</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1060 | <tr><td class="h"><a name="874"></a>874</td><td></td><td></td><td></td><td></td><td class="s">The first working version of the tokenizer ran at only 350 LPGC, so to</td></tr> |
1061 | <tr><td class="h"><a name="875"></a>875</td><td></td><td></td><td></td><td></td><td class="s">tokenize a typical large module such as L<ExtUtils::MakeMaker> took</td></tr> |
1062 | <tr><td class="h"><a name="876"></a>876</td><td></td><td></td><td></td><td></td><td class="s">10-15 seconds. This sluggishness made it unpractical for many uses.</td></tr> |
1063 | <tr><td class="h"><a name="877"></a>877</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1064 | <tr><td class="h"><a name="878"></a>878</td><td></td><td></td><td></td><td></td><td class="s">So in the current parser, there are multiple layers of optimisation</td></tr> |
1065 | <tr><td class="h"><a name="879"></a>879</td><td></td><td></td><td></td><td></td><td class="s">very carefully built in to the basic. This has brought the tokenizer</td></tr> |
1066 | <tr><td class="h"><a name="880"></a>880</td><td></td><td></td><td></td><td></td><td class="s">up to a more reasonable 1000 LPGC, at the expense of making the code</td></tr> |
1067 | <tr><td class="h"><a name="881"></a>881</td><td></td><td></td><td></td><td></td><td class="s">quite a bit twistier.</td></tr> |
1068 | <tr><td class="h"><a name="882"></a>882</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1069 | <tr><td class="h"><a name="883"></a>883</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster - Whole Line Classification</td></tr> |
1070 | <tr><td class="h"><a name="884"></a>884</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1071 | <tr><td class="h"><a name="885"></a>885</td><td></td><td></td><td></td><td></td><td class="s">The first step in the optimisation process was to add a hew handler to</td></tr> |
1072 | <tr><td class="h"><a name="886"></a>886</td><td></td><td></td><td></td><td></td><td class="s">enable several of the more basic classes (whitespace, comments) to be</td></tr> |
1073 | <tr><td class="h"><a name="887"></a>887</td><td></td><td></td><td></td><td></td><td class="s">able to be parsed a line at a time. At the start of each line, a</td></tr> |
1074 | <tr><td class="h"><a name="888"></a>888</td><td></td><td></td><td></td><td></td><td class="s">special optional handler (only supported by a few classes) is called to</td></tr> |
1075 | <tr><td class="h"><a name="889"></a>889</td><td></td><td></td><td></td><td></td><td class="s">check and see if the entire line can be parsed in one go.</td></tr> |
1076 | <tr><td class="h"><a name="890"></a>890</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1077 | <tr><td class="h"><a name="891"></a>891</td><td></td><td></td><td></td><td></td><td class="s">This is used mainly to handle things like POD, comments, empty lines,</td></tr> |
1078 | <tr><td class="h"><a name="892"></a>892</td><td></td><td></td><td></td><td></td><td class="s">and a few other minor special cases.</td></tr> |
1079 | <tr><td class="h"><a name="893"></a>893</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1080 | <tr><td class="h"><a name="894"></a>894</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster - Inlining</td></tr> |
1081 | <tr><td class="h"><a name="895"></a>895</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1082 | <tr><td class="h"><a name="896"></a>896</td><td></td><td></td><td></td><td></td><td class="s">The second stage of the optimisation involved inlining a small</td></tr> |
1083 | <tr><td class="h"><a name="897"></a>897</td><td></td><td></td><td></td><td></td><td class="s">number of critical methods that were repeated an extremely high number</td></tr> |
1084 | <tr><td class="h"><a name="898"></a>898</td><td></td><td></td><td></td><td></td><td class="s">of times. Profiling suggested that there were about 1,000,000 individual</td></tr> |
1085 | <tr><td class="h"><a name="899"></a>899</td><td></td><td></td><td></td><td></td><td class="s">method calls per gigacycle, and by cutting these by two thirds a significant</td></tr> |
1086 | <tr><td class="h"><a name="900"></a>900</td><td></td><td></td><td></td><td></td><td class="s">speed improvement was gained, in the order of about 50%.</td></tr> |
1087 | <tr><td class="h"><a name="901"></a>901</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1088 | <tr><td class="h"><a name="902"></a>902</td><td></td><td></td><td></td><td></td><td class="s">You may notice that many methods in the C<PPI::Tokenizer> code look</td></tr> |
1089 | <tr><td class="h"><a name="903"></a>903</td><td></td><td></td><td></td><td></td><td class="s">very nested and long hand. This is primarily due to this inlining.</td></tr> |
1090 | <tr><td class="h"><a name="904"></a>904</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1091 | <tr><td class="h"><a name="905"></a>905</td><td></td><td></td><td></td><td></td><td class="s">At around this time, some statistics code that existed in the early</td></tr> |
1092 | <tr><td class="h"><a name="906"></a>906</td><td></td><td></td><td></td><td></td><td class="s">versions of the parser was also removed, as it was determined that</td></tr> |
1093 | <tr><td class="h"><a name="907"></a>907</td><td></td><td></td><td></td><td></td><td class="s">it was consuming around 15% of the CPU for the entire parser, while</td></tr> |
1094 | <tr><td class="h"><a name="908"></a>908</td><td></td><td></td><td></td><td></td><td class="s">making the core more complicated.</td></tr> |
1095 | <tr><td class="h"><a name="909"></a>909</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1096 | <tr><td class="h"><a name="910"></a>910</td><td></td><td></td><td></td><td></td><td class="s">A judgment call was made that with the difficulties likely to be</td></tr> |
1097 | <tr><td class="h"><a name="911"></a>911</td><td></td><td></td><td></td><td></td><td class="s">encountered with future planned enhancements, and given the relatively</td></tr> |
1098 | <tr><td class="h"><a name="912"></a>912</td><td></td><td></td><td></td><td></td><td class="s">high cost involved, the statistics features would be removed from the</td></tr> |
1099 | <tr><td class="h"><a name="913"></a>913</td><td></td><td></td><td></td><td></td><td class="s">Tokenizer.</td></tr> |
1100 | <tr><td class="h"><a name="914"></a>914</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1101 | <tr><td class="h"><a name="915"></a>915</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster - Quote Engine</td></tr> |
1102 | <tr><td class="h"><a name="916"></a>916</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1103 | <tr><td class="h"><a name="917"></a>917</td><td></td><td></td><td></td><td></td><td class="s">Once inlining had reached diminishing returns, it became obvious from</td></tr> |
1104 | <tr><td class="h"><a name="918"></a>918</td><td></td><td></td><td></td><td></td><td class="s">the profiling results that a huge amount of time was being spent</td></tr> |
1105 | <tr><td class="h"><a name="919"></a>919</td><td></td><td></td><td></td><td></td><td class="s">stepping a char at a time though long, simple and "syntactically boring"</td></tr> |
1106 | <tr><td class="h"><a name="920"></a>920</td><td></td><td></td><td></td><td></td><td class="s">code such as comments and strings.</td></tr> |
1107 | <tr><td class="h"><a name="921"></a>921</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1108 | <tr><td class="h"><a name="922"></a>922</td><td></td><td></td><td></td><td></td><td class="s">The existing regex engine was expanded to also encompass quotes and</td></tr> |
1109 | <tr><td class="h"><a name="923"></a>923</td><td></td><td></td><td></td><td></td><td class="s">other quote-like things, and a special abstract base class was added</td></tr> |
1110 | <tr><td class="h"><a name="924"></a>924</td><td></td><td></td><td></td><td></td><td class="s">that provided a number of specialised parsing methods that would "scan</td></tr> |
1111 | <tr><td class="h"><a name="925"></a>925</td><td></td><td></td><td></td><td></td><td class="s">ahead", looking out ahead to find the end of a string, and updating</td></tr> |
1112 | <tr><td class="h"><a name="926"></a>926</td><td></td><td></td><td></td><td></td><td class="s">the cursor to leave it in a valid position for the next call.</td></tr> |
1113 | <tr><td class="h"><a name="927"></a>927</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1114 | <tr><td class="h"><a name="928"></a>928</td><td></td><td></td><td></td><td></td><td class="s">This is also the point at which the number of character handler calls began</td></tr> |
1115 | <tr><td class="h"><a name="929"></a>929</td><td></td><td></td><td></td><td></td><td class="s">to greatly differ from the number of characters. But it has been done</td></tr> |
1116 | <tr><td class="h"><a name="930"></a>930</td><td></td><td></td><td></td><td></td><td class="s">in a way that allows the parser to retain the power of the original</td></tr> |
1117 | <tr><td class="h"><a name="931"></a>931</td><td></td><td></td><td></td><td></td><td class="s">version at the critical points, while skipping through the "boring bits"</td></tr> |
1118 | <tr><td class="h"><a name="932"></a>932</td><td></td><td></td><td></td><td></td><td class="s">as needed for additional speed.</td></tr> |
1119 | <tr><td class="h"><a name="933"></a>933</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1120 | <tr><td class="h"><a name="934"></a>934</td><td></td><td></td><td></td><td></td><td class="s">The addition of this feature allowed the tokenizer to exceed 1000 LPGC</td></tr> |
1121 | <tr><td class="h"><a name="935"></a>935</td><td></td><td></td><td></td><td></td><td class="s">for the first time.</td></tr> |
1122 | <tr><td class="h"><a name="936"></a>936</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1123 | <tr><td class="h"><a name="937"></a>937</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster - The "Complete" Mechanism</td></tr> |
1124 | <tr><td class="h"><a name="938"></a>938</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1125 | <tr><td class="h"><a name="939"></a>939</td><td></td><td></td><td></td><td></td><td class="s">As it became evident that great speed increases were available by using</td></tr> |
1126 | <tr><td class="h"><a name="940"></a>940</td><td></td><td></td><td></td><td></td><td class="s">this "skipping ahead" mechanism, a new handler method was added that</td></tr> |
1127 | <tr><td class="h"><a name="941"></a>941</td><td></td><td></td><td></td><td></td><td class="s">explicitly handles the parsing of an entire token, where the structure</td></tr> |
1128 | <tr><td class="h"><a name="942"></a>942</td><td></td><td></td><td></td><td></td><td class="s">of the token is relatively simple. Tokens such as symbols fit this case,</td></tr> |
1129 | <tr><td class="h"><a name="943"></a>943</td><td></td><td></td><td></td><td></td><td class="s">as once we are passed the initial sigil and word char, we know that we</td></tr> |
1130 | <tr><td class="h"><a name="944"></a>944</td><td></td><td></td><td></td><td></td><td class="s">can skip ahead and "complete" the rest of the token much more easily.</td></tr> |
1131 | <tr><td class="h"><a name="945"></a>945</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1132 | <tr><td class="h"><a name="946"></a>946</td><td></td><td></td><td></td><td></td><td class="s">A number of these have been added for most or possibly all of the common</td></tr> |
1133 | <tr><td class="h"><a name="947"></a>947</td><td></td><td></td><td></td><td></td><td class="s">cases, with most of these "complete" handlers implemented using regular</td></tr> |
1134 | <tr><td class="h"><a name="948"></a>948</td><td></td><td></td><td></td><td></td><td class="s">expressions.</td></tr> |
1135 | <tr><td class="h"><a name="949"></a>949</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1136 | <tr><td class="h"><a name="950"></a>950</td><td></td><td></td><td></td><td></td><td class="s">In fact, so many have been added that at this point, you could arguably</td></tr> |
1137 | <tr><td class="h"><a name="951"></a>951</td><td></td><td></td><td></td><td></td><td class="s">reclassify the tokenizer as a "hybrid regex, char-by=char heuristic</td></tr> |
1138 | <tr><td class="h"><a name="952"></a>952</td><td></td><td></td><td></td><td></td><td class="s">tokenizer". More tokens are now consumed in "complete" methods in a</td></tr> |
1139 | <tr><td class="h"><a name="953"></a>953</td><td></td><td></td><td></td><td></td><td class="s">typical program than are handled by the normal char-by-char methods.</td></tr> |
1140 | <tr><td class="h"><a name="954"></a>954</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1141 | <tr><td class="h"><a name="955"></a>955</td><td></td><td></td><td></td><td></td><td class="s">Many of the these complete-handlers were implemented during the writing</td></tr> |
1142 | <tr><td class="h"><a name="956"></a>956</td><td></td><td></td><td></td><td></td><td class="s">of the Lexer, and this has allowed the full parser to maintain around</td></tr> |
1143 | <tr><td class="h"><a name="957"></a>957</td><td></td><td></td><td></td><td></td><td class="s">1000 LPGC despite the increasing weight of the Lexer.</td></tr> |
1144 | <tr><td class="h"><a name="958"></a>958</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1145 | <tr><td class="h"><a name="959"></a>959</td><td></td><td></td><td></td><td></td><td class="s">=head2 Making It Faster - Porting To C (In Progress)</td></tr> |
1146 | <tr><td class="h"><a name="960"></a>960</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1147 | <tr><td class="h"><a name="961"></a>961</td><td></td><td></td><td></td><td></td><td class="s">While it would be extraordinarily difficult to port all of the Tokenizer</td></tr> |
1148 | <tr><td class="h"><a name="962"></a>962</td><td></td><td></td><td></td><td></td><td class="s">to C, work has started on a L<PPI::XS> "accelerator" package which acts as</td></tr> |
1149 | <tr><td class="h"><a name="963"></a>963</td><td></td><td></td><td></td><td></td><td class="s">a separate and automatically-detected add-on to the main PPI package.</td></tr> |
1150 | <tr><td class="h"><a name="964"></a>964</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1151 | <tr><td class="h"><a name="965"></a>965</td><td></td><td></td><td></td><td></td><td class="s">L<PPI::XS> implements faster versions of a variety of functions scattered</td></tr> |
1152 | <tr><td class="h"><a name="966"></a>966</td><td></td><td></td><td></td><td></td><td class="s">over the entire PPI codebase, from the Tokenizer Core, Quote Engine, and</td></tr> |
1153 | <tr><td class="h"><a name="967"></a>967</td><td></td><td></td><td></td><td></td><td class="s">various other places, and implements them identically in XS/C.</td></tr> |
1154 | <tr><td class="h"><a name="968"></a>968</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1155 | <tr><td class="h"><a name="969"></a>969</td><td></td><td></td><td></td><td></td><td class="s">In particular, the skip-ahead methods from the Quote Engine would appear</td></tr> |
1156 | <tr><td class="h"><a name="970"></a>970</td><td></td><td></td><td></td><td></td><td class="s">to be extremely amenable to being done in C, and a number of other</td></tr> |
1157 | <tr><td class="h"><a name="971"></a>971</td><td></td><td></td><td></td><td></td><td class="s">functions could be cherry-picked one at a time and implemented in C.</td></tr> |
1158 | <tr><td class="h"><a name="972"></a>972</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1159 | <tr><td class="h"><a name="973"></a>973</td><td></td><td></td><td></td><td></td><td class="s">Each method is heavily tested to ensure that the functionality is</td></tr> |
1160 | <tr><td class="h"><a name="974"></a>974</td><td></td><td></td><td></td><td></td><td class="s">identical, and a versioning mechanism is included to ensure that if a</td></tr> |
1161 | <tr><td class="h"><a name="975"></a>975</td><td></td><td></td><td></td><td></td><td class="s">function gets out of sync, L<PPI::XS> will degrade gracefully and just</td></tr> |
1162 | <tr><td class="h"><a name="976"></a>976</td><td></td><td></td><td></td><td></td><td class="s">not replace that single method.</td></tr> |
1163 | <tr><td class="h"><a name="977"></a>977</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1164 | <tr><td class="h"><a name="978"></a>978</td><td></td><td></td><td></td><td></td><td class="s">=head1 TO DO</td></tr> |
1165 | <tr><td class="h"><a name="979"></a>979</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1166 | <tr><td class="h"><a name="980"></a>980</td><td></td><td></td><td></td><td></td><td class="s">- Add an option to reset or seek the token stream...</td></tr> |
1167 | <tr><td class="h"><a name="981"></a>981</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1168 | <tr><td class="h"><a name="982"></a>982</td><td></td><td></td><td></td><td></td><td class="s">- Implement more Tokenizer functions in L<PPI::XS></td></tr> |
1169 | <tr><td class="h"><a name="983"></a>983</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1170 | <tr><td class="h"><a name="984"></a>984</td><td></td><td></td><td></td><td></td><td class="s">=head1 SUPPORT</td></tr> |
1171 | <tr><td class="h"><a name="985"></a>985</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1172 | <tr><td class="h"><a name="986"></a>986</td><td></td><td></td><td></td><td></td><td class="s">See the L<support section|PPI/SUPPORT> in the main module.</td></tr> |
1173 | <tr><td class="h"><a name="987"></a>987</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1174 | <tr><td class="h"><a name="988"></a>988</td><td></td><td></td><td></td><td></td><td class="s">=head1 AUTHOR</td></tr> |
1175 | <tr><td class="h"><a name="989"></a>989</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1176 | <tr><td class="h"><a name="990"></a>990</td><td></td><td></td><td></td><td></td><td class="s">Adam Kennedy E<lt>adamk@cpan.orgE<gt></td></tr> |
1177 | <tr><td class="h"><a name="991"></a>991</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1178 | <tr><td class="h"><a name="992"></a>992</td><td></td><td></td><td></td><td></td><td class="s">=head1 COPYRIGHT</td></tr> |
1179 | <tr><td class="h"><a name="993"></a>993</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1180 | <tr><td class="h"><a name="994"></a>994</td><td></td><td></td><td></td><td></td><td class="s">Copyright 2001 - 2011 Adam Kennedy.</td></tr> |
1181 | <tr><td class="h"><a name="995"></a>995</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1182 | <tr><td class="h"><a name="996"></a>996</td><td></td><td></td><td></td><td></td><td class="s">This program is free software; you can redistribute</td></tr> |
1183 | <tr><td class="h"><a name="997"></a>997</td><td></td><td></td><td></td><td></td><td class="s">it and/or modify it under the same terms as Perl itself.</td></tr> |
1184 | <tr><td class="h"><a name="998"></a>998</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1185 | <tr><td class="h"><a name="999"></a>999</td><td></td><td></td><td></td><td></td><td class="s">The full text of the license can be found in the</td></tr> |
1186 | <tr><td class="h"><a name="1000"></a>1000</td><td></td><td></td><td></td><td></td><td class="s">LICENSE file included with this module.</td></tr> |
1187 | <tr><td class="h"><a name="1001"></a>1001</td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1188 | <tr><td class="h"><a name="1002"></a>1002</td><td></td><td></td><td></td><td></td><td class="s">=cut</td></tr> |
1189 | <tr><td class="s"><a name=""></a> </td><td></td><td></td><td></td><td></td><td class="s"></td></tr> |
1190 | <tr><td class="h"><a name="PPI__Tokenizer__CORE_match"></a></td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 20µs within PPI::Tokenizer::CORE:match which was called 16 times, avg 1µs/call: |
1191 | # 16 times (20µs+0s) by List::MoreUtils::any or PPI::Tokenizer::new at <a href="PPI-Tokenizer-pm-1003-block.html#211">line 211</a>, avg 1µs/call</div></div>sub PPI::Tokenizer::CORE:match; # opcode<br /> </td></tr> |
1192 | <tr><td class="h"><a name="PPI__Tokenizer__CORE_subst"></a></td><td></td><td></td><td></td><td></td><td class="s"><div class="calls"><div class="calls_in"># spent 63µs within PPI::Tokenizer::CORE:subst which was called 8 times, avg 8µs/call: |
1193 | # 8 times (63µs+0s) by PPI::Tokenizer::new at <a href="PPI-Tokenizer-pm-1003-block.html#186">line 186</a>, avg 8µs/call</div></div>sub PPI::Tokenizer::CORE:subst; # opcode<br /> </td></tr> |
1194 | </tbody></table></div> |
1195 | |
1196 | <script type="text/javascript"> $(document).ready(function() { |
1197 | |
1198 | $("#subs_table").tablesorter({ |
1199 | sortList: [[3,1]], |
1200 | headers: { |
1201 | 3: { sorter: 'fmt_time' }, |
1202 | 4: { sorter: 'fmt_time' } |
1203 | } |
1204 | }); |
1205 | |
1206 | } ); </script> |
1207 | |
1208 | <div class="footer">Report produced by the |
1209 | <a href="http://search.cpan.org/dist/Devel-NYTProf/">NYTProf 4.06</a> |
1210 | Perl profiler, developed by |
1211 | <a href="http://www.linkedin.com/in/timbunce">Tim Bunce</a> and |
1212 | <a href="http://code.nytimes.com">Adam Kaplan</a>. |
1213 | </div> |
1214 | <br /><br /><br /><br /><br /><br /><br /><br /><br /><br /> |
1215 | </body></html> |