3 # Read the raw memory data from Devel::Memory and process the tree
4 # (as a stack, propagating data such as totals, up the tree).
5 # Output completed nodes in the request formats.
7 # Needs to be generalized to support pluggable output formats.
8 # Actually it needs to be split so sizeme_store.pl only does the store
9 # and another program drives the output with plugins.
10 # Making nodes into (lightweight fast) objects would be smart.
11 # Tests would be even smarter!
13 # When working on this code it's important to have a sense of the flow.
14 # Specifically the way that depth drives the completion of nodes.
15 # It's a depth-first stream processing machine, which only ever holds
16 # a single stack of the currently incomplete nodes, which is always the same as
17 # the current depth. I.e., when a node of depth N arrives, all nodes >N are
18 # popped off the stack and 'completed', each rippling data up to its parent.
24 use DBI qw(looks_like_number);
28 use HTML::Entities qw(encode_entities);;
31 use Carp qw(carp croak confess);
33 # XXX import these from the XS code
34 use constant NPtype_NAME => 0x01;
35 use constant NPtype_LINK => 0x02;
36 use constant NPtype_SV => 0x03;
37 use constant NPtype_MAGIC => 0x04;
38 use constant NPtype_OP => 0x05;
40 use constant NPattr_LEAFSIZE => 0x00;
41 use constant NPattr_NAME => 0x01;
42 use constant NPattr_PADFAKE => 0x02;
43 use constant NPattr_PADNAME => 0x03;
44 use constant NPattr_PADTMP => 0x04;
45 use constant NPattr_NOTE => 0x05;
46 use constant NPattr_PRE_ATTR => 0x06;
47 my @attr_type_name = (qw(size NAME PADFAKE my PADTMP NOTE PREATTR)); # XXX get from XS in some way
51 'text!' => \my $opt_text,
52 'dot=s' => \my $opt_dot,
53 'db=s' => \my $opt_db,
54 'verbose|v!' => \my $opt_verbose,
55 'debug|d!' => \my $opt_debug,
56 'showid!' => \my $opt_showid,
63 my $j = JSON::XS->new->ascii->pretty(0);
65 my ($dbh, $node_ins_sth);
67 $dbh = DBI->connect("dbi:SQLite:dbname=$opt_db","","", {
68 RaiseError => 1, PrintError => 0, AutoCommit => 0
70 $dbh->do("PRAGMA synchronous = OFF");
77 my $name = encode_entities(shift);
87 my $kb = $size / 1024;
88 return $size if $kb < 5;
89 return sprintf "%.1fKb", $kb if $kb < 1000;
90 return sprintf "%.1fMb", $kb/1024;
96 warn ">> enter_node $x->{id}\n" if $opt_debug;
98 my $parent = $stack[-1];
101 if ($x->{name} eq 'AVelem' and $parent->{name} eq 'SV(PVAV)') {
102 my $index = $x->{attr}{+NPattr_NOTE}{i};
105 # If node is an AVelem of a CvPADLIST propagate pad name to AVelem
106 if (@stack >= 4 and (my $cvpl = $stack[-4])->{name} eq 'CvPADLIST') {
107 my $padnames = $cvpl->{_cached}{padnames} ||= do {
108 my @names = @{ $cvpl->{attr}{+NPattr_PADNAME} || []};
109 $_ = "my(".($_||'').")" for @names;
113 $x->{name} = (defined $index and $padnames->[$index]) || "?";
114 $x->{name} =~ s/my\(SVs_PADTMP\)/PADTMP/; # XXX hack for neatness
117 $x->{name} = "[$index]" if defined $index;
129 confess unless defined $x->{id};
130 warn "<< leave_node $x->{id}\n" if $opt_debug;
131 delete $seqn2node{$x->{id}};
133 my $self_size = 0; $self_size += $_ for values %{$x->{leaves}};
134 $x->{self_size} = $self_size;
136 if ($x->{name} eq 'AVelem') {
137 my $index = $x->{attr}{+NPattr_NOTE}{i};
138 $x->{name} = "[$index]" if defined $index;
141 my $parent = $stack[-1];
144 $x->{parent_id} = $parent->{id};
145 # accumulate into parent
146 $parent->{kids_node_count} += 1 + ($x->{kids_node_count}||0);
147 $parent->{kids_size} += $self_size + $x->{kids_size};
148 push @{$parent->{child_id}}, $x->{id};
151 $x->{kids_node_count} ||= 0;
157 printf "// n%d parent=%s(type=%s)\n", $x->{id},
158 $parent ? $parent->{id} : "",
159 $parent ? $parent->{type} : ""
162 if ($x->{type} != NPtype_LINK) {
163 my $name = $x->{title} ? "\"$x->{title}\" $x->{name}" : $x->{name};
165 if ($x->{kids_size}) {
166 $name .= sprintf " %s+%s=%s", fmt_size($x->{self_size}), fmt_size($x->{kids_size}), fmt_size($x->{self_size}+$x->{kids_size});
169 $name .= sprintf " +%s", fmt_size($x->{self_size});
171 $name .= " #$x->{id}" if $opt_showid;
174 sprintf("label=%s", $dotnode->($name)),
177 printf $dot_fh qq{n%d [ %s ];\n}, $x->{id}, join(",", @node_attr);
180 my @kids = @{$x->{child_id}||[]};
181 die "panic: NPtype_LINK has more than one child: @kids"
183 for my $child_id (@kids) { # wouldn't work right, eg id= attr
185 my @link_attr = ("id=$x->{id}");
186 (my $link_name = $x->{name}) =~ s/->$//;
187 $link_name .= " #$x->{id}" if $opt_showid;
188 push @link_attr, (sprintf "label=%s", $dotnode->($link_name));
189 printf $dot_fh qq{n%d -> n%d [%s];\n},
190 $x->{parent_id}, $child_id, join(",", @link_attr);
196 my $attr_json = $j->encode($x->{attr});
197 my $leaves_json = $j->encode($x->{leaves});
198 $node_ins_sth->execute(
199 $x->{id}, $x->{name}, $x->{title}, $x->{type}, $x->{depth}, $x->{parent_id},
200 $x->{self_size}, $x->{kids_size}, $x->{kids_node_count},
201 $x->{child_id} ? join(",", @{$x->{child_id}}) : undef,
202 $attr_json, $leaves_json,
211 my $pending_pre_attr = {};
214 warn "\t\t\t\t== $_" if $opt_debug;
217 my ($type, $id, $val, $name, $extra) = split / /, $_, 5;
219 if ($type =~ s/^-//) { # Node type ($val is depth)
221 printf "%s%s%s %s [#%d @%d]\n", $indent x $val, $name,
222 ($type == NPtype_LINK) ? "->" : "",
223 $extra||'', $id, $val
226 # this is the core driving logic
227 while ($val < @stack) {
228 my $x = leave_node(pop @stack);
229 warn "N $id d$val ends $x->{id} d$x->{depth}: size $x->{self_size}+$x->{kids_size}\n"
232 die "panic: stack already has item at depth $val"
234 die "Depth out of sync\n" if $val != @stack;
235 my $node = enter_node({
236 id => $id, type => $type, name => $name, extra => $extra,
237 attr => { %$pending_pre_attr },
238 leaves => {}, depth => $val, self_size=>0, kids_size=>0
240 %$pending_pre_attr = ();
241 $stack[$val] = $node;
242 $seqn2node{$id} = $node;
245 # --- Leaf name and memory size
246 elsif ($type eq "L") {
247 my $node = $seqn2node{$id} || die;
248 $node->{leaves}{$name} += $val;
250 printf "%s+%d=%d %s\n", $indent x ($node->{depth}+1), $val, $run_size, $name
254 # --- Attribute type, name and value (all rather hackish)
255 elsif (looks_like_number($type)) {
256 my $node = $seqn2node{$id} || die;
257 my $attr = $node->{attr} || die;
259 # attributes to queue up and apply to the next node
260 if (NPattr_PRE_ATTR == $type) {
261 $pending_pre_attr->{$name} = $val;
263 # attributes where the string is a key (or always empty and the type is the key)
264 elsif ($type == NPattr_NAME or $type == NPattr_NOTE) {
265 printf "%s~%s(%s) %d [t%d]\n", $indent x ($node->{depth}+1), $attr_type_name[$type], $name, $val, $type
267 warn "Node $id already has attribute $type:$name (value $attr->{$type}{$name})\n"
268 if exists $attr->{$type}{$name};
269 $attr->{$type}{$name} = $val;
271 $node->{title} = $name if $type == NPattr_NAME and !$val; # XXX hack
273 # attributes where the number is a key (or always zero)
274 elsif (NPattr_PADFAKE==$type or NPattr_PADTMP==$type or NPattr_PADNAME==$type) {
275 printf "%s~%s('%s') %d [t%d]\n", $indent x ($node->{depth}+1), $attr_type_name[$type], $name, $val, $type
277 warn "Node $id already has attribute $type:$name (value $attr->{$type}[$val])\n"
278 if defined $attr->{$type}[$val];
279 $attr->{+NPattr_PADNAME}[$val] = $name; # store all as NPattr_PADNAME
282 printf "%s~%s %d [t%d]\n", $indent x ($node->{depth}+1), $name, $val, $type
284 warn "Invalid attribute type '$type' on line $. ($_)";
287 elsif ($type eq 'S') { # start of a run
288 die "Unexpected start token" if @stack;
290 open $dot_fh, ">$opt_dot";
291 print $dot_fh "digraph {\n"; # }
292 print $dot_fh "graph [overlap=false]\n"; # target="???", URL="???"
295 # XXX add a size_run table records each run
296 # XXX pick a table name to store the run nodes in
297 #$run_ins_sth->execute(
299 $dbh->do("DROP TABLE IF EXISTS $table");
301 CREATE TABLE $table (
302 id integer primary key,
311 kids_node_count integer,
317 $node_ins_sth = $dbh->prepare(qq{
318 INSERT INTO $table VALUES (?,?,?,?,?,?, ?,?,?,?,?,?)
322 elsif ($type eq 'E') { # end of a run
324 my $top = $stack[0]; # grab top node before we pop all the nodes
325 leave_node(pop @stack) while @stack;
327 my $top_size = $top->{self_size}+$top->{kids_size};
329 printf "Stored %d nodes (${.}n) sizing %s (%d) in %.2fs\n",
330 $top->{kids_node_count}, fmt_size($top_size), $top_size,
332 # the duration here ($val) is from Devel::SizeMe perspective
333 # ie doesn't include time to read file/pipe and commit to database.
335 if ($opt_verbose or $run_size != $top_size) {
336 warn "EOF ends $top->{id} d$top->{depth}: size $top->{self_size}+$top->{kids_size}\n";
339 die "panic: seqn2node should be empty ". Dumper(\%seqn2node)
341 %$pending_pre_attr = ();
346 system("open -a Graphviz $opt_dot") if $^O eq 'darwin'; # OSX
349 $dbh->commit if $dbh;
352 warn "Invalid type '$type' on line $. ($_)";
356 $dbh->commit if $dbh and $id % 10_000 == 0;
358 die "EOF without end token" if @stack;
361 =for This is out of date but gives you an idea of the data and stream
363 SV(PVAV) fill=1/1 [#1 @0]
370 : : : : SV(PVAV) fill=-1/-1 [#5 @4]
371 : : : : : +64 sv =168
378 N 1 0 SV(PVAV) fill=1/1
385 N 5 4 SV(PVAV) fill=-1/-1