Commit | Line | Data |
d71100ed |
1 | package Text::Tradition::Analysis; |
2 | |
3 | use strict; |
4 | use warnings; |
e4386ba9 |
5 | use Benchmark; |
d1348d38 |
6 | use Exporter 'import'; |
d71100ed |
7 | use Text::Tradition; |
8 | use Text::Tradition::Stemma; |
9 | |
d1348d38 |
10 | use vars qw/ @EXPORT_OK /; |
a2cf85dd |
11 | @EXPORT_OK = qw/ run_analysis group_variants analyze_variant_location wit_stringify /; |
d1348d38 |
12 | |
d71100ed |
13 | sub run_analysis { |
56cf65bd |
14 | my( $tradition ) = @_; |
d71100ed |
15 | # What we will return |
d71100ed |
16 | my $variants = []; |
e367f5c0 |
17 | my $data = {}; |
56cf65bd |
18 | |
19 | # We need a stemma in order to run this... |
20 | unless( $tradition->has_stemma ) { |
21 | warn "Tradition '" . $tradition->name . "' has no stemma to analyze"; |
22 | return undef; |
23 | } |
24 | my $stemma = $tradition->stemma; |
3837c155 |
25 | |
d71100ed |
26 | # We have the collation, so get the alignment table with witnesses in rows. |
27 | # Also return the reading objects in the table, rather than just the words. |
08e0fb85 |
28 | my $wits = {}; |
29 | map { $wits->{$_} = 1 } $stemma->witnesses; |
d71100ed |
30 | # For each column in the alignment table, we want to see if the existing |
f6c8ea08 |
31 | # groupings of witnesses match our stemma hypothesis. We also need to keep |
32 | # track of the maximum number of variants at any one location. |
33 | my $max_variants = 0; |
a2cf85dd |
34 | my ( $genealogical, $conflicts ) = ( 0, 0, 0 ); |
d71100ed |
35 | |
f6c8ea08 |
36 | my $variant_groups = group_variants( $tradition->collation, $wits ); |
37 | foreach my $rank ( 0 .. $#{$variant_groups} ) { |
38 | my $groups = $variant_groups->[$rank]->{'groups'}; |
39 | my $readings = $variant_groups->[$rank]->{'readings'}; |
40 | my $lacunose = $variant_groups->[$rank]->{'lacunose'}; |
d71100ed |
41 | |
f6c8ea08 |
42 | $max_variants = scalar @$groups if scalar @$groups > $max_variants; |
d71100ed |
43 | |
44 | # We can already look up witnesses for a reading; we also want to look |
45 | # up readings for a given witness. |
46 | my $group_readings = {}; |
47 | foreach my $x ( 0 .. $#$groups ) { |
48 | $group_readings->{wit_stringify( $groups->[$x] )} = $readings->[$x]; |
49 | } |
50 | |
51 | # For all the groups with more than one member, collect the list of all |
52 | # contiguous vertices needed to connect them. |
f6c8ea08 |
53 | my $variant_loc = analyze_variant_location( $group_readings, $groups, |
08e0fb85 |
54 | $stemma->graph, $lacunose ); |
f6c8ea08 |
55 | $variant_loc->{'id'} = $rank; |
56 | $genealogical++ if $variant_loc->{'genealogical'}; |
57 | $conflicts += grep { $_->{'conflict'} } @{$variant_loc->{'readings'}}; |
732152b1 |
58 | |
d71100ed |
59 | # Now run the same analysis given the calculated distance tree(s). |
732152b1 |
60 | # my @trees = @{$stemma->distance_trees}; |
61 | # if( @trees ) { |
62 | # foreach my $tree ( 0 .. $#trees ) { |
c4a4fb1b |
63 | # my $dc = analyze_variant_location( $group_readings, $groups, $tree, $lacunose, 'undirected' ); |
732152b1 |
64 | # foreach my $rdg ( keys %$dc ) { |
65 | # my $var = $dc->{$rdg}; |
66 | # # TODO Do something with this |
67 | # } |
68 | # } |
69 | # } |
70 | |
d71100ed |
71 | # Record that we used this variant in an analysis |
f6c8ea08 |
72 | push( @$variants, $variant_loc ); |
d71100ed |
73 | } |
a2cf85dd |
74 | |
f6c8ea08 |
75 | # Go through our variant locations, after we have seen all of them once, |
732152b1 |
76 | # and add the number of empty columns needed by each. |
d71100ed |
77 | foreach my $row ( @$variants ) { |
f6c8ea08 |
78 | my $empty = $max_variants - scalar @{$row->{'readings'}}; |
d71100ed |
79 | $row->{'empty'} = $empty; |
80 | } |
81 | |
e367f5c0 |
82 | $data->{'variants'} = $variants; |
a2cf85dd |
83 | $data->{'variant_count'} = $tradition->collation->end->rank - 1; |
e367f5c0 |
84 | $data->{'conflict_count'} = $conflicts; |
85 | $data->{'genealogical_count'} = $genealogical; |
3837c155 |
86 | return $data; |
d71100ed |
87 | } |
88 | |
d1348d38 |
89 | sub group_variants { |
90 | my( $c, $wits ) = @_; |
91 | my $variant_groups = []; |
92 | |
f6c8ea08 |
93 | # We have the collation, so get the alignment table with witnesses in rows. |
94 | # Also return the reading objects in the table, rather than just the words. |
d1348d38 |
95 | my $all_wits_table = $c->make_alignment_table( 'refs', $wits ); |
96 | # Strip the list of sigla and save it for correlation to the readings. |
b5c47c25 |
97 | my @table_wits = map { $_->{'witness'} } @{$all_wits_table->{'alignment'}}; |
d1348d38 |
98 | # Any witness in the stemma that has no row should be noted. |
b5c47c25 |
99 | foreach ( @table_wits ) { |
d1348d38 |
100 | $wits->{$_}++; # Witnesses present in table and stemma now have value 2. |
101 | } |
f6c8ea08 |
102 | my @not_collated = grep { $wits->{$_} == 1 } keys %$wits; |
b5c47c25 |
103 | foreach my $i ( 0 .. $all_wits_table->{'length'} - 1 ) { |
d1348d38 |
104 | # For each column in the table, group the readings by witness. |
105 | my $rdg_wits = {}; |
b5c47c25 |
106 | my @col_rdgs = map { $_->{tokens}->[$i] } @{$all_wits_table->{'alignment'}}; |
d1348d38 |
107 | my $lacunose = [ @not_collated ]; |
b5c47c25 |
108 | foreach my $j ( 0 .. $#col_rdgs ) { |
109 | my $rdg = $col_rdgs[$j]; |
d1348d38 |
110 | my $rdg_text = '(omitted)'; # Initialize in case of empty reading |
111 | if( $rdg ) { |
b5c47c25 |
112 | if( $rdg->{'t'}->is_lacuna ) { |
d1348d38 |
113 | $rdg_text = undef; # Don't count lacunae |
b5c47c25 |
114 | push( @$lacunose, $table_wits[$j] ); |
d1348d38 |
115 | } else { |
b5c47c25 |
116 | $rdg_text = $rdg->{'t'}->text; |
d1348d38 |
117 | } |
118 | } |
119 | if( defined $rdg_text ) { |
120 | # Initialize the witness array if we haven't got one yet |
121 | $rdg_wits->{$rdg_text} = [] unless $rdg_wits->{$rdg_text}; |
122 | # Add the relevant witness, subject to a.c. logic |
b5c47c25 |
123 | add_variant_wit( $rdg_wits->{$rdg_text}, $table_wits[$j], |
d1348d38 |
124 | $c->ac_label ); |
125 | } |
126 | } |
127 | |
128 | # See if this column has any potentially genealogical variants. |
129 | # If not, skip to the next. |
130 | my( $groups, $readings ) = useful_variant( $rdg_wits ); |
131 | next unless $groups && $readings; |
132 | |
f6c8ea08 |
133 | push( @$variant_groups, |
134 | { 'groups' => $groups, 'readings' => $readings, 'lacunose' => $lacunose } ); |
d1348d38 |
135 | } |
136 | return $variant_groups; |
137 | } |
138 | |
f6c8ea08 |
139 | |
140 | |
732152b1 |
141 | # variant_row -> genealogical |
142 | # -> readings [ { text, group, conflict, missing } ] |
143 | |
d71100ed |
144 | sub analyze_variant_location { |
c4a4fb1b |
145 | my( $group_readings, $groups, $graph, $lacunose, $undirected ) = @_; |
231d71fc |
146 | my $contig = {}; |
147 | my $subgraph = {}; |
c4a4fb1b |
148 | my $is_conflicted; |
d71100ed |
149 | my $conflict = {}; |
231d71fc |
150 | my $missing = {}; |
151 | map { $missing->{$_} = 1 } @$lacunose; |
732152b1 |
152 | my $variant_row = { 'readings' => [] }; |
94a077d6 |
153 | # Mark each ms as in its own group, first. |
154 | foreach my $g ( @$groups ) { |
155 | my $gst = wit_stringify( $g ); |
231d71fc |
156 | map { $contig->{$_} = $gst } @$g; |
94a077d6 |
157 | } |
c4a4fb1b |
158 | # Now for each unmarked node in the graph, initialize an array |
159 | # for possible group memberships. We will use this later to |
160 | # resolve potential conflicts. |
231d71fc |
161 | map { $contig->{$_} = [] unless $contig->{$_} } $graph->vertices; |
d71100ed |
162 | foreach my $g ( sort { scalar @$b <=> scalar @$a } @$groups ) { |
c4a4fb1b |
163 | my $gst = wit_stringify( $g ); # This is the group name |
164 | my $reachable = { $g->[0] => 1 }; |
08e0fb85 |
165 | # Copy the graph, and delete all non-members from the new graph. |
c4a4fb1b |
166 | my $part = $graph->copy; |
167 | my $group_root; |
168 | $part->delete_vertices( |
231d71fc |
169 | grep { !ref( $contig->{$_} ) && $contig->{$_} ne $gst } $graph->vertices ); |
c4a4fb1b |
170 | |
171 | # Now look to see if our group is connected. |
172 | if( $undirected ) { # For use with distance trees etc. |
173 | # Find all vertices reachable from the first (arbitrary) group |
174 | # member. If we are genealogical this should include them all. |
175 | map { $reachable->{$_} = 1 } $part->all_reachable( $g->[0] ); |
176 | # TODO This is a terrible way to do distance trees, since all |
177 | # non-leaf nodes are included in every graph part now. We may |
178 | # have to go back to SPDP. |
179 | } else { |
180 | if( @$g > 1 ) { |
181 | # Dispense with the trivial case of one reading. |
182 | # We have to take directionality into account. |
183 | # How many root nodes do we have? |
231d71fc |
184 | my @roots = grep { ref( $contig->{$_} ) || $contig->{$_} eq $gst } |
c4a4fb1b |
185 | $part->source_vertices; |
186 | # Assuming that @$g > 1, find the first root node that has at |
187 | # least one successor belonging to our group. If this reading |
188 | # is genealogical, there should be only one, but we will check |
189 | # that implicitly later. |
190 | my $nodes_in_subtree = 0; |
191 | foreach my $root ( @roots ) { |
231d71fc |
192 | # Prune the tree to get rid of extraneous hypotheticals. |
193 | $root = prune_subtree( $part, $root, $contig ); |
c4a4fb1b |
194 | # Get all the successor nodes of our root. |
195 | my $tmp_reach = { $root => 1 }; |
196 | map { $tmp_reach->{$_} = 1 } $part->all_successors( $root ); |
197 | # Skip this root if none of our successors are in our group |
198 | # (e.g. isolated 'hypothetical' witnesses with no group) |
231d71fc |
199 | next unless grep { $contig->{$_} } keys %$tmp_reach; |
c4a4fb1b |
200 | if( keys %$tmp_reach > $nodes_in_subtree ) { |
201 | $nodes_in_subtree = keys %$tmp_reach; |
202 | $reachable = $tmp_reach; |
203 | $group_root = $root; |
204 | } |
205 | } |
206 | } # else it is a single-node group, nothing to calculate. |
207 | } |
208 | |
209 | # None of the 'reachable' nodes should be marked as being in another |
210 | # group. Paint the 'hypotheticals' with our group while we are at it, |
211 | # unless there is a conflict present. |
212 | foreach ( keys %$reachable ) { |
231d71fc |
213 | if( ref $contig->{$_} ) { |
214 | push( @{$contig->{$_}}, $gst ); |
215 | } elsif( $contig->{$_} ne $gst ) { |
216 | $conflict->{$group_readings->{$gst}} = $group_readings->{$contig->{$_}}; |
c4a4fb1b |
217 | } # else it is an 'extant' node marked with our group already. |
d71100ed |
218 | } |
08e0fb85 |
219 | # None of the unreachable nodes should be in our group either. |
220 | foreach ( $part->vertices ) { |
c4a4fb1b |
221 | next if $reachable->{$_}; |
231d71fc |
222 | if( $contig->{$_} eq $gst ) { |
c4a4fb1b |
223 | $conflict->{$group_readings->{$gst}} = $group_readings->{$gst}; |
224 | last; |
225 | } |
08e0fb85 |
226 | } |
227 | |
c4a4fb1b |
228 | # Now, if we have a conflict, we can write the reading in full. If not, |
229 | # we have to save the subgraph so that we can resolve possible conflicts |
230 | # on hypothetical nodes. |
231 | $is_conflicted = 1 if exists $conflict->{$group_readings->{$gst}}; |
232 | |
732152b1 |
233 | # Write the reading. |
234 | my $reading = { 'text' => $group_readings->{$gst}, |
235 | 'missing' => wit_stringify( $lacunose ), |
c4a4fb1b |
236 | 'group' => $gst }; # This will change if we find no conflict |
237 | if( $is_conflicted ) { |
238 | $reading->{'conflict'} = $conflict->{$group_readings->{$gst}} |
732152b1 |
239 | } else { |
c4a4fb1b |
240 | # Save the relevant subgraph. |
231d71fc |
241 | $subgraph->{$gst} = { 'graph' => $part, |
c4a4fb1b |
242 | 'root' => $group_root, |
243 | 'reachable' => $reachable }; |
732152b1 |
244 | } |
245 | push( @{$variant_row->{'readings'}}, $reading ); |
d71100ed |
246 | } |
c4a4fb1b |
247 | |
248 | # Now that we have gone through all the rows, check the hypothetical |
249 | # readings for conflict if we haven't found one yet. |
231d71fc |
250 | if( keys %$subgraph && !keys %$conflict ) { |
c4a4fb1b |
251 | my @resolve; |
231d71fc |
252 | foreach ( keys %$contig ) { |
253 | next unless ref $contig->{$_}; |
254 | if( scalar @{$contig->{$_}} > 1 ) { |
c4a4fb1b |
255 | push( @resolve, $_ ); |
256 | } else { |
231d71fc |
257 | $contig->{$_} = scalar @{$contig->{$_}} ? $contig->{$_}->[0] : ''; |
c4a4fb1b |
258 | } |
259 | } |
260 | # Do we still have a possible conflict? |
231d71fc |
261 | my $still_contig = {}; |
c4a4fb1b |
262 | foreach my $h ( @resolve ) { |
263 | # For each of the hypothetical readings with more than one possibility, |
264 | # try deleting it from each of its member subgraphs in turn, and see |
265 | # if that breaks the contiguous grouping. |
266 | # TODO This can still break in a corner case where group A can use |
267 | # either vertex 1 or 2, and group B can use either vertex 2 or 1. |
268 | # Revisit this if necessary; it could get brute-force nasty. |
231d71fc |
269 | foreach my $gst ( @{$contig->{$h}} ) { |
270 | my $gpart = $subgraph->{$gst}->{'graph'}->copy; |
271 | my $reachable = $subgraph->{$gst}->{'reachable'}; |
c4a4fb1b |
272 | $gpart->delete_vertex( $h ); |
273 | # Is everything else still reachable from the root? |
274 | # TODO If $h was the root, see if we still have a single root. |
231d71fc |
275 | my %still_reachable = ( $subgraph->{$gst}->{'root'} => 1 ); |
c4a4fb1b |
276 | map { $still_reachable{$_} = 1 } |
231d71fc |
277 | $gpart->all_successors( $subgraph->{$gst}->{'root'} ); |
c4a4fb1b |
278 | foreach my $v ( keys %$reachable ) { |
279 | next if $v eq $h; |
280 | if( !$still_reachable{$v} |
231d71fc |
281 | && ( $contig->{$v} eq $gst |
282 | || ( exists $still_contig->{$v} |
283 | && $still_contig->{$v} eq $gst ) ) ) { |
c4a4fb1b |
284 | # We need $h. |
231d71fc |
285 | if( exists $still_contig->{$h} ) { |
c4a4fb1b |
286 | # Conflict! |
287 | $conflict->{$group_readings->{$gst}} = |
231d71fc |
288 | $group_readings->{$still_contig->{$h}}; |
c4a4fb1b |
289 | } else { |
231d71fc |
290 | $still_contig->{$h} = $gst; |
c4a4fb1b |
291 | } |
292 | last; |
293 | } # else we don't need $h in this group. |
294 | } |
295 | } |
296 | } |
297 | |
298 | # Now, assuming no conflict, we have some hypothetical vertices in |
299 | # $still_contig that are the "real" group memberships. Replace these |
300 | # in $contig. |
301 | unless ( keys %$conflict ) { |
231d71fc |
302 | foreach my $v ( keys %$contig ) { |
303 | next unless ref $contig->{$v}; |
304 | $contig->{$v} = $still_contig->{$v}; |
c4a4fb1b |
305 | } |
306 | } |
307 | } |
308 | |
309 | # Now write the group and conflict information into the respective rows. |
310 | foreach my $rdg ( @{$variant_row->{'readings'}} ) { |
311 | $rdg->{'conflict'} = $conflict->{$rdg->{'text'}}; |
312 | next if $rdg->{'conflict'}; |
231d71fc |
313 | my @members = grep { $contig->{$_} eq $rdg->{'group'} && !$missing->{$_} } |
314 | keys %$contig; |
c4a4fb1b |
315 | $rdg->{'group'} = wit_stringify( \@members ); |
316 | } |
317 | |
08e0fb85 |
318 | $variant_row->{'genealogical'} = !( keys %$conflict ); |
732152b1 |
319 | return $variant_row; |
d71100ed |
320 | } |
321 | |
231d71fc |
322 | sub prune_subtree { |
323 | my( $tree, $root, $contighash ) = @_; |
324 | # First, delete hypothetical leaves / orphans until there are none left. |
325 | my @orphan_hypotheticals = grep { ref( $contighash->{$_} ) } |
326 | $tree->successorless_vertices; |
327 | while( @orphan_hypotheticals ) { |
328 | $tree->delete_vertices( @orphan_hypotheticals ); |
329 | @orphan_hypotheticals = grep { ref( $contighash->{$_} ) } |
330 | $tree->successorless_vertices; |
331 | } |
332 | # Then delete a hypothetical root with only one successor, moving the |
333 | # root to the child. |
334 | while( $tree->successors( $root ) == 1 && ref $contighash->{$root} ) { |
335 | my @nextroot = $tree->successors( $root ); |
336 | $tree->delete_vertex( $root ); |
337 | $root = $nextroot[0]; |
338 | } |
339 | # The tree has been modified in place, but we need to know the new root. |
340 | return $root; |
341 | } |
d71100ed |
342 | # Add the variant, subject to a.c. representation logic. |
343 | # This assumes that we will see the 'main' version before the a.c. version. |
344 | sub add_variant_wit { |
345 | my( $arr, $wit, $acstr ) = @_; |
346 | my $skip; |
347 | if( $wit =~ /^(.*)\Q$acstr\E$/ ) { |
348 | my $real = $1; |
349 | $skip = grep { $_ =~ /^\Q$real\E$/ } @$arr; |
350 | } |
351 | push( @$arr, $wit ) unless $skip; |
352 | } |
353 | |
354 | # Return an answer if the variant is useful, i.e. if there are at least 2 variants |
355 | # with at least 2 witnesses each. |
356 | sub useful_variant { |
357 | my( $readings ) = @_; |
358 | my $total = keys %$readings; |
359 | foreach my $var ( keys %$readings ) { |
360 | $total-- if @{$readings->{$var}} == 1; |
361 | } |
362 | return( undef, undef ) if $total <= 1; |
363 | my( $groups, $text ); |
364 | foreach my $var ( keys %$readings ) { |
365 | push( @$groups, $readings->{$var} ); |
366 | push( @$text, $var ); |
367 | } |
368 | return( $groups, $text ); |
369 | } |
370 | |
371 | # Take an array of witness groupings and produce a string like |
372 | # ['A','B'] / ['C','D','E'] / ['F'] |
373 | |
374 | sub wit_stringify { |
375 | my $groups = shift; |
376 | my @gst; |
377 | # If we were passed an array of witnesses instead of an array of |
378 | # groupings, then "group" the witnesses first. |
379 | unless( ref( $groups->[0] ) ) { |
380 | my $mkgrp = [ $groups ]; |
381 | $groups = $mkgrp; |
382 | } |
383 | foreach my $g ( @$groups ) { |
384 | push( @gst, '[' . join( ',', map { "'$_'" } @$g ) . ']' ); |
385 | } |
386 | return join( ' / ', @gst ); |
387 | } |
388 | |
389 | 1; |