Commit | Line | Data |
f6066bac |
1 | package Text::Tradition::Parser::TEI; |
2 | |
3 | use strict; |
4 | use warnings; |
910a0a6d |
5 | use Text::Tradition::Parser::Util qw( collate_variants ); |
f6066bac |
6 | use XML::LibXML; |
7 | use XML::LibXML::XPathContext; |
8 | |
9 | =head1 NAME |
10 | |
11 | Text::Tradition::Parser::TEI |
12 | |
3b853983 |
13 | =head1 SYNOPSIS |
14 | |
15 | use Text::Tradition; |
16 | |
17 | my $t_from_file = Text::Tradition->new( |
18 | 'name' => 'my text', |
19 | 'input' => 'TEI', |
20 | 'file' => '/path/to/parallel_seg_file.xml' |
21 | ); |
22 | |
23 | my $t_from_string = Text::Tradition->new( |
24 | 'name' => 'my text', |
25 | 'input' => 'TEI', |
26 | 'string' => $parallel_seg_xml, |
27 | ); |
28 | |
29 | |
f6066bac |
30 | =head1 DESCRIPTION |
31 | |
3b853983 |
32 | Parser module for Text::Tradition, given a TEI parallel-segmentation file |
33 | that describes a text and its variants. Normally called upon |
34 | initialization of Text::Tradition. |
35 | |
36 | The witnesses for the tradition are taken from the <listWit/> element |
37 | within the TEI header; the readings are taken from any <p/> element that |
38 | appears in the text body (including <head/> elements therein.) |
f6066bac |
39 | |
40 | =head1 METHODS |
41 | |
42 | =over |
43 | |
3b853983 |
44 | =item B<parse>( $tradition, $option_hash ) |
45 | |
46 | Takes an initialized tradition and a set of options; creates the |
47 | appropriate nodes and edges on the graph, as well as the appropriate |
48 | witness objects. The $option_hash must contain either a 'file' or a |
49 | 'string' argument with the XML to be parsed. |
50 | |
51 | =begin testing |
52 | |
53 | use Text::Tradition; |
54 | binmode STDOUT, ":utf8"; |
55 | binmode STDERR, ":utf8"; |
56 | eval { no warnings; binmode $DB::OUT, ":utf8"; }; |
57 | |
58 | my $par_seg = 't/data/florilegium_tei_ps.xml'; |
59 | my $t = Text::Tradition->new( |
60 | 'name' => 'inline', |
61 | 'input' => 'TEI', |
62 | 'file' => $par_seg, |
63 | ); |
f6066bac |
64 | |
3b853983 |
65 | is( ref( $t ), 'Text::Tradition', "Parsed parallel-segmentation TEI" ); |
66 | if( $t ) { |
67 | is( scalar $t->collation->readings, 319, "Collation has all readings" ); |
68 | is( scalar $t->collation->paths, 2854, "Collation has all paths" ); |
69 | } |
f6066bac |
70 | |
3b853983 |
71 | =end testing |
f6066bac |
72 | |
73 | =cut |
74 | |
910a0a6d |
75 | my $text = {}; # Hash of arrays, one per eventual witness we find. |
910a0a6d |
76 | my $substitutions = {}; # Keep track of merged readings |
77 | my $app_anchors = {}; # Track apparatus references |
78 | my $app_ac = {}; # Save a.c. readings |
eca16057 |
79 | my $app_count; # Keep track of how many apps we have |
910a0a6d |
80 | |
81 | # Create the package variables for tag names. |
82 | |
83 | # Would really like to do this with varname variables, but apparently this |
84 | # is considered a bad idea. The long way round then. |
85 | my( $LISTWIT, $WITNESS, $TEXT, $W, $SEG, $APP, $RDG, $LEM ) |
86 | = ( 'listWit', 'witness', 'text', 'w', 'seg', 'app', 'rdg', 'lem' ); |
3b853983 |
87 | sub _make_tagnames { |
910a0a6d |
88 | my( $ns ) = @_; |
89 | if( $ns ) { |
90 | $LISTWIT = "$ns:$LISTWIT"; |
91 | $WITNESS = "$ns:$WITNESS"; |
92 | $TEXT = "$ns:$TEXT"; |
93 | $W = "$ns:$W"; |
94 | $SEG = "$ns:$SEG"; |
95 | $APP = "$ns:$APP"; |
96 | $RDG = "$ns:$RDG"; |
97 | $LEM = "$ns:$LEM"; |
98 | } |
99 | } |
100 | |
101 | # Parse the TEI file. |
f6066bac |
102 | sub parse { |
dfc37e38 |
103 | my( $tradition, $opts ) = @_; |
f6066bac |
104 | |
105 | # First, parse the XML. |
106 | my $parser = XML::LibXML->new(); |
dfc37e38 |
107 | my $doc; |
108 | if( exists $opts->{'string'} ) { |
109 | $doc = $parser->parse_string( $opts->{'string'} ); |
110 | } elsif ( exists $opts->{'file'} ) { |
111 | $doc = $parser->parse_file( $opts->{'file'} ); |
112 | } else { |
113 | warn "Could not find string or file option to parse"; |
114 | return; |
115 | } |
f6066bac |
116 | my $tei = $doc->documentElement(); |
f2b9605f |
117 | my $xpc = XML::LibXML::XPathContext->new( $tei ); |
910a0a6d |
118 | my $ns; |
119 | if( $tei->namespaceURI ) { |
120 | $ns = 'tei'; |
121 | $xpc->registerNs( $ns, $tei->namespaceURI ); |
122 | } |
3b853983 |
123 | _make_tagnames( $ns ); |
910a0a6d |
124 | |
f6066bac |
125 | # Then get the witnesses and create the witness objects. |
910a0a6d |
126 | foreach my $wit_el ( $xpc->findnodes( "//$LISTWIT/$WITNESS" ) ) { |
127 | my $sig = $wit_el->getAttribute( 'xml:id' ); |
128 | my $source = $wit_el->toString(); |
129 | $tradition->add_witness( sigil => $sig, source => $source ); |
f6066bac |
130 | } |
910a0a6d |
131 | map { $text->{$_->sigil} = [] } $tradition->witnesses; |
eca16057 |
132 | |
910a0a6d |
133 | # Look for all word/seg node IDs and note their pre-existence. |
134 | my @attrs = $xpc->findnodes( "//$W|$SEG/attribute::xml:id" ); |
3b853983 |
135 | _save_preexisting_nodeids( @attrs ); |
910a0a6d |
136 | |
eca16057 |
137 | # Count up how many apps we have. |
138 | my @apps = $xpc->findnodes( "//$APP" ); |
139 | $app_count = scalar( @apps ); |
140 | |
910a0a6d |
141 | # Now go through the children of the text element and pull out the |
142 | # actual text. |
143 | foreach my $xml_el ( $xpc->findnodes( "//$TEXT" ) ) { |
144 | foreach my $xn ( $xml_el->childNodes ) { |
145 | _get_readings( $tradition, $xn ); |
146 | } |
147 | } |
148 | # Our $text global now has lists of readings, one per witness. |
149 | # Join them up. |
150 | my $c = $tradition->collation; |
151 | foreach my $sig ( keys %$text ) { |
152 | next if $sig eq 'base'; # Skip base text readings with no witnesses. |
153 | # Determine the list of readings for |
154 | my $sequence = $text->{$sig}; |
155 | my @real_sequence = ( $c->start ); |
156 | push( @$sequence, $c->end ); |
157 | my $source = $c->start; |
158 | foreach( _clean_sequence( $sig, $sequence ) ) { |
159 | my $rdg = _return_rdg( $_ ); |
160 | push( @real_sequence, $rdg ); |
161 | $c->add_path( $source, $rdg, $sig ); |
162 | $source = $rdg; |
163 | } |
164 | $tradition->witness( $sig )->path( \@real_sequence ); |
165 | # See if we need to make an a.c. version of the witness. |
166 | if( exists $app_ac->{$sig} ) { |
167 | my @uncorrected; |
168 | push( @uncorrected, @real_sequence ); |
169 | foreach my $app ( keys %{$app_ac->{$sig}} ) { |
170 | my $start = _return_rdg( $app_anchors->{$app}->{$sig}->{'start'} ); |
171 | my $end = _return_rdg( $app_anchors->{$app}->{$sig}->{'end'} ); |
172 | my @new = map { _return_rdg( $_ ) } @{$app_ac->{$sig}->{$app}}; |
173 | _replace_sequence( \@uncorrected, $start, $end, @new ); |
174 | } |
175 | my $source = $c->start; |
176 | foreach my $rdg ( @uncorrected ) { |
177 | my $has_base = grep { $_->label eq $sig } $source->edges_to( $rdg ); |
178 | if( $rdg ne $c->start && !$has_base ) { |
179 | print STDERR sprintf( "Adding path %s from %s -> %s\n", |
180 | $sig.$c->ac_label, $source->name, $rdg->name ); |
181 | $c->add_path( $source, $rdg, $sig.$c->ac_label ); |
182 | } |
183 | $source = $rdg; |
184 | } |
78fab1cf |
185 | print STDERR "Adding a.c. version for witness $sig\n"; |
910a0a6d |
186 | $tradition->witness( $sig )->uncorrected_path( \@uncorrected ); |
187 | } |
188 | } |
3b853983 |
189 | |
190 | # Calculate the ranks for the nodes. |
c9bf3dbf |
191 | $tradition->collation->calculate_ranks(); |
3bc0cd18 |
192 | |
193 | # Now that we have ranks, see if we have distinct nodes with identical |
194 | # text and identical rank that can be merged. |
0e476982 |
195 | $tradition->collation->flatten_ranks(); |
910a0a6d |
196 | } |
197 | |
198 | sub _clean_sequence { |
199 | my( $wit, $sequence ) = @_; |
200 | my @clean_sequence; |
201 | foreach my $rdg ( @$sequence ) { |
202 | if( $rdg =~ /^PH-(.*)$/ ) { |
203 | # It is a placeholder. Keep it only if we need it. |
204 | my $app_id = $1; |
78fab1cf |
205 | if( exists $app_ac->{$wit} && |
206 | exists $app_ac->{$wit}->{$app_id} ) { |
910a0a6d |
207 | print STDERR "Retaining empty placeholder for $app_id\n"; |
208 | push( @clean_sequence, $rdg ); |
209 | } |
210 | } else { |
211 | push( @clean_sequence, $rdg ); |
212 | } |
f6066bac |
213 | } |
910a0a6d |
214 | return @clean_sequence; |
215 | } |
f6066bac |
216 | |
910a0a6d |
217 | sub _replace_sequence { |
218 | my( $arr, $start, $end, @new ) = @_; |
219 | my( $start_idx, $end_idx ); |
220 | foreach my $i ( 0 .. $#{$arr} ) { |
221 | $start_idx = $i if( $arr->[$i]->name eq $start ); |
222 | if( $arr->[$i]->name eq $end ) { |
223 | $end_idx = $i; |
224 | last; |
225 | } |
226 | } |
227 | unless( $start_idx && $end_idx ) { |
228 | warn "Could not find start and end"; |
229 | return; |
f2b9605f |
230 | } |
910a0a6d |
231 | my $length = $end_idx - $start_idx + 1; |
232 | splice( @$arr, $start_idx, $length, @new ); |
233 | } |
f6066bac |
234 | |
910a0a6d |
235 | sub _return_rdg { |
236 | my( $rdg ) = @_; |
237 | # If we were passed a reading name, return the name. If we were |
238 | # passed a reading object, return the object. |
239 | my $wantobj = ref( $rdg ) eq 'Text::Tradition::Collation::Reading'; |
240 | my $real = $rdg; |
241 | if( exists $substitutions->{ $wantobj ? $rdg->name : $rdg } ) { |
242 | $real = $substitutions->{ $wantobj ? $rdg->name : $rdg }; |
243 | $real = $real->name unless $wantobj; |
244 | } |
245 | return $real; |
f6066bac |
246 | } |
247 | |
3b853983 |
248 | =begin testing |
249 | |
250 | ## TODO test specific sorts of nodes of the parallel-seg XML. |
251 | |
252 | =end testing |
253 | |
910a0a6d |
254 | ## Recursive helper function to help us navigate through nested XML, |
255 | ## picking out the text. $tradition is the tradition, needed for |
256 | ## making readings; $xn is the XML node currently being looked at, |
257 | ## $in_var is a flag to say that we are inside a variant, $ac is a |
258 | ## flag to say that we are inside an ante-correctionem reading, and |
259 | ## @cur_wits is the list of witnesses to which this XML node applies. |
260 | ## Returns the list of readings, if any, created on the run. |
261 | |
262 | { |
3bc0cd18 |
263 | my %active_wits; |
910a0a6d |
264 | my $current_app; |
eca16057 |
265 | my $seen_apps; |
910a0a6d |
266 | |
267 | sub _get_readings { |
268 | my( $tradition, $xn, $in_var, $ac, @cur_wits ) = @_; |
3bc0cd18 |
269 | @cur_wits = grep { $active_wits{$_} } keys %active_wits unless $in_var; |
910a0a6d |
270 | |
271 | my @new_readings; |
272 | if( $xn->nodeType == XML_TEXT_NODE ) { |
273 | # Some words, thus make some readings. |
274 | my $str = $xn->data; |
275 | return unless $str =~ /\S/; # skip whitespace-only text nodes |
276 | #print STDERR "Handling text node " . $str . "\n"; |
277 | # Check that all the witnesses we have are active. |
278 | foreach my $c ( @cur_wits ) { |
3bc0cd18 |
279 | warn "$c is not among active wits" unless $active_wits{$c}; |
910a0a6d |
280 | } |
281 | $str =~ s/^\s+//; |
282 | my $final = $str =~ s/\s+$//; |
283 | foreach my $w ( split( /\s+/, $str ) ) { |
284 | # For now, skip punctuation. |
285 | next if $w !~ /[[:alnum:]]/; |
3b853983 |
286 | my $rdg = _make_reading( $tradition->collation, $w ); |
910a0a6d |
287 | push( @new_readings, $rdg ); |
288 | unless( $in_var ) { |
910a0a6d |
289 | $rdg->make_common; |
290 | } |
291 | foreach ( @cur_wits ) { |
292 | warn "Empty wit!" unless $_; |
293 | warn "Empty reading!" unless $rdg; |
294 | push( @{$text->{$_}}, $rdg ) unless $ac; |
295 | } |
296 | } |
297 | } elsif( $xn->nodeName eq 'w' ) { |
298 | # Everything in this tag is one word. Also save any original XML ID. |
299 | #print STDERR "Handling word " . $xn->toString . "\n"; |
300 | # Check that all the witnesses we have are active. |
301 | foreach my $c ( @cur_wits ) { |
3bc0cd18 |
302 | warn "$c is not among active wits" unless $active_wits{$c}; |
910a0a6d |
303 | } |
304 | my $xml_id = $xn->getAttribute( 'xml:id' ); |
3b853983 |
305 | my $rdg = _make_reading( $tradition->collation, $xn->textContent, $xml_id ); |
910a0a6d |
306 | push( @new_readings, $rdg ); |
307 | unless( $in_var ) { |
910a0a6d |
308 | $rdg->make_common; |
309 | } |
310 | foreach( @cur_wits ) { |
311 | warn "Empty wit!" unless $_; |
312 | warn "Empty reading!" unless $rdg; |
313 | push( @{$text->{$_}}, $rdg ) unless $ac; |
314 | } |
315 | } elsif ( $xn->nodeName eq 'app' ) { |
eca16057 |
316 | $seen_apps++; |
910a0a6d |
317 | $current_app = $xn->getAttribute( 'xml:id' ); |
318 | # print STDERR "Handling app $current_app\n"; |
319 | # Keep the reading sets in this app. |
320 | my @sets; |
321 | # Recurse through all children (i.e. rdgs) for sets of words. |
322 | foreach ( $xn->childNodes ) { |
323 | my @rdg_set = _get_readings( $tradition, $_, $in_var, $ac, @cur_wits ); |
324 | push( @sets, \@rdg_set ) if @rdg_set; |
325 | } |
326 | # Now collate these sets if we have more than one. |
327 | my $subs = collate_variants( $tradition->collation, @sets ) if @sets > 1; |
328 | map { $substitutions->{$_} = $subs->{$_} } keys %$subs; |
329 | # TODO Look through substitutions to see if we can make anything common now. |
330 | # Return the entire set of unique readings. |
331 | my %unique; |
332 | foreach my $s ( @sets ) { |
333 | map { $unique{$_->name} = $_ } @$s; |
334 | } |
335 | push( @new_readings, values( %unique ) ); |
336 | # Exit the current app. |
337 | $current_app = ''; |
338 | } elsif ( $xn->nodeName eq 'lem' || $xn->nodeName eq 'rdg' ) { |
339 | # Alter the current witnesses and recurse. |
340 | #print STDERR "Handling reading for " . $xn->getAttribute( 'wit' ) . "\n"; |
3bc0cd18 |
341 | # TODO handle p.c. and s.l. designations too |
910a0a6d |
342 | $ac = $xn->getAttribute( 'type' ) && $xn->getAttribute( 'type' ) eq 'a.c.'; |
3b853983 |
343 | my @rdg_wits = _get_sigla( $xn ); |
910a0a6d |
344 | @rdg_wits = ( 'base' ) unless @rdg_wits; # Allow for editorially-supplied readings |
345 | my @words; |
346 | foreach ( $xn->childNodes ) { |
347 | my @rdg_set = _get_readings( $tradition, $_, 1, $ac, @rdg_wits ); |
348 | push( @words, @rdg_set ) if @rdg_set; |
349 | } |
350 | # If we have more than one word in a reading, it should become a segment. |
351 | # $tradition->collation->add_segment( @words ) if @words > 1; |
352 | |
353 | if( $ac ) { |
354 | # Add the reading set to the a.c. readings. |
355 | foreach ( @rdg_wits ) { |
356 | $app_ac->{$_}->{$current_app} = \@words; |
357 | } |
358 | } else { |
359 | # Add the reading set to the app anchors for each witness |
360 | # or put in placeholders for empty p.c. readings |
361 | foreach ( @rdg_wits ) { |
362 | my $start = @words ? $words[0]->name : "PH-$current_app"; |
363 | my $end = @words ? $words[-1]->name : "PH-$current_app"; |
364 | $app_anchors->{$current_app}->{$_}->{'start'} = $start; |
365 | $app_anchors->{$current_app}->{$_}->{'end'} = $end; |
366 | push( @{$text->{$_}}, $start ) unless @words; |
367 | } |
368 | } |
369 | push( @new_readings, @words ); |
370 | } elsif( $xn->nodeName eq 'witStart' ) { |
371 | # Add the relevant wit(s) to the active list. |
372 | #print STDERR "Handling witStart\n"; |
3bc0cd18 |
373 | map { $active_wits{$_} = 1 } @cur_wits; |
374 | # Record a lacuna in all non-active witnesses if this is |
375 | # the first app. Get the full list from $text. |
376 | if( $seen_apps == 1 ) { |
377 | my $i = 0; |
378 | foreach my $sig ( keys %$text ) { |
379 | next if $active_wits{$sig}; |
380 | my $l = $tradition->collation->add_lacuna( $current_app . "_$i" ); |
381 | $i++; |
382 | push( @{$text->{$sig}}, $l ); |
383 | } |
384 | } |
910a0a6d |
385 | } elsif( $xn->nodeName eq 'witEnd' ) { |
386 | # Take the relevant wit(s) out of the list. |
387 | #print STDERR "Handling witEnd\n"; |
3bc0cd18 |
388 | map { $active_wits{$_} = undef } @cur_wits; |
eca16057 |
389 | # Record a lacuna, unless this is the last app. |
390 | unless( $seen_apps == $app_count ) { |
391 | foreach my $i ( 0 .. $#cur_wits ) { |
392 | my $w = $cur_wits[$i]; |
393 | my $l = $tradition->collation->add_lacuna( $current_app . "_$i" ); |
394 | push( @{$text->{$w}}, $l ); |
395 | } |
396 | } |
910a0a6d |
397 | } elsif( $xn->nodeName eq 'witDetail' ) { |
398 | # Ignore these for now. |
399 | return; |
400 | } else { |
401 | # Recurse as if this tag weren't there. |
402 | #print STDERR "Recursing on tag " . $xn->nodeName . "\n"; |
403 | foreach( $xn->childNodes ) { |
404 | push( @new_readings, _get_readings( $tradition, $_, $in_var, $ac, @cur_wits ) ); |
405 | } |
406 | } |
407 | return @new_readings; |
408 | } |
409 | |
410 | } |
411 | |
3b853983 |
412 | =begin testing |
413 | |
414 | use XML::LibXML; |
415 | use XML::LibXML::XPathContext; |
416 | use Text::Tradition::Parser::TEI; |
417 | |
418 | my $xml_str = '<tei><rdg wit="#A #B #C #D">some text</rdg></tei>'; |
419 | my $el = XML::LibXML->new()->parse_string( $xml_str )->documentElement; |
420 | my $xpc = XML::LibXML::XPathContext->new( $el ); |
421 | my $obj = $xpc->find( '//rdg' ); |
422 | |
423 | my @wits = Text::Tradition::Parser::TEI::_get_sigla( $obj ); |
424 | is( join( ' ', @wits) , "A B C D", "correctly parsed reading wit string" ); |
425 | |
426 | =end testing |
427 | |
428 | =cut |
429 | |
910a0a6d |
430 | # Helper to extract a list of witness sigla from a reading element. |
3b853983 |
431 | sub _get_sigla { |
f6066bac |
432 | my( $rdg ) = @_; |
433 | # Cope if we have been handed a NodeList. There is only |
434 | # one reading here. |
435 | if( ref( $rdg ) eq 'XML::LibXML::NodeList' ) { |
910a0a6d |
436 | $rdg = $rdg->shift; |
f6066bac |
437 | } |
438 | |
439 | my @wits; |
440 | if( ref( $rdg ) eq 'XML::LibXML::Element' ) { |
910a0a6d |
441 | my $witstr = $rdg->getAttribute( 'wit' ); |
442 | $witstr =~ s/^\s+//; |
443 | $witstr =~ s/\s+$//; |
444 | @wits = split( /\s+/, $witstr ); |
445 | map { $_ =~ s/^\#// } @wits; |
f6066bac |
446 | } |
447 | return @wits; |
448 | } |
449 | |
910a0a6d |
450 | # Helper with its counters to actually make the readings. |
f2b9605f |
451 | { |
452 | my $word_ctr = 0; |
453 | my %used_nodeids; |
454 | |
3b853983 |
455 | sub _save_preexisting_nodeids { |
910a0a6d |
456 | foreach( @_ ) { |
457 | $used_nodeids{$_->getValue()} = 1; |
458 | } |
459 | } |
460 | |
3b853983 |
461 | sub _make_reading { |
910a0a6d |
462 | my( $graph, $word, $xml_id ) = @_; |
463 | if( $xml_id ) { |
464 | if( exists $used_nodeids{$xml_id} ) { |
465 | if( $used_nodeids{$xml_id} != 1 ) { |
466 | warn "Already used assigned XML ID somewhere else!"; |
467 | $xml_id = undef; |
468 | } |
469 | } else { |
470 | warn "Undetected pre-existing XML ID"; |
471 | } |
472 | } |
473 | if( !$xml_id ) { |
474 | until( $xml_id ) { |
475 | my $try_id = 'w'.$word_ctr++; |
476 | next if exists $used_nodeids{$try_id}; |
477 | $xml_id = $try_id; |
478 | } |
479 | } |
480 | my $rdg = $graph->add_reading( $xml_id ); |
481 | $rdg->text( $word ); |
482 | $used_nodeids{$xml_id} = $rdg; |
483 | return $rdg; |
f2b9605f |
484 | } |
485 | } |
486 | |
f6066bac |
487 | 1; |
3b853983 |
488 | |
489 | =head1 BUGS / TODO |
490 | |
491 | =over |
492 | |
493 | =item * More unit testing |
494 | |
495 | =back |
496 | |
497 | =head1 LICENSE |
498 | |
499 | This package is free software and is provided "as is" without express |
500 | or implied warranty. You can redistribute it and/or modify it under |
501 | the same terms as Perl itself. |
502 | |
503 | =head1 AUTHOR |
504 | |
505 | Tara L Andrews E<lt>aurum@cpan.orgE<gt> |