Commit | Line | Data |
6f4946fb |
1 | package Text::Tradition::Parser::CTE; |
2 | |
3 | use strict; |
4 | use warnings; |
a188b944 |
5 | use feature 'say'; |
c9158e60 |
6 | use Encode qw/ decode /; |
00311328 |
7 | use Text::Tradition::Error; |
4d85a60e |
8 | use Text::Tradition::Parser::Util qw/ collate_variants /; |
6f4946fb |
9 | use XML::LibXML; |
10 | use XML::LibXML::XPathContext; |
11 | |
12 | =head1 NAME |
13 | |
14 | Text::Tradition::Parser::CTE |
15 | |
16 | =head1 DESCRIPTION |
17 | |
18 | Parser module for Text::Tradition, given a TEI file exported from |
19 | Classical Text Editor. |
20 | |
21 | =head1 METHODS |
22 | |
a445ce40 |
23 | =head2 parse |
6f4946fb |
24 | |
25 | my @apparatus = read( $xml_file ); |
26 | |
27 | Takes a Tradition object and a TEI file exported from Classical Text |
4d85a60e |
28 | Editor using double-endpoint-attachment critical apparatus encoding; |
29 | initializes the Tradition from the file. |
6f4946fb |
30 | |
31 | =cut |
32 | |
4d85a60e |
33 | my %sigil_for; # Save the XML IDs for witnesses. |
34 | my %apps; # Save the apparatus XML for a given ID. |
f6e19c7c |
35 | my %has_ac; # Keep track of witnesses that have corrections. |
6f4946fb |
36 | |
37 | sub parse { |
dfc37e38 |
38 | my( $tradition, $opts ) = @_; |
4d85a60e |
39 | my $c = $tradition->collation; # Some shorthand |
40 | |
41 | # First, parse the XML. |
c9158e60 |
42 | my( $tei, $xpc ) = _remove_formatting( $opts ); |
43 | return unless $tei; # we have already warned. |
4d85a60e |
44 | |
45 | # CTE uses a DTD rather than any xmlns-based parsing. Thus we |
46 | # need no namespace handling. |
4d85a60e |
47 | # Get the witnesses and create the witness objects. |
48 | foreach my $wit_el ( $xpc->findnodes( '//sourceDesc/listWit/witness' ) ) { |
49 | # The witness xml:id is used internally, and is *not* the sigil name. |
50 | my $id= $wit_el->getAttribute( 'xml:id' ); |
c9158e60 |
51 | my @sig_parts = $xpc->findnodes( 'descendant::text()', $wit_el ); |
4d85a60e |
52 | my $sig = _stringify_sigil( @sig_parts ); |
a188b944 |
53 | say STDERR "Adding witness $sig"; |
82fa4d57 |
54 | $tradition->add_witness( sigil => $sig, sourcetype => 'collation' ); |
b8f262e8 |
55 | $sigil_for{'#'.$id} = $sig; # Make life easy by keying on the ID ref syntax |
4d85a60e |
56 | } |
c9158e60 |
57 | |
4d85a60e |
58 | # Now go through the text and find the base tokens, apparatus tags, and |
59 | # anchors. Make a giant array of all of these things in sequence. |
60 | # TODO consider combining this with creation of graph below |
61 | my @base_text; |
62 | foreach my $pg_el ( $xpc->findnodes( '/TEI/text/body/p' ) ) { |
63 | foreach my $xn ( $pg_el->childNodes ) { |
a188b944 |
64 | my @items = _get_base( $xn ); |
65 | foreach my $i ( @items ) { |
66 | $DB::single = 1 if $i->{'type'} eq 'anchor' && !$i->{'content'}; |
67 | } |
68 | push( @base_text, @items ); |
4d85a60e |
69 | } |
6f4946fb |
70 | } |
4d85a60e |
71 | # We now have to work through this array applying the alternate |
72 | # apparatus readings to the base text. Essentially we will put |
73 | # everything on the graph, from which we will delete the apps and |
74 | # anchors when we are done. |
f6e19c7c |
75 | |
76 | # First, put the base tokens, apps, and anchors in the graph. |
4d85a60e |
77 | my $counter = 0; |
78 | my $last = $c->start; |
79 | foreach my $item ( @base_text ) { |
80 | my $r; |
81 | if( $item->{'type'} eq 'token' ) { |
12720144 |
82 | $r = $c->add_reading( { id => 'n'.$counter++, |
83 | text => $item->{'content'} } ); |
4d85a60e |
84 | } elsif ( $item->{'type'} eq 'anchor' ) { |
10e4b1ac |
85 | $r = $c->add_reading( { id => '__ANCHOR_' . $item->{'content'} . '__', |
12720144 |
86 | is_ph => 1 } ); |
4d85a60e |
87 | } elsif ( $item->{'type'} eq 'app' ) { |
10e4b1ac |
88 | my $tag = '__APP_' . $counter++ . '__'; |
12720144 |
89 | $r = $c->add_reading( { id => $tag, is_ph => 1 } ); |
4d85a60e |
90 | $apps{$tag} = $item->{'content'}; |
91 | } |
f6e19c7c |
92 | $c->add_path( $last, $r, $c->baselabel ); |
4d85a60e |
93 | $last = $r; |
6f4946fb |
94 | } |
f6e19c7c |
95 | $c->add_path( $last, $c->end, $c->baselabel ); |
4d85a60e |
96 | |
97 | # Now we can parse the apparatus entries, and add the variant readings |
98 | # to the graph. |
99 | |
100 | foreach my $app_id ( keys %apps ) { |
101 | _add_readings( $c, $app_id ); |
6f4946fb |
102 | } |
4d85a60e |
103 | |
f6e19c7c |
104 | # Finally, add explicit witness paths, remove the base paths, and remove |
105 | # the app/anchor tags. |
a445ce40 |
106 | _expand_all_paths( $c ); |
861c3e27 |
107 | |
108 | # Save the text for each witness so that we can ensure consistency |
109 | # later on |
a188b944 |
110 | unless( $opts->{'nocalc'} ) { |
111 | $tradition->collation->text_from_paths(); |
112 | $tradition->collation->calculate_ranks(); |
113 | $tradition->collation->flatten_ranks(); |
114 | } |
6f4946fb |
115 | } |
116 | |
4d85a60e |
117 | sub _stringify_sigil { |
118 | my( @nodes ) = @_; |
119 | my @parts = grep { /\w/ } map { $_->data } @nodes; |
222d58f1 |
120 | my $whole = join( '', @parts ); |
121 | $whole =~ s/\W//g; |
122 | return $whole; |
4d85a60e |
123 | } |
6f4946fb |
124 | |
c9158e60 |
125 | # Get rid of all the formatting elements that get in the way of tokenization. |
126 | sub _remove_formatting { |
127 | my( $opts ) = @_; |
128 | |
129 | # First, parse the original XML |
130 | my $parser = XML::LibXML->new(); |
131 | my $doc; |
132 | if( exists $opts->{'string'} ) { |
133 | $doc = $parser->parse_string( $opts->{'string'} ); |
134 | } elsif ( exists $opts->{'file'} ) { |
135 | $doc = $parser->parse_file( $opts->{'file'} ); |
136 | } else { |
137 | warn "Could not find string or file option to parse"; |
138 | return; |
139 | } |
00311328 |
140 | |
c9158e60 |
141 | # Second, remove the formatting |
142 | my $xpc = XML::LibXML::XPathContext->new( $doc->documentElement ); |
143 | my @useless = $xpc->findnodes( '//hi' ); |
144 | foreach my $n ( @useless ) { |
145 | my $parent = $n->parentNode(); |
146 | my @children = $n->childNodes(); |
147 | my $first = shift @children; |
7c2ed85e |
148 | if( $first ) { |
149 | $parent->replaceChild( $first, $n ); |
150 | foreach my $c ( @children ) { |
151 | $parent->insertAfter( $c, $first ); |
152 | $first = $c; |
153 | } |
154 | } else { |
155 | $parent->removeChild( $n ); |
c9158e60 |
156 | } |
157 | } |
158 | |
159 | # Third, write out and reparse to merge the text nodes. |
00311328 |
160 | my $enc = $doc->encoding || 'UTF-8'; |
161 | my $result = decode( $enc, $doc->toString() ); |
c9158e60 |
162 | my $tei = $parser->parse_string( $result )->documentElement; |
00311328 |
163 | unless( $tei->nodeName =~ /^tei(corpus)?$/i ) { |
164 | throw( "Parsed document has non-TEI root element " . $tei->nodeName ); |
165 | } |
c9158e60 |
166 | $xpc = XML::LibXML::XPathContext->new( $tei ); |
167 | return( $tei, $xpc ); |
168 | } |
169 | |
170 | ## Helper function to help us navigate through nested XML, picking out |
171 | ## the words, the apparatus, and the anchors. |
4d85a60e |
172 | |
173 | sub _get_base { |
174 | my( $xn ) = @_; |
175 | my @readings; |
176 | if( $xn->nodeType == XML_TEXT_NODE ) { |
177 | # Base text, just split the words on whitespace and add them |
178 | # to our sequence. |
179 | my $str = $xn->data; |
180 | $str =~ s/^\s+//; |
c9158e60 |
181 | my @tokens = split( /\s+/, $str ); |
182 | push( @readings, map { { 'type' => 'token', 'content' => $_ } } @tokens ); |
4d85a60e |
183 | } elsif( $xn->nodeName eq 'app' ) { |
184 | # Apparatus, just save the entire XML node. |
185 | push( @readings, { 'type' => 'app', 'content' => $xn } ); |
186 | } elsif( $xn->nodeName eq 'anchor' ) { |
187 | # Anchor to mark the end of some apparatus; save its ID. |
a188b944 |
188 | unless( $xn->getAttribute('type') ) { |
189 | push( @readings, { 'type' => 'anchor', |
190 | 'content' => $xn->getAttribute( 'xml:id' ) } ); |
191 | } |
192 | } elsif ( $xn->nodeName !~ /^(note|seg|milestone|emph)$/ ) { # Any tag we don't know to disregard |
193 | say STDERR "Unrecognized tag " . $xn->nodeName; |
6f4946fb |
194 | } |
4d85a60e |
195 | return @readings; |
6f4946fb |
196 | } |
197 | |
c9158e60 |
198 | sub _append_tokens { |
199 | my( $list, @tokens ) = @_; |
200 | if( @$list && $list->[-1]->{'content'} =~ /\#JOIN\#$/ ) { |
201 | # The list evidently ended mid-word; join the next token onto it. |
202 | my $t = shift @tokens; |
203 | if( ref $t && $t->{'type'} eq 'token' ) { |
204 | # Join the word |
205 | $t = $t->{'content'}; |
206 | } elsif( ref $t ) { |
207 | # An app or anchor intervened; end the word. |
208 | unshift( @tokens, $t ); |
209 | $t = ''; |
210 | } |
211 | $list->[-1]->{'content'} =~ s/\#JOIN\#$/$t/; |
212 | } |
213 | foreach my $t ( @tokens ) { |
214 | unless( ref( $t ) ) { |
215 | $t = { 'type' => 'token', 'content' => $t }; |
216 | } |
217 | push( @$list, $t ); |
218 | } |
219 | } |
220 | |
4d85a60e |
221 | sub _add_readings { |
222 | my( $c, $app_id ) = @_; |
223 | my $xn = $apps{$app_id}; |
224 | my $anchor = _anchor_name( $xn->getAttribute( 'to' ) ); |
225 | # Get the lemma, which is all the readings between app and anchor, |
226 | # excluding other apps or anchors. |
227 | my @lemma = _return_lemma( $c, $app_id, $anchor ); |
10e4b1ac |
228 | my $lemma_str = join( ' ', grep { $_ !~ /^__/ } map { $_->text } @lemma ); |
4d85a60e |
229 | |
230 | # For each reading, send its text to 'interpret' along with the lemma, |
231 | # and then save the list of witnesses that these tokens belong to. |
3a5d151b |
232 | my %wit_rdgs; # Maps from witnesses to the variant text |
4d85a60e |
233 | my $ctr = 0; |
234 | my $tag = $app_id; |
10e4b1ac |
235 | $tag =~ s/^\__APP_(.*)\__$/$1/; |
c9158e60 |
236 | |
4d85a60e |
237 | foreach my $rdg ( $xn->getChildrenByTagName( 'rdg' ) ) { |
238 | my @text; |
4d85a60e |
239 | foreach ( $rdg->childNodes ) { |
240 | push( @text, _get_base( $_ ) ); |
241 | } |
12720144 |
242 | my( $interpreted, $flag ) = ( '', undef ); |
243 | if( @text ) { |
244 | ( $interpreted, $flag ) = interpret( |
245 | join( ' ', map { $_->{'content'} } @text ), $lemma_str ); |
246 | } |
b8f262e8 |
247 | next if( $interpreted eq $lemma_str ) && !$flag; # Reading is lemma. |
248 | |
4d85a60e |
249 | my @rdg_nodes; |
b8f262e8 |
250 | if( $interpreted eq '#LACUNA#' ) { |
10e4b1ac |
251 | push( @rdg_nodes, $c->add_reading( { id => 'r'.$tag.".".$ctr++, |
b8f262e8 |
252 | is_lacuna => 1 } ) ); |
253 | } else { |
254 | foreach my $w ( split( /\s+/, $interpreted ) ) { |
10e4b1ac |
255 | my $r = $c->add_reading( { id => 'r'.$tag.".".$ctr++, |
b8f262e8 |
256 | text => $w } ); |
257 | push( @rdg_nodes, $r ); |
258 | } |
4d85a60e |
259 | } |
f6e19c7c |
260 | # For each listed wit, save the reading. |
261 | foreach my $wit ( split( /\s+/, $rdg->getAttribute( 'wit' ) ) ) { |
12720144 |
262 | $wit .= $flag if $flag; |
f6e19c7c |
263 | $wit_rdgs{$wit} = \@rdg_nodes; |
264 | } |
12720144 |
265 | |
3a5d151b |
266 | # Does the reading have an ID? If so it probably has a witDetail |
f6e19c7c |
267 | # attached, and we need to read it. |
3a5d151b |
268 | if( $rdg->hasAttribute( 'xml:id' ) ) { |
12720144 |
269 | warn "Witdetail on meta reading" if $flag; # this could get complicated. |
f6e19c7c |
270 | my $rid = $rdg->getAttribute( 'xml:id' ); |
271 | my $xpc = XML::LibXML::XPathContext->new( $xn ); |
272 | my @details = $xpc->findnodes( './witDetail[@target="'.$rid.'"]' ); |
273 | foreach my $d ( @details ) { |
274 | _parse_wit_detail( $d, \%wit_rdgs, \@lemma ); |
275 | } |
3a5d151b |
276 | } |
f6e19c7c |
277 | } |
278 | |
4d85a60e |
279 | # Now collate the variant readings, since it is not done for us. |
12720144 |
280 | collate_variants( $c, \@lemma, values %wit_rdgs ); |
b8f262e8 |
281 | |
4d85a60e |
282 | # Now add the witness paths for each reading. |
a445ce40 |
283 | my $aclabel = $c->ac_label; |
f6e19c7c |
284 | foreach my $wit_id ( keys %wit_rdgs ) { |
a445ce40 |
285 | my $witstr = _get_sigil( $wit_id, $aclabel ); |
f6e19c7c |
286 | my $rdg_list = $wit_rdgs{$wit_id}; |
287 | _add_wit_path( $c, $rdg_list, $app_id, $anchor, $witstr ); |
6f4946fb |
288 | } |
4d85a60e |
289 | } |
6f4946fb |
290 | |
4d85a60e |
291 | sub _anchor_name { |
292 | my $xmlid = shift; |
293 | $xmlid =~ s/^\#//; |
10e4b1ac |
294 | return sprintf( "__ANCHOR_%s__", $xmlid ); |
6f4946fb |
295 | } |
296 | |
4d85a60e |
297 | sub _return_lemma { |
298 | my( $c, $app, $anchor ) = @_; |
10e4b1ac |
299 | my @nodes = grep { $_->id !~ /^__A(PP|NCHOR)/ } |
12720144 |
300 | $c->reading_sequence( $c->reading( $app ), $c->reading( $anchor ), |
301 | $c->baselabel ); |
4d85a60e |
302 | return @nodes; |
303 | } |
6f4946fb |
304 | |
a445ce40 |
305 | =head2 interpret( $reading, $lemma ) |
306 | |
307 | Given a string in $reading and a corresponding lemma in $lemma, interpret what |
308 | the actual reading should be. Used to deal with apparatus-ese shorthands for |
309 | marking transpositions, prefixed or suffixed words, and the like. |
310 | |
311 | =cut |
312 | |
6f4946fb |
313 | sub interpret { |
4d85a60e |
314 | # A utility function to change apparatus-ese into a full variant. |
315 | my( $reading, $lemma ) = @_; |
316 | return $reading if $reading eq $lemma; |
317 | my $oldreading = $reading; |
318 | # $lemma =~ s/\s+[[:punct:]]+$//; |
12720144 |
319 | my $flag; # In case of p.c. indications |
4d85a60e |
320 | my @words = split( /\s+/, $lemma ); |
321 | if( $reading =~ /^(.*) praem.$/ ) { |
322 | $reading = "$1 $lemma"; |
323 | } elsif( $reading =~ /^(.*) add.$/ ) { |
324 | $reading = "$lemma $1"; |
b8f262e8 |
325 | } elsif( $reading =~ /add. alia manu/ |
326 | || $reading =~ /inscriptionem compegi e/ # TODO huh? |
327 | || $reading eq 'inc.' # TODO huh? |
328 | ) { |
12720144 |
329 | # Ignore it. |
330 | $reading = $lemma; |
b8f262e8 |
331 | } elsif( $reading =~ /locus [uv]acuus/ |
332 | || $reading eq 'def.' |
c9158e60 |
333 | || $reading eq 'illeg.' |
334 | || $reading eq 'onleesbar' |
f6e19c7c |
335 | ) { |
b8f262e8 |
336 | $reading = '#LACUNA#'; |
337 | } elsif( $reading eq 'om.' ) { |
4d85a60e |
338 | $reading = ''; |
c9158e60 |
339 | } elsif( $reading =~ /^in[uv]\.$/ |
340 | || $reading eq 'transp.' ) { |
4d85a60e |
341 | # Hope it is two words. |
a188b944 |
342 | say STDERR "WARNING: want to invert a lemma that is not two words" |
4d85a60e |
343 | unless scalar( @words ) == 2; |
344 | $reading = join( ' ', reverse( @words ) ); |
12720144 |
345 | } elsif( $reading =~ /^iter(\.|at)$/ ) { |
4d85a60e |
346 | # Repeat the lemma |
347 | $reading = "$lemma $lemma"; |
12720144 |
348 | } elsif( $reading eq 'in marg.' ) { |
349 | # There was nothing before a correction. |
350 | $reading = ''; |
351 | $flag = '_ac'; |
c9158e60 |
352 | } elsif( $reading =~ /^(.*?)\s*\(?sic([\s\w.]+)?\)?$/ ) { |
353 | # Discard any 'sic' notation; indeed, indeed. |
354 | $reading = $1; |
12720144 |
355 | } elsif( $reading =~ /^(.*) \.\.\. (.*)$/ ) { |
4d85a60e |
356 | # The first and last N words captured should replace the first and |
357 | # last N words of the lemma. |
358 | my @begin = split( /\s+/, $1 ); |
359 | my @end = split( /\s+/, $2 ); |
360 | if( scalar( @begin ) + scalar ( @end ) > scalar( @words ) ) { |
361 | # Something is wrong and we can't do the splice. |
a188b944 |
362 | say STDERR "ERROR: $lemma is too short to accommodate $oldreading"; |
4d85a60e |
363 | } else { |
364 | splice( @words, 0, scalar @begin, @begin ); |
365 | splice( @words, -(scalar @end), scalar @end, @end ); |
366 | $reading = join( ' ', @words ); |
367 | } |
6f4946fb |
368 | } |
12720144 |
369 | if( $oldreading ne $reading || $flag || $oldreading =~ /\./ ) { |
370 | my $int = $reading; |
371 | $int .= " ($flag)" if $flag; |
a188b944 |
372 | say STDERR "Interpreted $oldreading as $int given $lemma"; |
12720144 |
373 | } |
374 | return( $reading, $flag ); |
4d85a60e |
375 | } |
376 | |
f6e19c7c |
377 | sub _parse_wit_detail { |
378 | my( $detail, $readings, $lemma ) = @_; |
379 | my $wit = $detail->getAttribute( 'wit' ); |
380 | my $content = $detail->textContent; |
4ed2b212 |
381 | if( $content =~ /a\.\s*c\b/ ) { |
f6e19c7c |
382 | # Replace the key in the $readings hash |
383 | my $rdg = delete $readings->{$wit}; |
384 | $readings->{$wit.'_ac'} = $rdg; |
385 | $has_ac{$sigil_for{$wit}} = 1; |
4ed2b212 |
386 | } elsif( $content =~ /p\.\s*c\b/ ) { |
f6e19c7c |
387 | # If no key for the wit a.c. exists, add one pointing to the lemma |
388 | unless( exists $readings->{$wit.'_ac'} ) { |
389 | $readings->{$wit.'_ac'} = $lemma; |
390 | } |
391 | $has_ac{$sigil_for{$wit}} = 1; |
392 | } # else don't bother just yet |
393 | } |
394 | |
a445ce40 |
395 | sub _get_sigil { |
396 | my( $xml_id, $layerlabel ) = @_; |
f6e19c7c |
397 | if( $xml_id =~ /^(.*)_ac$/ ) { |
398 | my $real_id = $1; |
a445ce40 |
399 | return $sigil_for{$real_id} . $layerlabel; |
f6e19c7c |
400 | } else { |
401 | return $sigil_for{$xml_id}; |
402 | } |
403 | } |
404 | |
a445ce40 |
405 | sub _expand_all_paths { |
f6e19c7c |
406 | my( $c ) = @_; |
407 | |
408 | # Walk the collation and fish out the paths for each witness |
409 | foreach my $wit ( $c->tradition->witnesses ) { |
410 | my $sig = $wit->sigil; |
12720144 |
411 | my @path = grep { !$_->is_ph } |
f6e19c7c |
412 | $c->reading_sequence( $c->start, $c->end, $sig ); |
413 | $wit->path( \@path ); |
414 | if( $has_ac{$sig} ) { |
12720144 |
415 | my @ac_path = grep { !$_->is_ph } |
861c3e27 |
416 | $c->reading_sequence( $c->start, $c->end, $sig.$c->ac_label ); |
f6e19c7c |
417 | $wit->uncorrected_path( \@ac_path ); |
418 | } |
419 | } |
420 | |
421 | # Delete the anchors |
12720144 |
422 | foreach my $anchor ( grep { $_->is_ph } $c->readings ) { |
f6e19c7c |
423 | $c->del_reading( $anchor ); |
424 | } |
12720144 |
425 | # Delete the base edges |
426 | map { $c->del_path( $_, $c->baselabel ) } $c->paths; |
f6e19c7c |
427 | |
428 | # Make the path edges |
429 | $c->make_witness_paths(); |
7c2ed85e |
430 | |
431 | # Now remove any orphan nodes, and warn that we are doing so. |
432 | foreach my $v ( $c->sequence->isolated_vertices ) { |
433 | my $r = $c->reading( $v ); |
a188b944 |
434 | say STDERR "Deleting orphan reading $r / " . $r->text; |
7c2ed85e |
435 | $c->del_reading( $r ); |
436 | } |
4d85a60e |
437 | } |
438 | |
439 | sub _add_wit_path { |
f6e19c7c |
440 | my( $c, $rdg, $app, $anchor, $wit ) = @_; |
4d85a60e |
441 | my @nodes = @$rdg; |
f6e19c7c |
442 | push( @nodes, $c->reading( $anchor ) ); |
4d85a60e |
443 | |
f6e19c7c |
444 | my $cur = $c->reading( $app ); |
4d85a60e |
445 | foreach my $n ( @nodes ) { |
f6e19c7c |
446 | $c->add_path( $cur, $n, $wit ); |
4d85a60e |
447 | $cur = $n; |
6f4946fb |
448 | } |
6f4946fb |
449 | } |
450 | |
00311328 |
451 | sub throw { |
452 | Text::Tradition::Error->throw( |
453 | 'ident' => 'Parser::CTE error', |
454 | 'message' => $_[0], |
455 | ); |
456 | } |
457 | |
6f4946fb |
458 | =head1 LICENSE |
459 | |
460 | This package is free software and is provided "as is" without express |
461 | or implied warranty. You can redistribute it and/or modify it under |
462 | the same terms as Perl itself. |
463 | |
464 | =head1 AUTHOR |
465 | |
466 | Tara L Andrews, aurum@cpan.org |
467 | |
468 | =cut |
469 | |
470 | 1; |
471 | |