1 package Text::Tradition::Language::French;
5 use Encode qw/ encode_utf8 decode_utf8 /;
6 use IPC::Run qw/ run /;
7 use Lingua::TagSet::Multext;
8 use Lingua::TagSet::TreeTagger;
10 use Text::Tradition::Collation::Reading::Lexeme;
11 use Text::Tradition::Collation::Reading::WordForm;
14 my $MORPHDIR = '/Users/tla/Projects/morphology';
18 Text::Tradition::Language::French - language-specific module for French
22 Implements morphology lookup for French words in context. This module
23 depends on the Flemm module for French lemmatization
24 (L<http://www.univ-nancy2.fr/pers/namer/Outils.htm#fl3> in conjunction with
25 the TreeTagger software
26 (L<http://www.ims.uni-stuttgart.de/projekte/corplex/TreeTagger/>), which is
27 (for now) expected to be installed in $MORPHDIR/TreeTagger.
31 =head2 lemmatize( $text )
33 Evaluates the string using the Flemm package, and returns the results.
37 binmode STDOUT, ':utf8';
39 use_ok( 'Text::Tradition::Language::French' );
45 skip "Package Flemm not found" if $err;
46 my $tf = Text::Tradition->new(
48 'file' => 't/data/besoin.xml',
49 'language' => 'French' );
51 is( $tf->language, 'French', "Set language okay" );
53 # Test the lemmatization. How many readings now have morphological info?
54 # Do the lexemes match the reading?
56 foreach my $r ( $tf->collation->readings ) {
58 ok( $r->has_lexemes, "Reading $r has one or more lexemes" );
59 my @lex = $r->lexemes;
60 my $lexstr = join( '', map { $_->string } @lex );
61 my $textstr = $r->text;
63 is( $textstr, $lexstr, "Lexemes for reading $r match the reading" );
64 foreach my $l ( @lex ) {
65 next if $l->is_disambiguated;
66 # printf( "Ambiguous lexeme %s for reading %s:\n\t%s\n", $l->string, $r->id,
67 # join( "\n\t", map { $_->lemma . ': ' . $_->morphology->to_string } $l->matching_forms ) );
71 is( $ambig, 102, "Found 102 ambiguous forms as expected" );
73 # Try setting the normal form of a reading and re-analyzing
74 my $mr = $tf->collation->reading('99,2');
75 is( $mr->text, 'minspire', "Picked correct test reading" );
76 is( $mr->language, 'French', "Reading has correct language setting" );
77 $mr->normal_form( "m'inspire" );
79 is( $mr->lexemes, 2, "Got two lexemes for new m'inspire reading" );
87 my $tradition = shift;
89 # Given a tradition, lemmatize it witness by witness and see what we get.
90 my $workdir = File::Temp->newdir();
91 my $c = $tradition->collation;
92 # First, clear out all existing lexemes from the readings. Save the
93 # path as long as we went to the trouble of generating it.
95 foreach my $wit ( $tradition->witnesses ) {
96 my @sigla = ( $wit->sigil );
97 push( @sigla, $wit->sigil . $c->ac_label ) if $wit->is_layered;
98 foreach my $sig ( @sigla ) {
99 my @path = grep { !$_->is_meta }
100 $c->reading_sequence( $c->start, $c->end, $sig );
101 map { $_->clear_lexemes } @path;
102 $witness_paths{$sig} = \@path;
106 foreach my $sig ( keys %witness_paths ) {
107 # Get the text as a sequence of readings and as a string
108 print STDERR "Morphologizing witness $sig\n";
109 _lemmatize_sequence( undef, @{$witness_paths{$sig}} );
113 sub _update_reading_lexemes {
114 my( $replace, $reading, @lexemes ) = @_;
115 if( $reading->has_lexemes && !$replace ) {
116 # We need to merge what is in @lexemes with what we have already.
117 my @oldlex = $reading->lexemes;
118 my $cmp1 = join( '||', map { $_->string } @oldlex );
119 my $cmp2 = join( '||', map { $_->string } @lexemes );
120 if ( @oldlex == @lexemes && $cmp1 eq $cmp2 ) {
121 # The lexeme strings are the same, so merge the possible
122 # word forms from new to old.
123 foreach my $i ( 0 .. $#lexemes ) {
124 my $ol = $oldlex[$i];
125 my $nl = $lexemes[$i];
127 map { $ofw{$_->to_string} = 1 } $ol->matching_forms;
128 foreach my $form ( $nl->matching_forms ) {
129 unless( $ofw{$form->to_string} ) {
130 print STDERR "Adding form " . $form->to_string .
131 " to lexeme " . $nl->string . " at $reading\n";
132 $ol->add_matching_form( $form );
133 $ol->is_disambiguated(0);
138 warn "Lexeme layout for $reading changed; replacing the lot";
139 $reading->clear_lexemes;
140 $reading->add_lexeme( @lexemes );
143 $reading->clear_lexemes if $replace;
144 $reading->add_lexeme( @lexemes );
148 =head2 reading_lookup( $rdg[, $rdg, ...] )
150 Looks up one or more readings using the Flemm package, and returns the
151 possible results. This uses the same logic as L<lemmatize> above for the
152 entire tradition, but can also be used to (re-)analyze individual readings.
157 return _lemmatize_sequence( 1, @_ );
160 sub _lemmatize_sequence {
161 my( $replace, @path ) = @_;
162 my $tagresult = _treetag_string( _text_from_path( 1, @path ) );
164 # Map the tagged words onto the original readings, splitting
165 # them up into lexemes where necessary.
166 # NOTE we can have multiple lexemes in a reading, but not
167 # multiple readings to a lexeme.
168 my @tags = split( /\n/, $tagresult );
170 my $curr_rdg = shift @path;
173 foreach my $tag ( @tags ) {
174 # Get the original word
175 my( $lexeme, @rest ) = split( /\t/, $tag );
176 # Lemmatize the whole
177 my @forms = _parse_wordform( _flemm_lookup( $tag ) );
178 my $lexobj = Text::Tradition::Collation::Reading::Lexeme->new(
179 'string' => $lexeme, 'language' => 'French',
180 'wordform_matchlist' => \@forms );
181 # Find the next non-meta reading
182 while( $curr_rdg && $curr_rdg->is_meta ) {
183 $curr_rdg = shift @path;
185 unless( $curr_rdg ) {
186 warn "Ran out of readings in sequence at $lexeme";
189 my $curr_rdg_text = $curr_rdg->has_normal_form
190 ? $curr_rdg->normal_form : $curr_rdg->text;
191 if( $unused_rdg_part &&
192 $unused_rdg_part =~ /^\Q$lexeme\E(\s*)(.*)$/ ) {
193 # Nth part of curr_rdg
194 $unused_rdg_part = $2;
195 push( @curr_lexemes, $lexobj );
196 } elsif( $curr_rdg_text =~ /^\Q$lexeme\E(\s*)(.*)$/ ) {
197 # Flag an error if there is already an unused reading part.
198 warn "Skipped over unused text $unused_rdg_part at $curr_rdg"
200 $unused_rdg_part = $2; # will be empty if the whole reading matched
201 push( @curr_lexemes, $lexobj );
203 # We do not cope with the idea of a lexeme being
204 # spread across multiple readings.
205 warn "Word sequence changed unexpectedly in text";
206 # See if we can find a matching reading
209 while( my $nr = shift @path ) {
210 my $nrtext = $nr->has_normal_form ? $nr->normal_form : $nr->text;
211 if( $nrtext =~ /^\Q$lexeme\E/ ) {
212 $curr_rdg = $lookahead[-1] if @lookahead;
216 push( @lookahead, $nr );
219 # No match? Restore the state we had
221 unshift( @path, @lookahead );
224 $unused_rdg_part = '';
227 unless( $unused_rdg_part ) {
228 # Record the lexemes for the given reading.
229 #print STDERR sprintf( "Adding lexeme(s) %s to reading %s (%s)\n",
230 # join( ' ', map { $_->string } @curr_lexemes ),
231 # $curr_rdg->id, $curr_rdg->text );
232 _update_reading_lexemes( $replace, $curr_rdg, @curr_lexemes );
233 $curr_rdg = shift @path;
240 # Utility function so that we can cheat and use it when we need both the path
242 sub _text_from_path {
243 my( $normalize, @path ) = @_;
246 foreach my $r ( @path ) {
247 unless ( $r->join_prior || !$last || $last->join_next ) {
250 $pathtext .= ( $normalize && $r->has_normal_form )
251 ? $r->normal_form : $r->text;
257 # Utility function that actually calls the tree tagger.
258 sub _treetag_string {
260 my $wittext = encode_utf8( $text );
261 # Then see if we have TreeTagger
262 my $taggercmd = "$MORPHDIR/TreeTagger/cmd/tree-tagger-french-utf8";
263 unless( -f $taggercmd ) {
264 warn "Cannot do French word lemmatization without TreeTagger";
267 # OK, we can run it then.
268 my @cmd = ( $taggercmd );
269 my( $tagresult, $err ); # Capture the output and error
270 run( \@cmd, \$wittext, \$tagresult, \$err );
271 # TODO check for error
272 return decode_utf8( $tagresult );
275 # Closure and utility function for the package lemmatizer
280 # First try to load Flemm
281 unless( $lemmatizer ) {
284 $lemmatizer = Flemm->new( 'Encoding' => 'utf8', 'Tagger' => 'treetagger' );
286 warn "Cannot do French word lemmatization without Flemm: @_";
290 return $lemmatizer->lemmatize( @_ )
295 # Utility function to turn a Flemm result into a WordForm
296 sub _parse_wordform {
297 my $flemmobj = shift;
298 # For now just parse the string, until we make sense of the documentation.
299 my @results = split( / \|\| /, $flemmobj->getResult );
301 foreach ( @results ) {
302 my( $orig, $tag, $lemma ) = split( /\t/, $_ );
303 my( $pos, $morph ) = split( /:/, $tag );
306 $morphobj = Lingua::TagSet::Multext->tag2structure( $morph );
308 # Use the TreeTagger info if there is no Flemm morphology.
309 $morphobj = Lingua::TagSet::TreeTagger->tag2structure( $pos );
312 my $wf = Text::Tradition::Collation::Reading::WordForm->new(
313 'language' => 'French',
315 'morphology' => $morphobj,
319 warn "No morphology found for word: $_";
331 =item * Handle package dependencies more gracefully
337 This package is free software and is provided "as is" without express
338 or implied warranty. You can redistribute it and/or modify it under
339 the same terms as Perl itself.
343 Tara L Andrews E<lt>aurum@cpan.orgE<gt>