3 package Pod::Simple::PullParser;
6 BEGIN {@ISA = ('Pod::Simple')}
11 use Pod::Simple::PullParserStartToken;
12 use Pod::Simple::PullParserEndToken;
13 use Pod::Simple::PullParserTextToken;
15 BEGIN { *DEBUG = \&Pod::Simple::DEBUG unless defined &DEBUG }
17 __PACKAGE__->_accessorize(
18 'source_fh', # the filehandle we're reading from
19 'source_scalar_ref', # the scalarref we're reading from
20 'source_arrayref', # the arrayref we're reading from
23 #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
25 # And here is how we implement a pull-parser on top of a push-parser...
28 my($self, $source) = @_;
29 $self = $self->new unless ref $self;
31 $source = *STDIN{IO} unless defined $source;
32 $self->set_source($source);
33 $self->output_fh(*STDOUT{IO});
35 $self->run; # define run() in a subclass if you want to use filter()!
39 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
41 sub parse_string_document {
43 $this->set_source(\ $_[0]);
48 my($this, $filename) = @_;
49 $this->set_source($filename);
53 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
54 # In case anyone tries to use them:
58 if( __PACKAGE__ eq ref($_[0]) || $_[0]) { # I'm not being subclassed!
59 Carp::croak "You can call run() only on subclasses of "
63 "You can't call run() because ",
64 ref($_[0]) || $_[0], " didn't define a run() method";
70 Carp::croak "Use set_source with ", __PACKAGE__,
71 " and subclasses, not parse_lines";
76 Carp::croak "Use set_source with ", __PACKAGE__,
77 " and subclasses, not parse_line";
80 #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
84 my $self = $class->SUPER::new(@_);
85 die "Couldn't construct for $class" unless $self;
87 $self->{'token_buffer'} ||= [];
88 $self->{'start_token_class'} ||= 'Pod::Simple::PullParserStartToken';
89 $self->{'text_token_class'} ||= 'Pod::Simple::PullParserTextToken';
90 $self->{'end_token_class'} ||= 'Pod::Simple::PullParserEndToken';
92 DEBUG > 1 and print "New pullparser object: $self\n";
97 # ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
101 DEBUG > 1 and print "\nget_token starting up on $self.\n";
102 DEBUG > 2 and print " Items in token-buffer (",
103 scalar( @{ $self->{'token_buffer'} } ) ,
105 " " . $_->dump . "\n", @{ $self->{'token_buffer'} }
107 @{ $self->{'token_buffer'} } ? '' : ' (no tokens)',
111 until( @{ $self->{'token_buffer'} } ) {
112 DEBUG > 3 and print "I need to get something into my empty token buffer...\n";
113 if($self->{'source_dead'}) {
114 DEBUG and print "$self 's source is dead.\n";
115 push @{ $self->{'token_buffer'} }, undef;
116 } elsif(exists $self->{'source_fh'}) {
118 my $fh = $self->{'source_fh'}
119 || Carp::croak('You have to call set_source before you can call get_token');
121 DEBUG and print "$self 's source is filehandle $fh.\n";
122 # Read those many lines at a time
123 for(my $i = Pod::Simple::MANY_LINES; $i--;) {
124 DEBUG > 3 and print " Fetching a line from source filehandle $fh...\n";
125 local $/ = $Pod::Simple::NL;
126 push @lines, scalar(<$fh>); # readline
127 DEBUG > 3 and print " Line is: ",
128 defined($lines[-1]) ? $lines[-1] : "<undef>\n";
129 unless( defined $lines[-1] ) {
130 DEBUG and print "That's it for that source fh! Killing.\n";
131 delete $self->{'source_fh'}; # so it can be GC'd
134 # but pass thru the undef, which will set source_dead to true
136 # TODO: look to see if $lines[-1] is =encoding, and if so,
137 # do horribly magic things
142 print "* I've gotten ", scalar(@lines), " lines:\n";
143 foreach my $l (@lines) {
145 print " line {$l}\n";
147 print " line undef\n";
150 print "* end of ", scalar(@lines), " lines\n";
153 $self->SUPER::parse_lines(@lines);
155 } elsif(exists $self->{'source_arrayref'}) {
156 DEBUG and print "$self 's source is arrayref $self->{'source_arrayref'}, with ",
157 scalar(@{$self->{'source_arrayref'}}), " items left in it.\n";
159 DEBUG > 3 and print " Fetching ", Pod::Simple::MANY_LINES, " lines.\n";
160 $self->SUPER::parse_lines(
161 splice @{ $self->{'source_arrayref'} },
163 Pod::Simple::MANY_LINES
165 unless( @{ $self->{'source_arrayref'} } ) {
166 DEBUG and print "That's it for that source arrayref! Killing.\n";
167 $self->SUPER::parse_lines(undef);
168 delete $self->{'source_arrayref'}; # so it can be GC'd
170 # to make sure that an undef is always sent to signal end-of-stream
172 } elsif(exists $self->{'source_scalar_ref'}) {
174 DEBUG and print "$self 's source is scalarref $self->{'source_scalar_ref'}, with ",
175 length(${ $self->{'source_scalar_ref'} }) -
176 (pos(${ $self->{'source_scalar_ref'} }) || 0),
177 " characters left to parse.\n";
179 DEBUG > 3 and print " Fetching a line from source-string...\n";
180 if( ${ $self->{'source_scalar_ref'} } =~
181 m/([^\n\r]*)((?:\r?\n)?)/g
184 $self->SUPER::parse_lines($1)
185 if length($1) or length($2)
186 or pos( ${ $self->{'source_scalar_ref'} })
187 != length( ${ $self->{'source_scalar_ref'} });
188 # I.e., unless it's a zero-length "empty line" at the very
189 # end of "foo\nbar\n" (i.e., between the \n and the EOS).
190 } else { # that's the end. Byebye
191 $self->SUPER::parse_lines(undef);
192 delete $self->{'source_scalar_ref'};
193 DEBUG and print "That's it for that source scalarref! Killing.\n";
201 DEBUG and print "get_token about to return ",
202 Pod::Simple::pretty( @{$self->{'token_buffer'}}
203 ? $self->{'token_buffer'}[-1] : undef
205 return shift @{$self->{'token_buffer'}}; # that's an undef if empty
211 DEBUG and print "Ungetting ", scalar(@_), " tokens: ",
212 @_ ? "@_\n" : "().\n";
214 Carp::croak "Can't unget that, because it's not a token -- it's undef!"
216 Carp::croak "Can't unget $t, because it's not a token -- it's a string!"
218 Carp::croak "Can't unget $t, because it's not a token object!"
219 unless UNIVERSAL::can($t, 'type');
222 unshift @{$self->{'token_buffer'}}, @_;
223 DEBUG > 1 and print "Token buffer now has ",
224 scalar(@{$self->{'token_buffer'}}), " items in it.\n";
228 #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
230 # $self->{'source_filename'} = $source;
234 return $self->{'source_fh'} unless @_;
237 Carp::croak("Can't use empty-string as a source for set_source");
238 } elsif(ref(\( $_[0] )) eq 'GLOB') {
239 $self->{'source_filename'} = '' . ($handle = $_[0]);
240 DEBUG and print "$self 's source is glob $_[0]\n";
242 } elsif(ref( $_[0] ) eq 'SCALAR') {
243 $self->{'source_scalar_ref'} = $_[0];
244 DEBUG and print "$self 's source is scalar ref $_[0]\n";
246 } elsif(ref( $_[0] ) eq 'ARRAY') {
247 $self->{'source_arrayref'} = $_[0];
248 DEBUG and print "$self 's source is array ref $_[0]\n";
251 $self->{'source_filename'} = '' . ($handle = $_[0]);
252 DEBUG and print "$self 's source is fh-obj $_[0]\n";
253 } elsif(!length $_[0]) {
254 Carp::croak("Can't use empty-string as a source for set_source");
255 } else { # It's a filename!
256 DEBUG and print "$self 's source is filename $_[0]\n";
259 open(PODSOURCE, "<$_[0]") || Carp::croak "Can't open $_[0]: $!";
260 $handle = *PODSOURCE{IO};
262 $self->{'source_filename'} = $_[0];
263 DEBUG and print " Its name is $_[0].\n";
265 # TODO: file-discipline things here!
268 $self->{'source_fh'} = $handle;
269 DEBUG and print " Its handle is $handle\n";
273 # ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
275 sub get_title_short { shift->get_short_title(@_) } # alias
277 sub get_short_title {
278 my $title = shift->get_title(@_);
279 $title = $1 if $title =~ m/^(\S{1,60})\s+--?\s+./s;
280 # turn "Foo::Bar -- bars for your foo" into "Foo::Bar"
284 sub get_title { shift->_get_titled_section(
285 'NAME', max_token => 50, desperate => 1, @_)
287 sub get_version { shift->_get_titled_section(
290 accept_verbatim => 1,
291 max_content_length => 3_000,
295 sub get_description { shift->_get_titled_section(
298 max_content_length => 3_000,
302 sub get_authors { shift->get_author(@_) } # a harmless alias
306 # Max_token is so high because these are
307 # typically at the end of the document:
308 $this->_get_titled_section('AUTHOR' , max_token => 10_000, @_) ||
309 $this->_get_titled_section('AUTHORS', max_token => 10_000, @_);
312 #--------------------------------------------------------------------------
314 sub _get_titled_section {
315 # Based on a get_title originally contributed by Graham Barr
316 my($self, $titlename, %options) = (@_);
318 my $max_token = delete $options{'max_token'};
319 my $desperate_for_title = delete $options{'desperate'};
320 my $accept_verbatim = delete $options{'accept_verbatim'};
321 my $max_content_length = delete $options{'max_content_length'};
322 $max_content_length = 120 unless defined $max_content_length;
324 Carp::croak( "Unknown " . ((1 == keys %options) ? "option: " : "options: ")
325 . join " ", map "[$_]", sort keys %options
329 my %content_containers;
330 $content_containers{'Para'} = 1;
331 if($accept_verbatim) {
332 $content_containers{'Verbatim'} = 1;
333 $content_containers{'VerbatimFormatted'} = 1;
342 Carp::croak "What kind of titlename is \"$titlename\"?!" unless
343 defined $titlename and $titlename =~ m/^[A-Z ]{1,60}$/s; #sanity
344 my $titlename_re = quotemeta($titlename);
346 my $head1_text_content;
347 my $para_text_content;
350 ++$token_count <= ($max_token || 1_000_000)
351 and defined(my $token = $self->get_token)
353 push @to_unget, $token;
355 if ($state == 0) { # seeking =head1
356 if( $token->is_start and $token->tagname eq 'head1' ) {
357 DEBUG and print " Found head1. Seeking content...\n";
359 $head1_text_content = '';
363 elsif($state == 1) { # accumulating text until end of head1
364 if( $token->is_text ) {
365 DEBUG and print " Adding \"", $token->text, "\" to head1-content.\n";
366 $head1_text_content .= $token->text;
367 } elsif( $token->is_end and $token->tagname eq 'head1' ) {
368 DEBUG and print " Found end of head1. Considering content...\n";
369 if($head1_text_content eq $titlename
370 or $head1_text_content =~ m/\($titlename_re\)/s
371 # We accept "=head1 Nomen Modularis (NAME)" for sake of i18n
373 DEBUG and print " Yup, it was $titlename. Seeking next para-content...\n";
377 # if we're so desperate we'll take the first
378 # =head1's content as a title
379 and $head1_text_content =~ m/\S/
380 and $head1_text_content !~ m/^[ A-Z]+$/s
381 and $head1_text_content !~
383 NAME | TITLE | VERSION | AUTHORS? | DESCRIPTION | SYNOPSIS
384 | COPYRIGHT | LICENSE | NOTES? | FUNCTIONS? | METHODS?
385 | CAVEATS? | BUGS? | SEE\ ALSO | SWITCHES | ENVIRONMENT
387 # avoid accepting things like =head1 Thingy Thongy (DESCRIPTION)
388 and ($max_content_length
389 ? (length($head1_text_content) <= $max_content_length) # sanity
392 DEBUG and print " It looks titular: \"$head1_text_content\".\n",
394 $title = $head1_text_content;
398 DEBUG and print " Didn't look titular ($head1_text_content).\n",
399 "\n Dropping back to seeking-head1-content mode...\n";
405 # seeking start of para (which must immediately follow)
406 if($token->is_start and $content_containers{ $token->tagname }) {
407 DEBUG and print " Found start of Para. Accumulating content...\n";
408 $para_text_content = '';
412 " Didn't see an immediately subsequent start-Para. Reseeking H1\n";
418 # accumulating text until end of Para
419 if( $token->is_text ) {
420 DEBUG and print " Adding \"", $token->text, "\" to para-content.\n";
421 $para_text_content .= $token->text;
424 } elsif( $token->is_end and $content_containers{ $token->tagname } ) {
425 DEBUG and print " Found end of Para. Considering content: ",
426 $para_text_content, "\n";
428 if( $para_text_content =~ m/\S/
429 and ($max_content_length
430 ? (length($para_text_content) <= $max_content_length)
433 # Some minimal sanity constraints, I think.
434 DEBUG and print " It looks contentworthy, I guess. Using it.\n";
435 $title = $para_text_content;
438 DEBUG and print " Doesn't look at all contentworthy!\n Giving up.\n";
446 die "IMPOSSIBLE STATE $state!\n"; # should never happen
452 $self->unget_token(@to_unget);
455 if(defined $title) { print " Returing title <$title>\n" }
456 else { print "Returning title <>\n" }
459 return '' unless defined $title;
464 #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
466 # Methods that actually do work at parse-time:
468 sub _handle_element_start {
469 my $self = shift; # leaving ($element_name, $attr_hash_r)
470 DEBUG > 2 and print "++ $_[0] (", map("<$_> ", %{$_[1]}), ")\n";
472 push @{ $self->{'token_buffer'} },
473 $self->{'start_token_class'}->new(@_);
478 my $self = shift; # leaving ($text)
479 DEBUG > 2 and print "== $_[0]\n";
480 push @{ $self->{'token_buffer'} },
481 $self->{'text_token_class'}->new(@_);
485 sub _handle_element_end {
486 my $self = shift; # leaving ($element_name);
487 DEBUG > 2 and print "-- $_[0]\n";
488 push @{ $self->{'token_buffer'} },
489 $self->{'end_token_class'}->new(@_);
493 #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
502 Pod::Simple::PullParser -- a pull-parser interface to parsing Pod
506 my $parser = SomePodProcessor->new;
507 $parser->set_source( "whatever.pod" );
512 my $parser = SomePodProcessor->new;
513 $parser->set_source( $some_filehandle_object );
518 my $parser = SomePodProcessor->new;
519 $parser->set_source( \$document_source );
524 my $parser = SomePodProcessor->new;
525 $parser->set_source( \@document_lines );
531 package SomePodProcessor;
533 use base qw(Pod::Simple::PullParser);
538 while(my $token = $self->get_token) {
539 ...process each token...
545 This class is for using Pod::Simple to build a Pod processor -- but
546 one that uses an interface based on a stream of token objects,
547 instead of based on events.
549 This is a subclass of L<Pod::Simple> and inherits all its methods.
551 A subclass of Pod::Simple::PullParser should define a C<run> method
552 that calls C<< $token = $parser->get_token >> to pull tokens.
554 See the source for Pod::Simple::RTF for an example of a formatter
555 that uses Pod::Simple::PullParser.
561 =item my $token = $parser->get_token
563 This returns the next token object (which will be of a subclass of
564 L<Pod::Simple::PullParserToken>), or undef if the parser-stream has hit
565 the end of the document.
567 =item $parser->unget_token( $token )
569 =item $parser->unget_token( $token1, $token2, ... )
571 This restores the token object(s) to the front of the parser stream.
575 The source has to be set before you can parse anything. The lowest-level
576 way is to call C<set_source>:
580 =item $parser->set_source( $filename )
582 =item $parser->set_source( $filehandle_object )
584 =item $parser->set_source( \$document_source )
586 =item $parser->set_source( \@document_lines )
590 Or you can call these methods, which Pod::Simple::PullParser has defined
591 to work just like Pod::Simple's same-named methods:
595 =item $parser->parse_file(...)
597 =item $parser->parse_string_document(...)
599 =item $parser->filter(...)
601 =item $parser->parse_from_file(...)
605 For those to work, the Pod-processing subclass of
606 Pod::Simple::PullParser has to have defined a $parser->run method --
607 so it is advised that all Pod::Simple::PullParser subclasses do so.
608 See the Synopsis above, or the source for Pod::Simple::RTF.
610 Authors of formatter subclasses might find these methods useful to
611 call on a parser object that you haven't started pulling tokens
616 =item my $title_string = $parser->get_title
618 This tries to get the title string out of $parser, by getting some tokens,
619 and scanning them for the title, and then ungetting them so that you can
620 process the token-stream from the beginning.
622 For example, suppose you have a document that starts out:
626 Hoo::Boy::Wowza -- Stuff B<wow> yeah!
628 $parser->get_title on that document will return "Hoo::Boy::Wowza --
631 In cases where get_title can't find the title, it will return empty-string
634 =item my $title_string = $parser->get_short_title
636 This is just like get_title, except that it returns just the modulename, if
637 the title seems to be of the form "SomeModuleName -- description".
639 For example, suppose you have a document that starts out:
643 Hoo::Boy::Wowza -- Stuff B<wow> yeah!
645 then $parser->get_short_title on that document will return
648 But if the document starts out:
652 Hooboy, stuff B<wow> yeah!
654 then $parser->get_short_title on that document will return "Hooboy,
657 If the title can't be found, then get_short_title returns empty-string
660 =item $author_name = $parser->get_author
662 This works like get_title except that it returns the contents of the
663 "=head1 AUTHOR\n\nParagraph...\n" section, assuming that that section
666 (This method tolerates "AUTHORS" instead of "AUTHOR" too.)
668 =item $description_name = $parser->get_description
670 This works like get_title except that it returns the contents of the
671 "=head1 PARAGRAPH\n\nParagraph...\n" section, assuming that that section
674 =item $version_block = $parser->get_version
676 This works like get_title except that it returns the contents of
677 the "=head1 VERSION\n\n[BIG BLOCK]\n" block. Note that this does NOT
678 return the module's C<$VERSION>!!
685 You don't actually I<have> to define a C<run> method. If you're
686 writing a Pod-formatter class, you should define a C<run> just so
687 that users can call C<parse_file> etc, but you don't I<have> to.
689 And if you're not writing a formatter class, but are instead just
690 writing a program that does something simple with a Pod::PullParser
691 object (and not an object of a subclass), then there's no reason to
692 bother subclassing to add a C<run> method.
698 L<Pod::Simple::PullParserToken> -- and its subclasses
699 L<Pod::Simple::PullParserStartToken>,
700 L<Pod::Simple::PullParserTextToken>, and
701 L<Pod::Simple::PullParserEndToken>.
703 L<HTML::TokeParser>, which inspired this.
705 =head1 COPYRIGHT AND DISCLAIMERS
707 Copyright (c) 2002 Sean M. Burke. All rights reserved.
709 This library is free software; you can redistribute it and/or modify it
710 under the same terms as Perl itself.
712 This program is distributed in the hope that it will be useful, but
713 without any warranty; without even the implied warranty of
714 merchantability or fitness for a particular purpose.
718 Sean M. Burke C<sburke@cpan.org>
726 sub _old_get_title { # some witchery in here
732 push @to_unget, $self->get_token;
733 unless(defined $to_unget[-1]) { # whoops, short doc!
738 DEBUG and print "-Got token ", $to_unget[-1]->dump, "\n";
740 (DEBUG and print "Too much in the buffer.\n"),
741 last if @to_unget > 25; # sanity
744 if( #$to_unget[-1]->type eq 'end'
745 #and $to_unget[-1]->tagname eq 'Para'
749 ($_->type eq 'start') ? ("<" . $_->tagname .">")
750 : ($_->type eq 'end' ) ? ("</". $_->tagname .">")
751 : ($_->type eq 'text' ) ? ($_->text =~ m<^([A-Z]+)$>s ? $1 : 'X')
754 )) =~ m{<head1>NAME</head1><Para>(X|</?[BCIFLS]>)+</Para>$}s
756 # Whee, it fits the pattern
757 DEBUG and print "Seems to match =head1 NAME pattern.\n";
759 foreach my $t (reverse @to_unget) {
760 last if $t->type eq 'start' and $t->tagname eq 'Para';
761 $title = $t->text . $title if $t->type eq 'text';
763 undef $title if $title =~ m<^\s*$>; # make sure it's contentful!
766 } elsif ($pattern =~ m{<head(\d)>(.+)</head\d>$}
767 and !( $1 eq '1' and $2 eq 'NAME' )
769 # Well, it fits a fallback pattern
770 DEBUG and print "Seems to match NAMEless pattern.\n";
772 foreach my $t (reverse @to_unget) {
773 last if $t->type eq 'start' and $t->tagname =~ m/^head\d$/s;
774 $title = $t->text . $title if $t->type eq 'text';
776 undef $title if $title =~ m<^\s*$>; # make sure it's contentful!
780 DEBUG and $pattern and print "Leading pattern: $pattern\n";
785 $self->unget_token(@to_unget);
788 if(defined $title) { print " Returing title <$title>\n" }
789 else { print "Returning title <>\n" }
792 return '' unless defined $title;