3 package Pod::Simple::PullParser;
6 BEGIN {@ISA = ('Pod::Simple')}
11 use Pod::Simple::PullParserStartToken;
12 use Pod::Simple::PullParserEndToken;
13 use Pod::Simple::PullParserTextToken;
15 BEGIN { *DEBUG = \&Pod::Simple::DEBUG unless defined &DEBUG }
17 __PACKAGE__->_accessorize(
18 'source_fh', # the filehandle we're reading from
19 'source_scalar_ref', # the scalarref we're reading from
20 'source_arrayref', # the arrayref we're reading from
23 #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
25 # And here is how we implement a pull-parser on top of a push-parser...
28 my($self, $source) = @_;
29 $self = $self->new unless ref $self;
31 $source = *STDIN{IO} unless defined $source;
32 $self->set_source($source);
33 $self->output_fh(*STDOUT{IO});
35 $self->run; # define run() in a subclass if you want to use filter()!
39 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
41 sub parse_string_document {
43 $this->set_source(\ $_[0]);
48 my($this, $filename) = @_;
49 $this->set_source($filename);
53 # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
54 # In case anyone tries to use them:
58 if( __PACKAGE__ eq ref($_[0]) || $_[0]) { # I'm not being subclassed!
59 Carp::croak "You can call run() only on subclasses of "
63 "You can't call run() because ",
64 ref($_[0]) || $_[0], " didn't define a run() method";
70 Carp::croak "Use set_source with ", __PACKAGE__,
71 " and subclasses, not parse_lines";
76 Carp::croak "Use set_source with ", __PACKAGE__,
77 " and subclasses, not parse_line";
80 #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
84 my $self = $class->SUPER::new(@_);
85 die "Couldn't construct for $class" unless $self;
87 $self->{'token_buffer'} ||= [];
88 $self->{'start_token_class'} ||= 'Pod::Simple::PullParserStartToken';
89 $self->{'text_token_class'} ||= 'Pod::Simple::PullParserTextToken';
90 $self->{'end_token_class'} ||= 'Pod::Simple::PullParserEndToken';
92 DEBUG > 1 and print "New pullparser object: $self\n";
97 # ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
101 DEBUG > 1 and print "\nget_token starting up on $self.\n";
102 DEBUG > 2 and print " Items in token-buffer (",
103 scalar( @{ $self->{'token_buffer'} } ) ,
105 " " . $_->dump . "\n", @{ $self->{'token_buffer'} }
107 @{ $self->{'token_buffer'} } ? '' : ' (no tokens)',
111 until( @{ $self->{'token_buffer'} } ) {
112 DEBUG > 3 and print "I need to get something into my empty token buffer...\n";
113 if($self->{'source_dead'}) {
114 DEBUG and print "$self 's source is dead.\n";
115 push @{ $self->{'token_buffer'} }, undef;
116 } elsif(exists $self->{'source_fh'}) {
118 my $fh = $self->{'source_fh'}
119 || Carp::croak('You have to call set_source before you can call get_token');
121 DEBUG and print "$self 's source is filehandle $fh.\n";
122 # Read those many lines at a time
123 for(my $i = Pod::Simple::MANY_LINES; $i--;) {
124 DEBUG > 3 and print " Fetching a line from source filehandle $fh...\n";
125 local $/ = $Pod::Simple::NL;
126 push @lines, scalar(<$fh>); # readline
127 DEBUG > 3 and print " Line is: ",
128 defined($lines[-1]) ? $lines[-1] : "<undef>\n";
129 unless( defined $lines[-1] ) {
130 DEBUG and print "That's it for that source fh! Killing.\n";
131 delete $self->{'source_fh'}; # so it can be GC'd
134 # but pass thru the undef, which will set source_dead to true
136 # TODO: look to see if $lines[-1] is =encoding, and if so,
137 # do horribly magic things
142 print "* I've gotten ", scalar(@lines), " lines:\n";
143 foreach my $l (@lines) {
145 print " line {$l}\n";
147 print " line undef\n";
150 print "* end of ", scalar(@lines), " lines\n";
153 $self->SUPER::parse_lines(@lines);
155 } elsif(exists $self->{'source_arrayref'}) {
156 DEBUG and print "$self 's source is arrayref $self->{'source_arrayref'}, with ",
157 scalar(@{$self->{'source_arrayref'}}), " items left in it.\n";
159 DEBUG > 3 and print " Fetching ", Pod::Simple::MANY_LINES, " lines.\n";
160 $self->SUPER::parse_lines(
161 splice @{ $self->{'source_arrayref'} },
163 Pod::Simple::MANY_LINES
165 unless( @{ $self->{'source_arrayref'} } ) {
166 DEBUG and print "That's it for that source arrayref! Killing.\n";
167 $self->SUPER::parse_lines(undef);
168 delete $self->{'source_arrayref'}; # so it can be GC'd
170 # to make sure that an undef is always sent to signal end-of-stream
172 } elsif(exists $self->{'source_scalar_ref'}) {
174 DEBUG and print "$self 's source is scalarref $self->{'source_scalar_ref'}, with ",
175 length(${ $self->{'source_scalar_ref'} }) -
176 (pos(${ $self->{'source_scalar_ref'} }) || 0),
177 " characters left to parse.\n";
179 DEBUG > 3 and print " Fetching a line from source-string...\n";
180 if( ${ $self->{'source_scalar_ref'} } =~
181 m/([^\n\r]*)((?:\r?\n)?)/g
184 $self->SUPER::parse_lines($1)
185 if length($1) or length($2)
186 or pos( ${ $self->{'source_scalar_ref'} })
187 != length( ${ $self->{'source_scalar_ref'} });
188 # I.e., unless it's a zero-length "empty line" at the very
189 # end of "foo\nbar\n" (i.e., between the \n and the EOS).
190 } else { # that's the end. Byebye
191 $self->SUPER::parse_lines(undef);
192 delete $self->{'source_scalar_ref'};
193 DEBUG and print "That's it for that source scalarref! Killing.\n";
201 DEBUG and print "get_token about to return ",
202 Pod::Simple::pretty( @{$self->{'token_buffer'}}
203 ? $self->{'token_buffer'}[-1] : undef
205 return shift @{$self->{'token_buffer'}}; # that's an undef if empty
210 DEBUG and print "Ungetting ", scalar(@_), " tokens: ",
211 @_ ? "@_\n" : "().\n";
213 Carp::croak "Can't unget that, because it's not a token -- it's undef!"
215 Carp::croak "Can't unget $t, because it's not a token -- it's a string!"
217 Carp::croak "Can't unget $t, because it's not a token object!"
218 unless UNIVERSAL::can($t, 'type');
221 unshift @{$self->{'token_buffer'}}, @_;
222 DEBUG > 1 and print "Token buffer now has ",
223 scalar(@{$self->{'token_buffer'}}), " items in it.\n";
227 #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
229 # $self->{'source_filename'} = $source;
233 return $self->{'source_fh'} unless @_;
236 Carp::croak("Can't use empty-string as a source for set_source");
237 } elsif(ref(\( $_[0] )) eq 'GLOB') {
238 $self->{'source_filename'} = '' . ($handle = $_[0]);
239 DEBUG and print "$self 's source is glob $_[0]\n";
241 } elsif(ref( $_[0] ) eq 'SCALAR') {
242 $self->{'source_scalar_ref'} = $_[0];
243 DEBUG and print "$self 's source is scalar ref $_[0]\n";
245 } elsif(ref( $_[0] ) eq 'ARRAY') {
246 $self->{'source_arrayref'} = $_[0];
247 DEBUG and print "$self 's source is array ref $_[0]\n";
250 $self->{'source_filename'} = '' . ($handle = $_[0]);
251 DEBUG and print "$self 's source is fh-obj $_[0]\n";
252 } elsif(!length $_[0]) {
253 Carp::croak("Can't use empty-string as a source for set_source");
254 } else { # It's a filename!
255 DEBUG and print "$self 's source is filename $_[0]\n";
258 open(PODSOURCE, "<$_[0]") || Carp::croak "Can't open $_[0]: $!";
259 $handle = *PODSOURCE{IO};
261 $self->{'source_filename'} = $_[0];
262 DEBUG and print " Its name is $_[0].\n";
264 # TODO: file-discipline things here!
267 $self->{'source_fh'} = $handle;
268 DEBUG and print " Its handle is $handle\n";
272 # ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~ ~
274 sub get_title_short { shift->get_short_title(@_) } # alias
276 sub get_short_title {
277 my $title = shift->get_title(@_);
278 $title = $1 if $title =~ m/^(\S{1,60})\s+--?\s+./s;
279 # turn "Foo::Bar -- bars for your foo" into "Foo::Bar"
283 sub get_title { shift->_get_titled_section(
284 'NAME', max_token => 50, desperate => 1, @_)
286 sub get_version { shift->_get_titled_section(
289 accept_verbatim => 1,
290 max_content_length => 3_000,
294 sub get_description { shift->_get_titled_section(
297 max_content_length => 3_000,
301 sub get_authors { shift->get_author(@_) } # a harmless alias
305 # Max_token is so high because these are
306 # typically at the end of the document:
307 $this->_get_titled_section('AUTHOR' , max_token => 10_000, @_) ||
308 $this->_get_titled_section('AUTHORS', max_token => 10_000, @_);
311 #--------------------------------------------------------------------------
313 sub _get_titled_section {
314 # Based on a get_title originally contributed by Graham Barr
315 my($self, $titlename, %options) = (@_);
317 my $max_token = delete $options{'max_token'};
318 my $desperate_for_title = delete $options{'desperate'};
319 my $accept_verbatim = delete $options{'accept_verbatim'};
320 my $max_content_length = delete $options{'max_content_length'};
321 my $nocase = delete $options{'nocase'};
322 $max_content_length = 120 unless defined $max_content_length;
324 Carp::croak( "Unknown " . ((1 == keys %options) ? "option: " : "options: ")
325 . join " ", map "[$_]", sort keys %options
329 my %content_containers;
330 $content_containers{'Para'} = 1;
331 if($accept_verbatim) {
332 $content_containers{'Verbatim'} = 1;
333 $content_containers{'VerbatimFormatted'} = 1;
342 Carp::croak "What kind of titlename is \"$titlename\"?!" unless
343 defined $titlename and $titlename =~ m/^[A-Z ]{1,60}$/s; #sanity
344 my $titlename_re = quotemeta($titlename);
346 my $head1_text_content;
347 my $para_text_content;
350 ++$token_count <= ($max_token || 1_000_000)
351 and defined(my $token = $self->get_token)
353 push @to_unget, $token;
355 if ($state == 0) { # seeking =head1
356 if( $token->is_start and $token->tagname eq 'head1' ) {
357 DEBUG and print " Found head1. Seeking content...\n";
359 $head1_text_content = '';
363 elsif($state == 1) { # accumulating text until end of head1
364 if( $token->is_text ) {
365 DEBUG and print " Adding \"", $token->text, "\" to head1-content.\n";
366 $head1_text_content .= $token->text;
367 } elsif( $token->is_end and $token->tagname eq 'head1' ) {
368 DEBUG and print " Found end of head1. Considering content...\n";
369 $head1_text_content = uc $head1_text_content if $nocase;
370 if($head1_text_content eq $titlename
371 or $head1_text_content =~ m/\($titlename_re\)/s
372 # We accept "=head1 Nomen Modularis (NAME)" for sake of i18n
374 DEBUG and print " Yup, it was $titlename. Seeking next para-content...\n";
378 # if we're so desperate we'll take the first
379 # =head1's content as a title
380 and $head1_text_content =~ m/\S/
381 and $head1_text_content !~ m/^[ A-Z]+$/s
382 and $head1_text_content !~
384 NAME | TITLE | VERSION | AUTHORS? | DESCRIPTION | SYNOPSIS
385 | COPYRIGHT | LICENSE | NOTES? | FUNCTIONS? | METHODS?
386 | CAVEATS? | BUGS? | SEE\ ALSO | SWITCHES | ENVIRONMENT
388 # avoid accepting things like =head1 Thingy Thongy (DESCRIPTION)
389 and ($max_content_length
390 ? (length($head1_text_content) <= $max_content_length) # sanity
393 DEBUG and print " It looks titular: \"$head1_text_content\".\n",
395 $title = $head1_text_content;
399 DEBUG and print " Didn't look titular ($head1_text_content).\n",
400 "\n Dropping back to seeking-head1-content mode...\n";
406 # seeking start of para (which must immediately follow)
407 if($token->is_start and $content_containers{ $token->tagname }) {
408 DEBUG and print " Found start of Para. Accumulating content...\n";
409 $para_text_content = '';
413 " Didn't see an immediately subsequent start-Para. Reseeking H1\n";
419 # accumulating text until end of Para
420 if( $token->is_text ) {
421 DEBUG and print " Adding \"", $token->text, "\" to para-content.\n";
422 $para_text_content .= $token->text;
425 } elsif( $token->is_end and $content_containers{ $token->tagname } ) {
426 DEBUG and print " Found end of Para. Considering content: ",
427 $para_text_content, "\n";
429 if( $para_text_content =~ m/\S/
430 and ($max_content_length
431 ? (length($para_text_content) <= $max_content_length)
434 # Some minimal sanity constraints, I think.
435 DEBUG and print " It looks contentworthy, I guess. Using it.\n";
436 $title = $para_text_content;
439 DEBUG and print " Doesn't look at all contentworthy!\n Giving up.\n";
447 die "IMPOSSIBLE STATE $state!\n"; # should never happen
453 $self->unget_token(@to_unget);
456 if(defined $title) { print " Returing title <$title>\n" }
457 else { print "Returning title <>\n" }
460 return '' unless defined $title;
465 #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
467 # Methods that actually do work at parse-time:
469 sub _handle_element_start {
470 my $self = shift; # leaving ($element_name, $attr_hash_r)
471 DEBUG > 2 and print "++ $_[0] (", map("<$_> ", %{$_[1]}), ")\n";
473 push @{ $self->{'token_buffer'} },
474 $self->{'start_token_class'}->new(@_);
479 my $self = shift; # leaving ($text)
480 DEBUG > 2 and print "== $_[0]\n";
481 push @{ $self->{'token_buffer'} },
482 $self->{'text_token_class'}->new(@_);
486 sub _handle_element_end {
487 my $self = shift; # leaving ($element_name);
488 DEBUG > 2 and print "-- $_[0]\n";
489 push @{ $self->{'token_buffer'} },
490 $self->{'end_token_class'}->new(@_);
494 #@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
503 Pod::Simple::PullParser -- a pull-parser interface to parsing Pod
507 my $parser = SomePodProcessor->new;
508 $parser->set_source( "whatever.pod" );
513 my $parser = SomePodProcessor->new;
514 $parser->set_source( $some_filehandle_object );
519 my $parser = SomePodProcessor->new;
520 $parser->set_source( \$document_source );
525 my $parser = SomePodProcessor->new;
526 $parser->set_source( \@document_lines );
532 package SomePodProcessor;
534 use base qw(Pod::Simple::PullParser);
539 while(my $token = $self->get_token) {
540 ...process each token...
546 This class is for using Pod::Simple to build a Pod processor -- but
547 one that uses an interface based on a stream of token objects,
548 instead of based on events.
550 This is a subclass of L<Pod::Simple> and inherits all its methods.
552 A subclass of Pod::Simple::PullParser should define a C<run> method
553 that calls C<< $token = $parser->get_token >> to pull tokens.
555 See the source for Pod::Simple::RTF for an example of a formatter
556 that uses Pod::Simple::PullParser.
562 =item my $token = $parser->get_token
564 This returns the next token object (which will be of a subclass of
565 L<Pod::Simple::PullParserToken>), or undef if the parser-stream has hit
566 the end of the document.
568 =item $parser->unget_token( $token )
570 =item $parser->unget_token( $token1, $token2, ... )
572 This restores the token object(s) to the front of the parser stream.
576 The source has to be set before you can parse anything. The lowest-level
577 way is to call C<set_source>:
581 =item $parser->set_source( $filename )
583 =item $parser->set_source( $filehandle_object )
585 =item $parser->set_source( \$document_source )
587 =item $parser->set_source( \@document_lines )
591 Or you can call these methods, which Pod::Simple::PullParser has defined
592 to work just like Pod::Simple's same-named methods:
596 =item $parser->parse_file(...)
598 =item $parser->parse_string_document(...)
600 =item $parser->filter(...)
602 =item $parser->parse_from_file(...)
606 For those to work, the Pod-processing subclass of
607 Pod::Simple::PullParser has to have defined a $parser->run method --
608 so it is advised that all Pod::Simple::PullParser subclasses do so.
609 See the Synopsis above, or the source for Pod::Simple::RTF.
611 Authors of formatter subclasses might find these methods useful to
612 call on a parser object that you haven't started pulling tokens
617 =item my $title_string = $parser->get_title
619 This tries to get the title string out of $parser, by getting some tokens,
620 and scanning them for the title, and then ungetting them so that you can
621 process the token-stream from the beginning.
623 For example, suppose you have a document that starts out:
627 Hoo::Boy::Wowza -- Stuff B<wow> yeah!
629 $parser->get_title on that document will return "Hoo::Boy::Wowza --
630 Stuff wow yeah!". If the document starts with:
634 Hoo::Boy::W00t -- Stuff B<w00t> yeah!
636 Then you'll need to pass the C<nocase> option in order to recognize "Name":
638 $parser->get_title(nocase => 1);
640 In cases where get_title can't find the title, it will return empty-string
643 =item my $title_string = $parser->get_short_title
645 This is just like get_title, except that it returns just the modulename, if
646 the title seems to be of the form "SomeModuleName -- description".
648 For example, suppose you have a document that starts out:
652 Hoo::Boy::Wowza -- Stuff B<wow> yeah!
654 then $parser->get_short_title on that document will return
657 But if the document starts out:
661 Hooboy, stuff B<wow> yeah!
663 then $parser->get_short_title on that document will return "Hooboy,
664 stuff wow yeah!". If the document starts with:
668 Hoo::Boy::W00t -- Stuff B<w00t> yeah!
670 Then you'll need to pass the C<nocase> option in order to recognize "Name":
672 $parser->get_short_title(nocase => 1);
674 If the title can't be found, then get_short_title returns empty-string
677 =item $author_name = $parser->get_author
679 This works like get_title except that it returns the contents of the
680 "=head1 AUTHOR\n\nParagraph...\n" section, assuming that that section
681 isn't terribly long. To recognize a "=head1 Author\n\nParagraph\n"
682 section, pass the C<nocase> otpion:
684 $parser->get_author(nocase => 1);
686 (This method tolerates "AUTHORS" instead of "AUTHOR" too.)
688 =item $description_name = $parser->get_description
690 This works like get_title except that it returns the contents of the
691 "=head1 DESCRIPTION\n\nParagraph...\n" section, assuming that that section
692 isn't terribly long. To recognize a "=head1 Description\n\nParagraph\n"
693 section, pass the C<nocase> otpion:
695 $parser->get_description(nocase => 1);
697 =item $version_block = $parser->get_version
699 This works like get_title except that it returns the contents of
700 the "=head1 VERSION\n\n[BIG BLOCK]\n" block. Note that this does NOT
701 return the module's C<$VERSION>!! To recognize a
702 "=head1 Version\n\n[BIG BLOCK]\n" section, pass the C<nocase> otpion:
704 $parser->get_version(nocase => 1);
710 You don't actually I<have> to define a C<run> method. If you're
711 writing a Pod-formatter class, you should define a C<run> just so
712 that users can call C<parse_file> etc, but you don't I<have> to.
714 And if you're not writing a formatter class, but are instead just
715 writing a program that does something simple with a Pod::PullParser
716 object (and not an object of a subclass), then there's no reason to
717 bother subclassing to add a C<run> method.
723 L<Pod::Simple::PullParserToken> -- and its subclasses
724 L<Pod::Simple::PullParserStartToken>,
725 L<Pod::Simple::PullParserTextToken>, and
726 L<Pod::Simple::PullParserEndToken>.
728 L<HTML::TokeParser>, which inspired this.
732 Questions or discussion about POD and Pod::Simple should be sent to the
733 pod-people@perl.org mail list. Send an empty email to
734 pod-people-subscribe@perl.org to subscribe.
736 This module is managed in an open GitHub repository,
737 L<http://github.com/theory/pod-simple/>. Feel free to fork and contribute, or
738 to clone L<git://github.com/theory/pod-simple.git> and send patches!
740 Patches against Pod::Simple are welcome. Please send bug reports to
741 <bug-pod-simple@rt.cpan.org>.
743 =head1 COPYRIGHT AND DISCLAIMERS
745 Copyright (c) 2002 Sean M. Burke.
747 This library is free software; you can redistribute it and/or modify it
748 under the same terms as Perl itself.
750 This program is distributed in the hope that it will be useful, but
751 without any warranty; without even the implied warranty of
752 merchantability or fitness for a particular purpose.
756 Pod::Simple was created by Sean M. Burke <sburke@cpan.org>.
757 But don't bother him, he's retired.
759 Pod::Simple is maintained by:
763 =item * Allison Randal C<allison@perl.org>
765 =item * Hans Dieter Pearcey C<hdp@cpan.org>
767 =item * David E. Wheeler C<dwheeler@cpan.org>
774 sub _old_get_title { # some witchery in here
780 push @to_unget, $self->get_token;
781 unless(defined $to_unget[-1]) { # whoops, short doc!
786 DEBUG and print "-Got token ", $to_unget[-1]->dump, "\n";
788 (DEBUG and print "Too much in the buffer.\n"),
789 last if @to_unget > 25; # sanity
792 if( #$to_unget[-1]->type eq 'end'
793 #and $to_unget[-1]->tagname eq 'Para'
797 ($_->type eq 'start') ? ("<" . $_->tagname .">")
798 : ($_->type eq 'end' ) ? ("</". $_->tagname .">")
799 : ($_->type eq 'text' ) ? ($_->text =~ m<^([A-Z]+)$>s ? $1 : 'X')
802 )) =~ m{<head1>NAME</head1><Para>(X|</?[BCIFLS]>)+</Para>$}s
804 # Whee, it fits the pattern
805 DEBUG and print "Seems to match =head1 NAME pattern.\n";
807 foreach my $t (reverse @to_unget) {
808 last if $t->type eq 'start' and $t->tagname eq 'Para';
809 $title = $t->text . $title if $t->type eq 'text';
811 undef $title if $title =~ m<^\s*$>; # make sure it's contentful!
814 } elsif ($pattern =~ m{<head(\d)>(.+)</head\d>$}
815 and !( $1 eq '1' and $2 eq 'NAME' )
817 # Well, it fits a fallback pattern
818 DEBUG and print "Seems to match NAMEless pattern.\n";
820 foreach my $t (reverse @to_unget) {
821 last if $t->type eq 'start' and $t->tagname =~ m/^head\d$/s;
822 $title = $t->text . $title if $t->type eq 'text';
824 undef $title if $title =~ m<^\s*$>; # make sure it's contentful!
828 DEBUG and $pattern and print "Leading pattern: $pattern\n";
833 $self->unget_token(@to_unget);
836 if(defined $title) { print " Returing title <$title>\n" }
837 else { print "Returning title <>\n" }
840 return '' unless defined $title;