12 use vars qw($VERSION @ISA);
18 TAP::Harness - Run test scripts with statistics
28 $ENV{HARNESS_ACTIVE} = 1;
29 $ENV{HARNESS_VERSION} = $VERSION;
34 delete $ENV{HARNESS_ACTIVE};
35 delete $ENV{HARNESS_VERSION};
40 This is a simple test harness which allows tests to be run and results
41 automatically aggregated and output to STDOUT.
46 my $harness = TAP::Harness->new( \%args );
47 $harness->runtests(@tests);
56 return $self->{error} unless @_;
57 $self->{error} = shift;
63 directives verbosity timer failures errors stdout color show_count
68 my ( $self, $libs ) = @_;
69 $libs = [$libs] unless 'ARRAY' eq ref $libs;
71 return [ map {"-I$_"} @$libs ];
73 switches => sub { shift; shift },
74 exec => sub { shift; shift },
75 merge => sub { shift; shift },
76 aggregator_class => sub { shift; shift },
77 formatter_class => sub { shift; shift },
78 multiplexer_class => sub { shift; shift },
79 parser_class => sub { shift; shift },
80 scheduler_class => sub { shift; shift },
81 formatter => sub { shift; shift },
82 jobs => sub { shift; shift },
83 fork => sub { shift; shift },
84 test_args => sub { shift; shift },
85 ignore_exit => sub { shift; shift },
86 rules => sub { shift; shift },
89 for my $method ( sort keys %VALIDATION_FOR ) {
91 if ( $method eq 'lib' || $method eq 'switches' ) {
95 $self->{$method} ||= [];
97 ? @{ $self->{$method} }
100 $self->_croak("Too many arguments to method '$method'")
103 $args = [$args] unless ref $args;
104 $self->{$method} = $args;
111 return $self->{$method} unless @_;
112 $self->{$method} = shift;
117 for my $method (@FORMATTER_ARGS) {
121 return $self->formatter->$method(@_);
126 ##############################################################################
136 lib => [ 'lib', 'blib/lib' ],
138 my $harness = TAP::Harness->new( \%args );
140 The constructor returns a new C<TAP::Harness> object. It accepts an
141 optional hashref whose allowed keys are:
147 Set the verbosity level:
149 1 verbose Print individual test results to STDOUT.
151 -1 quiet Suppress some test output (mostly failures
152 while tests are running).
153 -2 really quiet Suppress everything but the tests summary.
154 -3 silent Suppress everything.
158 Append run time for each test to output. Uses L<Time::HiRes> if
163 Only show test failures (this is a no-op if C<verbose> is selected).
165 =item * C<show_count>
167 Update the running test count during testing.
171 Accepts a scalar value or array ref of scalar values indicating which
172 paths to allowed libraries should be included if Perl tests are
173 executed. Naturally, this only makes sense in the context of tests
178 Accepts a scalar value or array ref of scalar values indicating which
179 switches should be included if Perl tests are executed. Naturally, this
180 only makes sense in the context of tests written in Perl.
184 A reference to an C<@INC> style array of arguments to be passed to each
189 Attempt to produce color output.
193 Typically, Perl tests are run through this. However, anything which
194 spits out TAP is fine. You can use this argument to specify the name of
195 the program (and optional switches) to run your tests with:
197 exec => ['/usr/bin/ruby', '-w']
199 You can also pass a subroutine reference in order to determine and
200 return the proper program to run based on a given test script. The
201 subroutine reference should expect the TAP::Harness object itself as the
202 first argument, and the file name as the second argument. It should
203 return an array reference containing the command to be run and including
204 the test file name. It can also simply return C<undef>, in which case
205 TAP::Harness will fall back on executing the test script in Perl:
208 my ( $harness, $test_file ) = @_;
210 # Let Perl tests run.
211 return undef if $test_file =~ /[.]t$/;
212 return [ qw( /usr/bin/ruby -w ), $test_file ]
213 if $test_file =~ /[.]rb$/;
218 If C<merge> is true the harness will create parsers that merge STDOUT
219 and STDERR together for any processes they start.
221 =item * C<aggregator_class>
223 The name of the class to use to aggregate test results. The default is
224 L<TAP::Parser::Aggregator>.
226 =item * C<formatter_class>
228 The name of the class to use to format output. The default is
229 L<TAP::Formatter::Console>.
231 =item * C<multiplexer_class>
233 The name of the class to use to multiplex tests during parallel testing.
234 The default is L<TAP::Parser::Multiplexer>.
236 =item * C<parser_class>
238 The name of the class to use to parse TAP. The default is
241 =item * C<scheduler_class>
243 The name of the class to use to schedule test execution. The default is
244 L<TAP::Parser::Scheduler>.
248 If set C<formatter> must be an object that is capable of formatting the
249 TAP output. See L<TAP::Formatter::Console> for an example.
253 If parse errors are found in the TAP output, a note of this will be
254 made in the summary report. To see all of the parse errors, set this
259 =item * C<directives>
261 If set to a true value, only test results with directives will be
262 displayed. This overrides other settings such as C<verbose> or
265 =item * C<ignore_exit>
267 If set to a true value instruct C<TAP::Parser> to ignore exit and wait
268 status from test scripts.
272 The maximum number of parallel tests to run at any time. Which tests
273 can be run in parallel is controlled by C<rules>. The default is to
274 run only one test at a time.
278 If true the harness will attempt to fork and run the parser for each
279 test in a separate process. Currently this option requires
280 L<Parallel::Iterator> to be installed.
284 A reference to a hash of rules that control which tests may be
285 executed in parallel. This is an experimental feature and the
286 interface may change.
290 { seq => '../ext/DB_File/t/*' },
291 { seq => '../ext/IO_Compress_Zlib/t/*' },
292 { seq => '../lib/CPANPLUS/*' },
293 { seq => '../lib/ExtUtils/t/*' },
301 A filehandle for catching standard output.
305 Any keys for which the value is C<undef> will be ignored.
309 # new supplied by TAP::Base
312 my @legal_callback = qw(
320 my %default_class = (
321 aggregator_class => 'TAP::Parser::Aggregator',
322 formatter_class => 'TAP::Formatter::Console',
323 multiplexer_class => 'TAP::Parser::Multiplexer',
324 parser_class => 'TAP::Parser',
325 scheduler_class => 'TAP::Parser::Scheduler',
329 my ( $self, $arg_for ) = @_;
332 $self->SUPER::_initialize( $arg_for, \@legal_callback );
333 my %arg_for = %$arg_for; # force a shallow copy
335 for my $name ( sort keys %VALIDATION_FOR ) {
336 my $property = delete $arg_for{$name};
337 if ( defined $property ) {
338 my $validate = $VALIDATION_FOR{$name};
340 my $value = $self->$validate($property);
341 if ( $self->_error ) {
344 $self->$name($value);
348 $self->jobs(1) unless defined $self->jobs;
350 while ( my ( $attr, $class ) = each %default_class ) {
351 $self->$attr( $self->$attr() || $class );
354 unless ( $self->formatter ) {
356 # This is a little bodge to preserve legacy behaviour. It's
357 # pretty horrible that we know which args are destined for
359 my %formatter_args = ( jobs => $self->jobs );
360 for my $name (@FORMATTER_ARGS) {
361 if ( defined( my $property = delete $arg_for{$name} ) ) {
362 $formatter_args{$name} = $property;
367 $self->_construct( $self->formatter_class, \%formatter_args )
371 if ( my @props = sort keys %arg_for ) {
372 $self->_croak("Unknown arguments to TAP::Harness::new (@props)");
379 ##############################################################################
381 =head2 Instance Methods
385 $harness->runtests(@tests);
387 Accepts and array of C<@tests> to be run. This should generally be the
388 names of test files, but this is not required. Each element in C<@tests>
389 will be passed to C<TAP::Parser::new()> as a C<source>. See
390 L<TAP::Parser> for more information.
392 It is possible to provide aliases that will be displayed in place of the
393 test name by supplying the test as a reference to an array containing
394 C<< [ $test, $alias ] >>:
396 $harness->runtests( [ 't/foo.t', 'Foo Once' ],
397 [ 't/foo.t', 'Foo Twice' ] );
399 Normally it is an error to attempt to run the same test twice. Aliases
400 allow you to overcome this limitation by giving each run of the test a
403 Tests will be run in the order found.
405 If the environment variable C<PERL_TEST_HARNESS_DUMP_TAP> is defined it
406 should name a directory into which a copy of the raw TAP for each test
407 will be written. TAP is written to files named for each test.
408 Subdirectories will be created as needed.
410 Returns a L<TAP::Parser::Aggregator> containing the test results.
415 my ( $self, @tests ) = @_;
417 my $aggregate = $self->_construct( $self->aggregator_class );
419 $self->_make_callback( 'before_runtests', $aggregate );
421 $self->aggregate_tests( $aggregate, @tests );
423 $self->summary($aggregate);
424 $self->_make_callback( 'after_runtests', $aggregate );
431 Output the summary for a TAP::Parser::Aggregator.
436 my ( $self, $aggregate ) = @_;
437 $self->formatter->summary($aggregate);
441 my ( $self, $aggregate, $job, $parser ) = @_;
443 $self->_make_callback( 'after_test', $job->as_array_ref, $parser );
444 $aggregate->add( $job->description, $parser );
447 sub _aggregate_forked {
448 my ( $self, $aggregate, $scheduler ) = @_;
450 eval { require Parallel::Iterator };
452 croak "Parallel::Iterator required for --fork option ($@)"
455 my $iter = Parallel::Iterator::iterate(
456 { workers => $self->jobs || 0 },
460 return if $job->is_spinner;
462 my ( $parser, $session ) = $self->make_parser($job);
464 while ( defined( my $result = $parser->next ) ) {
465 exit 1 if $result->is_bailout;
468 $self->finish_parser( $parser, $session );
470 # Can't serialise coderefs...
471 delete $parser->{_iter};
472 delete $parser->{_stream};
473 delete $parser->{_grammar};
476 sub { $scheduler->get_job }
479 while ( my ( $job, $parser ) = $iter->() ) {
480 next if $job->is_spinner;
481 $self->_after_test( $aggregate, $job, $parser );
488 sub _aggregate_parallel {
489 my ( $self, $aggregate, $scheduler ) = @_;
491 my $jobs = $self->jobs;
492 my $mux = $self->_construct( $self->multiplexer_class );
496 # Keep multiplexer topped up
498 while ( $mux->parsers < $jobs ) {
499 my $job = $scheduler->get_job;
501 # If we hit a spinner stop filling and start running.
502 last FILL if !defined $job || $job->is_spinner;
504 my ( $parser, $session ) = $self->make_parser($job);
505 $mux->add( $parser, [ $session, $job ] );
508 if ( my ( $parser, $stash, $result ) = $mux->next ) {
509 my ( $session, $job ) = @$stash;
510 if ( defined $result ) {
511 $session->result($result);
512 exit 1 if $result->is_bailout;
516 # End of parser. Automatically removed from the mux.
517 $self->finish_parser( $parser, $session );
518 $self->_after_test( $aggregate, $job, $parser );
528 sub _aggregate_single {
529 my ( $self, $aggregate, $scheduler ) = @_;
532 while ( my $job = $scheduler->get_job ) {
533 next JOB if $job->is_spinner;
535 my ( $parser, $session ) = $self->make_parser($job);
537 while ( defined( my $result = $parser->next ) ) {
538 $session->result($result);
539 if ( $result->is_bailout ) {
541 # Keep reading until input is exhausted in the hope
542 # of allowing any pending diagnostics to show up.
543 1 while $parser->next;
548 $self->finish_parser( $parser, $session );
549 $self->_after_test( $aggregate, $job, $parser );
556 =head3 C<aggregate_tests>
558 $harness->aggregate_tests( $aggregate, @tests );
560 Run the named tests and display a summary of result. Tests will be run
563 Test results will be added to the supplied L<TAP::Parser::Aggregator>.
564 C<aggregate_tests> may be called multiple times to run several sets of
565 tests. Multiple C<Test::Harness> instances may be used to pass results
566 to a single aggregator so that different parts of a complex test suite
567 may be run using different C<TAP::Harness> settings. This is useful, for
568 example, in the case where some tests should run in parallel but others
569 are unsuitable for parallel execution.
571 my $formatter = TAP::Formatter::Console->new;
572 my $ser_harness = TAP::Harness->new( { formatter => $formatter } );
573 my $par_harness = TAP::Harness->new(
574 { formatter => $formatter,
578 my $aggregator = TAP::Parser::Aggregator->new;
580 $aggregator->start();
581 $ser_harness->aggregate_tests( $aggregator, @ser_tests );
582 $par_harness->aggregate_tests( $aggregator, @par_tests );
584 $formatter->summary($aggregator);
586 Note that for simpler testing requirements it will often be possible to
587 replace the above code with a single call to C<runtests>.
589 Each elements of the @tests array is either
593 =item * the file name of a test script to run
595 =item * a reference to a [ file name, display name ] array
599 When you supply a separate display name it becomes possible to run a
600 test more than once; the display name is effectively the alias by which
601 the test is known inside the harness. The harness doesn't care if it
602 runs the same script more than once when each invocation uses a
607 sub aggregate_tests {
608 my ( $self, $aggregate, @tests ) = @_;
610 my $jobs = $self->jobs;
611 my $scheduler = $self->make_scheduler(@tests);
614 local $ENV{HARNESS_IS_VERBOSE} = 1
615 if $self->formatter->verbosity > 0;
617 # Formatter gets only names.
618 $self->formatter->prepare( map { $_->description } $scheduler->get_all );
620 if ( $self->jobs > 1 ) {
622 $self->_aggregate_forked( $aggregate, $scheduler );
625 $self->_aggregate_parallel( $aggregate, $scheduler );
629 $self->_aggregate_single( $aggregate, $scheduler );
635 sub _add_descriptions {
638 # First transformation: turn scalars into single element arrays
639 my @tests = map { 'ARRAY' eq ref $_ ? $_ : [$_] } @_;
641 # Work out how many different extensions we have
643 for my $test (@tests) {
644 $ext{$1}++ if $test->[0] =~ /\.(\w+)$/;
647 for my $test (@tests) {
649 $test->[1] = $test->[0];
650 $test->[1] =~ s/\.\w+$//
657 =head3 C<make_scheduler>
659 Called by the harness when it needs to create a
660 L<TAP::Parser::Scheduler>. Override in a subclass to provide an
661 alternative scheduler. C<make_scheduler> is passed the list of tests
662 that was passed to C<aggregate_tests>.
667 my ( $self, @tests ) = @_;
668 return $self->_construct(
669 $self->scheduler_class,
670 tests => [ $self->_add_descriptions(@tests) ],
671 rules => $self->rules
677 Gets or sets the number of concurrent test runs the harness is handling.
678 For the default harness this value is always 1. A parallel harness such
679 as L<TAP::Harness::Parallel> will override this to return the number of
684 If true the harness will attempt to fork and run the parser for each
685 test in a separate process. Currently this option requires
686 L<Parallel::Iterator> to be installed.
690 ##############################################################################
694 C<TAP::Harness> is designed to be (mostly) easy to subclass. If you
695 don't like how a particular feature functions, just override the
700 TODO: This is out of date
702 The following methods are ones you may wish to override if you want to
703 subclass C<TAP::Harness>.
707 $harness->summary( \%args );
709 C<summary> prints the summary report after all tests are run. The
710 argument is a hashref with the following keys:
716 This is created with C<< Benchmark->new >> and it the time the tests
717 started. You can print a useful summary time, if desired, with:
720 timestr( timediff( Benchmark->new, $start_time ), 'nop' ) );
724 This is an array reference of all test names. To get the L<TAP::Parser>
725 object for individual tests:
727 my $aggregate = $args->{aggregate};
728 my $tests = $args->{tests};
730 for my $name ( @$tests ) {
731 my ($parser) = $aggregate->parsers($test);
732 ... do something with $parser
735 This is a bit clunky and will be cleaned up in a later release.
741 sub _get_parser_args {
742 my ( $self, $job ) = @_;
743 my $test_prog = $job->filename;
746 @switches = $self->lib if $self->lib;
747 push @switches => $self->switches if $self->switches;
748 $args{switches} = \@switches;
749 $args{spool} = $self->_open_spool($test_prog);
750 $args{merge} = $self->merge;
751 $args{ignore_exit} = $self->ignore_exit;
753 if ( my $exec = $self->exec ) {
755 = ref $exec eq 'CODE'
756 ? $exec->( $self, $test_prog )
757 : [ @$exec, $test_prog ];
758 $args{source} = $test_prog unless $args{exec};
761 $args{source} = $test_prog;
764 if ( defined( my $test_args = $self->test_args ) ) {
765 $args{test_args} = $test_args;
771 =head3 C<make_parser>
773 Make a new parser and display formatter session. Typically used and/or
774 overridden in subclasses.
776 my ( $parser, $session ) = $harness->make_parser;
781 my ( $self, $job ) = @_;
783 my $args = $self->_get_parser_args($job);
784 $self->_make_callback( 'parser_args', $args, $job->as_array_ref );
785 my $parser = $self->_construct( $self->parser_class, $args );
787 $self->_make_callback( 'made_parser', $parser, $job->as_array_ref );
788 my $session = $self->formatter->open_test( $job->description, $parser );
790 return ( $parser, $session );
793 =head3 C<finish_parser>
795 Terminate use of a parser. Typically used and/or overridden in
796 subclasses. The parser isn't destroyed as a result of this.
801 my ( $self, $parser, $session ) = @_;
803 $session->close_test;
804 $self->_close_spool($parser);
813 if ( my $spool_dir = $ENV{PERL_TEST_HARNESS_DUMP_TAP} ) {
815 my $spool = File::Spec->catfile( $spool_dir, $test );
818 my ( $vol, $dir, undef ) = File::Spec->splitpath($spool);
819 my $path = File::Spec->catpath( $vol, $dir, '' );
820 eval { mkpath($path) };
821 $self->_croak($@) if $@;
823 my $spool_handle = IO::Handle->new;
824 open( $spool_handle, ">$spool" )
825 or $self->_croak(" Can't write $spool ( $! ) ");
827 return $spool_handle;
837 if ( my $spool_handle = $parser->delete_spool ) {
839 or $self->_croak(" Error closing TAP spool file( $! ) \n ");
846 my ( $self, $message ) = @_;
848 $message = $self->_error;
850 $self->SUPER::_croak($message);
857 If you like the C<prove> utility and L<TAP::Parser> but you want your
858 own harness, all you need to do is write one and provide C<new> and
859 C<runtests> methods. Then you can use the C<prove> utility like so:
861 prove --harness My::Test::Harness
863 Note that while C<prove> accepts a list of tests (or things to be
864 tested), C<new> has a fairly rich set of arguments. You'll probably want
865 to read over this code carefully to see how all of them are being used.
875 # vim:ts=4:sw=4:et:sta