[p5sagit/p5-mst-13.2.git] / ext / Test / Harness / t / compat / test-harness-compat.t

#!/usr/bin/perl -w

BEGIN {
    if ( $ENV{PERL_CORE} ) {
        chdir 't';
        @INC = '../lib';
    }
    else {
        unshift @INC, 't/lib';
    }
}

use strict;

# use lib 't/lib';

use Test::More;
use File::Spec;
use Test::Harness qw(execute_tests);

# unset this global when self-testing ('testcover' and etc issue)
local $ENV{HARNESS_PERL_SWITCHES};

my $TEST_DIR
  = $ENV{PERL_CORE} ? '../ext/Test/Harness/t/sample-tests' : 't/sample-tests';

{

    # if the harness wants to save the resulting TAP we shouldn't
    # do it for our internal calls
    local $ENV{PERL_TEST_HARNESS_DUMP_TAP} = 0;

    my $PER_LOOP = 4;

    my $results = {
        'descriptive' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 5,
                'ok'          => 5,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        join(
            ',', qw(
              descriptive die die_head_end die_last_minute duplicates
              head_end head_fail inc_taint junk_before_plan lone_not_bug
              no_nums no_output schwern sequence_misparse shbang_misparse
              simple simple_fail skip skip_nomsg skipall skipall_nomsg
              stdout_stderr taint todo_inline
              todo_misparse too_many vms_nit
              )
          ) => {
            'failed' => {
                "$TEST_DIR/die" => {
                    'canon'  => '??',
                    'estat'  => 1,
                    'failed' => '??',
                    'max'    => '??',
                    'name'   => "$TEST_DIR/die",
                    'wstat'  => '256'
                },
                "$TEST_DIR/die_head_end" => {
                    'canon'  => '??',
                    'estat'  => 1,
                    'failed' => '??',
                    'max'    => '??',
                    'name'   => "$TEST_DIR/die_head_end",
                    'wstat'  => '256'
                },
                "$TEST_DIR/die_last_minute" => {
                    'canon'  => '??',
                    'estat'  => 1,
                    'failed' => 0,
                    'max'    => 4,
                    'name'   => "$TEST_DIR/die_last_minute",
                    'wstat'  => '256'
                },
                "$TEST_DIR/duplicates" => {
                    'canon'  => '??',
                    'estat'  => '',
                    'failed' => '??',
                    'max'    => 10,
                    'name'   => "$TEST_DIR/duplicates",
                    'wstat'  => ''
                },
                "$TEST_DIR/head_fail" => {
                    'canon'  => 2,
                    'estat'  => '',
                    'failed' => 1,
                    'max'    => 4,
                    'name'   => "$TEST_DIR/head_fail",
                    'wstat'  => ''
                },
                "$TEST_DIR/inc_taint" => {
                    'canon'  => 1,
                    'estat'  => 1,
                    'failed' => 1,
                    'max'    => 1,
                    'name'   => "$TEST_DIR/inc_taint",
                    'wstat'  => '256'
                },
                "$TEST_DIR/no_nums" => {
                    'canon'  => 3,
                    'estat'  => '',
                    'failed' => 1,
                    'max'    => 5,
                    'name'   => "$TEST_DIR/no_nums",
                    'wstat'  => ''
                },
                "$TEST_DIR/no_output" => {
                    'canon'  => '??',
                    'estat'  => '',
                    'failed' => '??',
                    'max'    => '??',
                    'name'   => "$TEST_DIR/no_output",
                    'wstat'  => ''
                },
                "$TEST_DIR/simple_fail" => {
                    'canon'  => '2 5',
                    'estat'  => '',
                    'failed' => 2,
                    'max'    => 5,
                    'name'   => "$TEST_DIR/simple_fail",
                    'wstat'  => ''
                },
                "$TEST_DIR/todo_misparse" => {
                    'canon'  => 1,
                    'estat'  => '',
                    'failed' => 1,
                    'max'    => 1,
                    'name'   => "$TEST_DIR/todo_misparse",
                    'wstat'  => ''
                },
                "$TEST_DIR/too_many" => {
                    'canon'  => '4-7',
                    'estat'  => 4,
                    'failed' => 4,
                    'max'    => 3,
                    'name'   => "$TEST_DIR/too_many",
                    'wstat'  => '1024'
                },
                "$TEST_DIR/vms_nit" => {
                    'canon'  => 1,
                    'estat'  => '',
                    'failed' => 1,
                    'max'    => 2,
                    'name'   => "$TEST_DIR/vms_nit",
                    'wstat'  => ''
                }
            },
            'todo' => {
                "$TEST_DIR/todo_inline" => {
                    'canon'  => 2,
                    'estat'  => '',
                    'failed' => 1,
                    'max'    => 2,
                    'name'   => "$TEST_DIR/todo_inline",
                    'wstat'  => ''
                }
            },
            'totals' => {
                'bad'         => 12,
                'bonus'       => 1,
                'files'       => 27,
                'good'        => 15,
                'max'         => 76,
                'ok'          => 78,
                'skipped'     => 2,
                'sub_skipped' => 2,
                'tests'       => 27,
                'todo'        => 2
            }
          },
        'die' => {
            'failed' => {
                "$TEST_DIR/die" => {
                    'canon'  => '??',
                    'estat'  => 1,
                    'failed' => '??',
                    'max'    => '??',
                    'name'   => "$TEST_DIR/die",
                    'wstat'  => '256'
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 0,
                'ok'          => 0,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'die_head_end' => {
            'failed' => {
                "$TEST_DIR/die_head_end" => {
                    'canon'  => '??',
                    'estat'  => 1,
                    'failed' => '??',
                    'max'    => '??',
                    'name'   => "$TEST_DIR/die_head_end",
                    'wstat'  => '256'
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 0,
                'ok'          => 4,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'die_last_minute' => {
            'failed' => {
                "$TEST_DIR/die_last_minute" => {
                    'canon'  => '??',
                    'estat'  => 1,
                    'failed' => 0,
                    'max'    => 4,
                    'name'   => "$TEST_DIR/die_last_minute",
                    'wstat'  => '256'
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 4,
                'ok'          => 4,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'duplicates' => {
            'failed' => {
                "$TEST_DIR/duplicates" => {
                    'canon'  => '??',
                    'estat'  => '',
                    'failed' => '??',
                    'max'    => 10,
                    'name'   => "$TEST_DIR/duplicates",
                    'wstat'  => ''
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 10,
                'ok'          => 11,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'head_end' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 4,
                'ok'          => 4,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'head_fail' => {
            'failed' => {
                "$TEST_DIR/head_fail" => {
                    'canon'  => 2,
                    'estat'  => '',
                    'failed' => 1,
                    'max'    => 4,
                    'name'   => "$TEST_DIR/head_fail",
                    'wstat'  => ''
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 4,
                'ok'          => 3,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'inc_taint' => {
            'failed' => {
                "$TEST_DIR/inc_taint" => {
                    'canon'  => 1,
                    'estat'  => 1,
                    'failed' => 1,
                    'max'    => 1,
                    'name'   => "$TEST_DIR/inc_taint",
                    'wstat'  => '256'
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 1,
                'ok'          => 0,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'junk_before_plan' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 1,
                'ok'          => 1,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'lone_not_bug' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 4,
                'ok'          => 4,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'no_nums' => {
            'failed' => {
                "$TEST_DIR/no_nums" => {
                    'canon'  => 3,
                    'estat'  => '',
                    'failed' => 1,
                    'max'    => 5,
                    'name'   => "$TEST_DIR/no_nums",
                    'wstat'  => ''
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 5,
                'ok'          => 4,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'no_output' => {
            'failed' => {
                "$TEST_DIR/no_output" => {
                    'canon'  => '??',
                    'estat'  => '',
                    'failed' => '??',
                    'max'    => '??',
                    'name'   => "$TEST_DIR/no_output",
                    'wstat'  => ''
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 0,
                'ok'          => 0,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'schwern' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 1,
                'ok'          => 1,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'sequence_misparse' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 5,
                'ok'          => 5,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'shbang_misparse' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 2,
                'ok'          => 2,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'simple' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 5,
                'ok'          => 5,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'simple_fail' => {
            'failed' => {
                "$TEST_DIR/simple_fail" => {
                    'canon'  => '2 5',
                    'estat'  => '',
                    'failed' => 2,
                    'max'    => 5,
                    'name'   => "$TEST_DIR/simple_fail",
                    'wstat'  => ''
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 5,
                'ok'          => 3,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'skip' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 5,
                'ok'          => 5,
                'skipped'     => 0,
                'sub_skipped' => 1,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'skip_nomsg' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 1,
                'ok'          => 1,
                'skipped'     => 0,
                'sub_skipped' => 1,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'skipall' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 0,
                'ok'          => 0,
                'skipped'     => 1,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'skipall_nomsg' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 0,
                'ok'          => 0,
                'skipped'     => 1,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'stdout_stderr' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 4,
                'ok'          => 4,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'switches' => {
            'skip_if' => sub {
                ( $ENV{PERL5OPT} || '' ) =~ m{(?:^|\s)-[dM]};
            },
            'failed' => {
                "$TEST_DIR/switches" => {
                    'canon'  => 1,
                    'estat'  => '',
                    'failed' => 1,
                    'max'    => 1,
                    'name'   => "$TEST_DIR/switches",
                    'wstat'  => ''
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 1,
                'ok'          => 0,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'taint' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 1,
                'ok'          => 1,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'taint_warn' => {
            'failed' => {},
            'todo'   => {},
            'totals' => {
                'bad'         => 0,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 1,
                'max'         => 1,
                'ok'          => 1,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            },
            'require' => 5.008001,
        },
        'todo_inline' => {
            'failed' => {},
            'todo'   => {
                "$TEST_DIR/todo_inline" => {
                    'canon'  => 2,
                    'estat'  => '',
                    'failed' => 1,
                    'max'    => 2,
                    'name'   => "$TEST_DIR/todo_inline",
                    'wstat'  => ''
                }
            },
            'totals' => {
                'bad'         => 0,
                'bonus'       => 1,
                'files'       => 1,
                'good'        => 1,
                'max'         => 3,
                'ok'          => 3,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 2
            }
        },
        'todo_misparse' => {
            'failed' => {
                "$TEST_DIR/todo_misparse" => {
                    'canon'  => 1,
                    'estat'  => '',
                    'failed' => 1,
                    'max'    => 1,
                    'name'   => "$TEST_DIR/todo_misparse",
                    'wstat'  => ''
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 1,
                'ok'          => 0,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'too_many' => {
            'failed' => {
                "$TEST_DIR/too_many" => {
                    'canon'  => '4-7',
                    'estat'  => 4,
                    'failed' => 4,
                    'max'    => 3,
                    'name'   => "$TEST_DIR/too_many",
                    'wstat'  => '1024'
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 3,
                'ok'          => 7,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        },
        'vms_nit' => {
            'failed' => {
                "$TEST_DIR/vms_nit" => {
                    'canon'  => 1,
                    'estat'  => '',
                    'failed' => 1,
                    'max'    => 2,
                    'name'   => "$TEST_DIR/vms_nit",
                    'wstat'  => ''
                }
            },
            'todo'   => {},
            'totals' => {
                'bad'         => 1,
                'bonus'       => 0,
                'files'       => 1,
                'good'        => 0,
                'max'         => 2,
                'ok'          => 1,
                'skipped'     => 0,
                'sub_skipped' => 0,
                'tests'       => 1,
                'todo'        => 0
            }
        }
    };

    my $num_tests = ( keys %$results ) * $PER_LOOP;

    plan tests => $num_tests;

    sub local_name {
        my $name = shift;
        return File::Spec->catfile( split /\//, $name );
    }

    sub local_result {
        my $hash = shift;
        my $new  = {};

        while ( my ( $file, $want ) = each %$hash ) {
            if ( exists $want->{name} ) {
                $want->{name} = local_name( $want->{name} );
            }
            $new->{ local_name($file) } = $want;
        }
        return $new;
    }

    sub vague_status {
        my $hash = shift;
        return $hash unless $^O eq 'VMS';

        while ( my ( $file, $want ) = each %$hash ) {
            for (qw( estat wstat )) {
                if ( exists $want->{$_} ) {
                    $want->{$_} = $want->{$_} ? 1 : 0;
                }
            }
        }
        return $hash;
    }

    {
        local $^W = 0;

        # Silence harness output
        *TAP::Formatter::Console::_output = sub {

            # do nothing
        };
    }

    for my $test_key ( sort keys %$results ) {
        my $result = $results->{$test_key};
        SKIP: {
            if ( $result->{require} && $] < $result->{require} ) {
                skip "Test requires Perl $result->{require}, we have $]", 4;
            }

            if ( my $skip_if = $result->{skip_if} ) {
                skip
                  "Test '$test_key' can't run properly in this environment", 4
                  if $skip_if->();
            }

            my @test_names = split( /,/, $test_key );
            my @test_files
              = map { File::Spec->catfile( $TEST_DIR, $_ ) } @test_names;

            # For now we supress STDERR because it crufts up /our/ test
            # results. Should probably capture and analyse it.
            local ( *OLDERR, *OLDOUT );
            open OLDERR, '>&STDERR' or die $!;
            open OLDOUT, '>&STDOUT' or die $!;
            my $devnull = File::Spec->devnull;
            open STDERR, ">$devnull" or die $!;
            open STDOUT, ">$devnull" or die $!;

            my ( $tot, $fail, $todo, $harness, $aggregate )
              = execute_tests( tests => \@test_files );

            open STDERR, '>&OLDERR' or die $!;
            open STDOUT, '>&OLDOUT' or die $!;

            my $bench = delete $tot->{bench};
            isa_ok $bench, 'Benchmark';

            # Localise filenames in failed, todo
            my $lfailed = vague_status( local_result( $result->{failed} ) );
            my $ltodo   = vague_status( local_result( $result->{todo} ) );

            # use Data::Dumper;
            # diag Dumper( [ $lfailed, $ltodo ] );

            is_deeply $tot, $result->{totals}, "totals match for $test_key";
            is_deeply vague_status($fail), $lfailed,
              "failure summary matches for $test_key";
            is_deeply vague_status($todo), $ltodo,
              "todo summary matches for $test_key";
        }
    }
}
Commit	Line	Data
b965d173	1	#!/usr/bin/perl -w
	2
	3	BEGIN {
27fc0087	4	if ( $ENV{PERL_CORE} ) {
	5	chdir 't';
	6	@INC = '../lib';
	7	}
	8	else {
	9	unshift @INC, 't/lib';
b965d173	10	}
	11	}
	12
	13	use strict;
f7c69158	14
27fc0087	15	# use lib 't/lib';
b965d173	16
b965d173	17	use Test::More;
b965d173	18	use File::Spec;
b965d173	19	use Test::Harness qw(execute_tests);
	20
	21	# unset this global when self-testing ('testcover' and etc issue)
	22	local $ENV{HARNESS_PERL_SWITCHES};
	23
27fc0087	24	my $TEST_DIR
	25	= $ENV{PERL_CORE} ? '../ext/Test/Harness/t/sample-tests' : 't/sample-tests';
	26
b965d173	27	{
	28
	29	# if the harness wants to save the resulting TAP we shouldn't
	30	# do it for our internal calls
	31	local $ENV{PERL_TEST_HARNESS_DUMP_TAP} = 0;
	32
b965d173	33	my $PER_LOOP = 4;
	34
	35	my $results = {
	36	'descriptive' => {
	37	'failed' => {},
	38	'todo' => {},
	39	'totals' => {
	40	'bad' => 0,
	41	'bonus' => 0,
	42	'files' => 1,
	43	'good' => 1,
	44	'max' => 5,
	45	'ok' => 5,
	46	'skipped' => 0,
	47	'sub_skipped' => 0,
	48	'tests' => 1,
	49	'todo' => 0
	50	}
	51	},
	52	join(
	53	',', qw(
	54	descriptive die die_head_end die_last_minute duplicates
	55	head_end head_fail inc_taint junk_before_plan lone_not_bug
	56	no_nums no_output schwern sequence_misparse shbang_misparse
	57	simple simple_fail skip skip_nomsg skipall skipall_nomsg
f7c69158	58	stdout_stderr taint todo_inline
b965d173	59	todo_misparse too_many vms_nit
	60	)
	61	) => {
	62	'failed' => {
27fc0087	63	"$TEST_DIR/die" => {
b965d173	64	'canon' => '??',
	65	'estat' => 1,
	66	'failed' => '??',
	67	'max' => '??',
27fc0087	68	'name' => "$TEST_DIR/die",
b965d173	69	'wstat' => '256'
b965d173	70	},
27fc0087	71	"$TEST_DIR/die_head_end" => {
b965d173	72	'canon' => '??',
	73	'estat' => 1,
	74	'failed' => '??',
	75	'max' => '??',
27fc0087	76	'name' => "$TEST_DIR/die_head_end",
b965d173	77	'wstat' => '256'
b965d173	78	},
27fc0087	79	"$TEST_DIR/die_last_minute" => {
b965d173	80	'canon' => '??',
	81	'estat' => 1,
	82	'failed' => 0,
	83	'max' => 4,
27fc0087	84	'name' => "$TEST_DIR/die_last_minute",
b965d173	85	'wstat' => '256'
b965d173	86	},
27fc0087	87	"$TEST_DIR/duplicates" => {
b965d173	88	'canon' => '??',
	89	'estat' => '',
	90	'failed' => '??',
	91	'max' => 10,
27fc0087	92	'name' => "$TEST_DIR/duplicates",
b965d173	93	'wstat' => ''
b965d173	94	},
27fc0087	95	"$TEST_DIR/head_fail" => {
b965d173	96	'canon' => 2,
	97	'estat' => '',
	98	'failed' => 1,
	99	'max' => 4,
27fc0087	100	'name' => "$TEST_DIR/head_fail",
b965d173	101	'wstat' => ''
b965d173	102	},
27fc0087	103	"$TEST_DIR/inc_taint" => {
b965d173	104	'canon' => 1,
	105	'estat' => 1,
	106	'failed' => 1,
	107	'max' => 1,
27fc0087	108	'name' => "$TEST_DIR/inc_taint",
b965d173	109	'wstat' => '256'
b965d173	110	},
27fc0087	111	"$TEST_DIR/no_nums" => {
b965d173	112	'canon' => 3,
	113	'estat' => '',
	114	'failed' => 1,
	115	'max' => 5,
27fc0087	116	'name' => "$TEST_DIR/no_nums",
b965d173	117	'wstat' => ''
b965d173	118	},
27fc0087	119	"$TEST_DIR/no_output" => {
b965d173	120	'canon' => '??',
	121	'estat' => '',
	122	'failed' => '??',
	123	'max' => '??',
27fc0087	124	'name' => "$TEST_DIR/no_output",
b965d173	125	'wstat' => ''
b965d173	126	},
27fc0087	127	"$TEST_DIR/simple_fail" => {
b965d173	128	'canon' => '2 5',
	129	'estat' => '',
	130	'failed' => 2,
	131	'max' => 5,
27fc0087	132	'name' => "$TEST_DIR/simple_fail",
b965d173	133	'wstat' => ''
b965d173	134	},
27fc0087	135	"$TEST_DIR/todo_misparse" => {
b965d173	136	'canon' => 1,
	137	'estat' => '',
	138	'failed' => 1,
	139	'max' => 1,
27fc0087	140	'name' => "$TEST_DIR/todo_misparse",
b965d173	141	'wstat' => ''
b965d173	142	},
27fc0087	143	"$TEST_DIR/too_many" => {
b965d173	144	'canon' => '4-7',
	145	'estat' => 4,
	146	'failed' => 4,
	147	'max' => 3,
27fc0087	148	'name' => "$TEST_DIR/too_many",
b965d173	149	'wstat' => '1024'
b965d173	150	},
27fc0087	151	"$TEST_DIR/vms_nit" => {
b965d173	152	'canon' => 1,
	153	'estat' => '',
	154	'failed' => 1,
	155	'max' => 2,
27fc0087	156	'name' => "$TEST_DIR/vms_nit",
b965d173	157	'wstat' => ''
	158	}
	159	},
	160	'todo' => {
27fc0087	161	"$TEST_DIR/todo_inline" => {
b965d173	162	'canon' => 2,
	163	'estat' => '',
	164	'failed' => 1,
	165	'max' => 2,
27fc0087	166	'name' => "$TEST_DIR/todo_inline",
b965d173	167	'wstat' => ''
	168	}
	169	},
	170	'totals' => {
f7c69158	171	'bad' => 12,
b965d173	172	'bonus' => 1,
f7c69158	173	'files' => 27,
b965d173	174	'good' => 15,
f7c69158	175	'max' => 76,
b965d173	176	'ok' => 78,
	177	'skipped' => 2,
	178	'sub_skipped' => 2,
f7c69158	179	'tests' => 27,
b965d173	180	'todo' => 2
	181	}
	182	},
	183	'die' => {
	184	'failed' => {
27fc0087	185	"$TEST_DIR/die" => {
b965d173	186	'canon' => '??',
	187	'estat' => 1,
	188	'failed' => '??',
	189	'max' => '??',
27fc0087	190	'name' => "$TEST_DIR/die",
b965d173	191	'wstat' => '256'
	192	}
	193	},
	194	'todo' => {},
	195	'totals' => {
	196	'bad' => 1,
	197	'bonus' => 0,
	198	'files' => 1,
	199	'good' => 0,
	200	'max' => 0,
	201	'ok' => 0,
	202	'skipped' => 0,
	203	'sub_skipped' => 0,
	204	'tests' => 1,
	205	'todo' => 0
	206	}
	207	},
	208	'die_head_end' => {
	209	'failed' => {
27fc0087	210	"$TEST_DIR/die_head_end" => {
b965d173	211	'canon' => '??',
	212	'estat' => 1,
	213	'failed' => '??',
	214	'max' => '??',
27fc0087	215	'name' => "$TEST_DIR/die_head_end",
b965d173	216	'wstat' => '256'
	217	}
	218	},
	219	'todo' => {},
	220	'totals' => {
	221	'bad' => 1,
	222	'bonus' => 0,
	223	'files' => 1,
	224	'good' => 0,
	225	'max' => 0,
	226	'ok' => 4,
	227	'skipped' => 0,
	228	'sub_skipped' => 0,
	229	'tests' => 1,
	230	'todo' => 0
	231	}
	232	},
	233	'die_last_minute' => {
	234	'failed' => {
27fc0087	235	"$TEST_DIR/die_last_minute" => {
b965d173	236	'canon' => '??',
	237	'estat' => 1,
	238	'failed' => 0,
	239	'max' => 4,
27fc0087	240	'name' => "$TEST_DIR/die_last_minute",
b965d173	241	'wstat' => '256'
	242	}
	243	},
	244	'todo' => {},
	245	'totals' => {
	246	'bad' => 1,
	247	'bonus' => 0,
	248	'files' => 1,
	249	'good' => 0,
	250	'max' => 4,
	251	'ok' => 4,
	252	'skipped' => 0,
	253	'sub_skipped' => 0,
	254	'tests' => 1,
	255	'todo' => 0
	256	}
	257	},
	258	'duplicates' => {
	259	'failed' => {
27fc0087	260	"$TEST_DIR/duplicates" => {
b965d173	261	'canon' => '??',
	262	'estat' => '',
	263	'failed' => '??',
	264	'max' => 10,
27fc0087	265	'name' => "$TEST_DIR/duplicates",
b965d173	266	'wstat' => ''
	267	}
	268	},
	269	'todo' => {},
	270	'totals' => {
	271	'bad' => 1,
	272	'bonus' => 0,
	273	'files' => 1,
	274	'good' => 0,
	275	'max' => 10,
	276	'ok' => 11,
	277	'skipped' => 0,
	278	'sub_skipped' => 0,
	279	'tests' => 1,
	280	'todo' => 0
	281	}
	282	},
	283	'head_end' => {
	284	'failed' => {},
	285	'todo' => {},
	286	'totals' => {
	287	'bad' => 0,
	288	'bonus' => 0,
	289	'files' => 1,
	290	'good' => 1,
	291	'max' => 4,
	292	'ok' => 4,
	293	'skipped' => 0,
	294	'sub_skipped' => 0,
	295	'tests' => 1,
	296	'todo' => 0
	297	}
	298	},
	299	'head_fail' => {
	300	'failed' => {
27fc0087	301	"$TEST_DIR/head_fail" => {
b965d173	302	'canon' => 2,
	303	'estat' => '',
	304	'failed' => 1,
	305	'max' => 4,
27fc0087	306	'name' => "$TEST_DIR/head_fail",
b965d173	307	'wstat' => ''
	308	}
	309	},
	310	'todo' => {},
	311	'totals' => {
	312	'bad' => 1,
	313	'bonus' => 0,
	314	'files' => 1,
	315	'good' => 0,
	316	'max' => 4,
	317	'ok' => 3,
	318	'skipped' => 0,
	319	'sub_skipped' => 0,
	320	'tests' => 1,
	321	'todo' => 0
	322	}
	323	},
	324	'inc_taint' => {
	325	'failed' => {
27fc0087	326	"$TEST_DIR/inc_taint" => {
b965d173	327	'canon' => 1,
	328	'estat' => 1,
	329	'failed' => 1,
	330	'max' => 1,
27fc0087	331	'name' => "$TEST_DIR/inc_taint",
b965d173	332	'wstat' => '256'
	333	}
	334	},
	335	'todo' => {},
	336	'totals' => {
	337	'bad' => 1,
	338	'bonus' => 0,
	339	'files' => 1,
	340	'good' => 0,
	341	'max' => 1,
	342	'ok' => 0,
	343	'skipped' => 0,
	344	'sub_skipped' => 0,
	345	'tests' => 1,
	346	'todo' => 0
	347	}
	348	},
	349	'junk_before_plan' => {
	350	'failed' => {},
	351	'todo' => {},
	352	'totals' => {
	353	'bad' => 0,
	354	'bonus' => 0,
	355	'files' => 1,
	356	'good' => 1,
	357	'max' => 1,
	358	'ok' => 1,
	359	'skipped' => 0,
	360	'sub_skipped' => 0,
	361	'tests' => 1,
	362	'todo' => 0
	363	}
	364	},
	365	'lone_not_bug' => {
	366	'failed' => {},
	367	'todo' => {},
	368	'totals' => {
	369	'bad' => 0,
	370	'bonus' => 0,
	371	'files' => 1,
	372	'good' => 1,
	373	'max' => 4,
	374	'ok' => 4,
	375	'skipped' => 0,
	376	'sub_skipped' => 0,
	377	'tests' => 1,
	378	'todo' => 0
	379	}
	380	},
	381	'no_nums' => {
	382	'failed' => {
27fc0087	383	"$TEST_DIR/no_nums" => {
b965d173	384	'canon' => 3,
	385	'estat' => '',
	386	'failed' => 1,
	387	'max' => 5,
27fc0087	388	'name' => "$TEST_DIR/no_nums",
b965d173	389	'wstat' => ''
	390	}
	391	},
	392	'todo' => {},
	393	'totals' => {
	394	'bad' => 1,
	395	'bonus' => 0,
	396	'files' => 1,
	397	'good' => 0,
	398	'max' => 5,
	399	'ok' => 4,
	400	'skipped' => 0,
	401	'sub_skipped' => 0,
	402	'tests' => 1,
	403	'todo' => 0
	404	}
	405	},
	406	'no_output' => {
	407	'failed' => {
27fc0087	408	"$TEST_DIR/no_output" => {
b965d173	409	'canon' => '??',
	410	'estat' => '',
	411	'failed' => '??',
	412	'max' => '??',
27fc0087	413	'name' => "$TEST_DIR/no_output",
b965d173	414	'wstat' => ''
	415	}
	416	},
	417	'todo' => {},
	418	'totals' => {
	419	'bad' => 1,
	420	'bonus' => 0,
	421	'files' => 1,
	422	'good' => 0,
	423	'max' => 0,
	424	'ok' => 0,
	425	'skipped' => 0,
	426	'sub_skipped' => 0,
	427	'tests' => 1,
	428	'todo' => 0
	429	}
	430	},
	431	'schwern' => {
	432	'failed' => {},
	433	'todo' => {},
	434	'totals' => {
	435	'bad' => 0,
	436	'bonus' => 0,
	437	'files' => 1,
	438	'good' => 1,
	439	'max' => 1,
	440	'ok' => 1,
	441	'skipped' => 0,
	442	'sub_skipped' => 0,
	443	'tests' => 1,
	444	'todo' => 0
	445	}
	446	},
	447	'sequence_misparse' => {
	448	'failed' => {},
	449	'todo' => {},
	450	'totals' => {
	451	'bad' => 0,
	452	'bonus' => 0,
	453	'files' => 1,
	454	'good' => 1,
	455	'max' => 5,
	456	'ok' => 5,
	457	'skipped' => 0,
	458	'sub_skipped' => 0,
	459	'tests' => 1,
	460	'todo' => 0
	461	}
	462	},
	463	'shbang_misparse' => {
	464	'failed' => {},
	465	'todo' => {},
	466	'totals' => {
	467	'bad' => 0,
	468	'bonus' => 0,
	469	'files' => 1,
	470	'good' => 1,
	471	'max' => 2,
	472	'ok' => 2,
	473	'skipped' => 0,
	474	'sub_skipped' => 0,
	475	'tests' => 1,
	476	'todo' => 0
	477	}
478	},
479	'simple' => {
480	'failed' => {},
481	'todo' => {},
482	'totals' => {
483	'bad' => 0,
484	'bonus' => 0,
485	'files' => 1,
486	'good' => 1,
487	'max' => 5,
488	'ok' => 5,
489	'skipped' => 0,
490	'sub_skipped' => 0,
491	'tests' => 1,
492	'todo' => 0
493	}
494	},
495	'simple_fail' => {
496	'failed' => {
27fc0087	497	"$TEST_DIR/simple_fail" => {
b965d173	498	'canon' => '2 5',
	499	'estat' => '',
	500	'failed' => 2,
	501	'max' => 5,
27fc0087	502	'name' => "$TEST_DIR/simple_fail",
b965d173	503	'wstat' => ''
	504	}
	505	},
	506	'todo' => {},
	507	'totals' => {
	508	'bad' => 1,
	509	'bonus' => 0,
	510	'files' => 1,
	511	'good' => 0,
	512	'max' => 5,
	513	'ok' => 3,
	514	'skipped' => 0,
	515	'sub_skipped' => 0,
	516	'tests' => 1,
	517	'todo' => 0
	518	}
	519	},
	520	'skip' => {
	521	'failed' => {},
	522	'todo' => {},
	523	'totals' => {
	524	'bad' => 0,
	525	'bonus' => 0,
	526	'files' => 1,
	527	'good' => 1,
	528	'max' => 5,
	529	'ok' => 5,
	530	'skipped' => 0,
	531	'sub_skipped' => 1,
	532	'tests' => 1,
	533	'todo' => 0
	534	}
	535	},
	536	'skip_nomsg' => {
	537	'failed' => {},
	538	'todo' => {},
	539	'totals' => {
	540	'bad' => 0,
	541	'bonus' => 0,
	542	'files' => 1,
	543	'good' => 1,
	544	'max' => 1,
	545	'ok' => 1,
	546	'skipped' => 0,
	547	'sub_skipped' => 1,
	548	'tests' => 1,
	549	'todo' => 0
	550	}
	551	},
	552	'skipall' => {
	553	'failed' => {},
	554	'todo' => {},
	555	'totals' => {
	556	'bad' => 0,
	557	'bonus' => 0,
	558	'files' => 1,
	559	'good' => 1,
	560	'max' => 0,
	561	'ok' => 0,
	562	'skipped' => 1,
	563	'sub_skipped' => 0,
	564	'tests' => 1,
	565	'todo' => 0
	566	}
567	},
568	'skipall_nomsg' => {
569	'failed' => {},
570	'todo' => {},
571	'totals' => {
572	'bad' => 0,
573	'bonus' => 0,
574	'files' => 1,
575	'good' => 1,
576	'max' => 0,
577	'ok' => 0,
578	'skipped' => 1,
579	'sub_skipped' => 0,
580	'tests' => 1,
581	'todo' => 0
582	}
583	},
584	'stdout_stderr' => {
585	'failed' => {},
586	'todo' => {},
587	'totals' => {
588	'bad' => 0,
589	'bonus' => 0,
590	'files' => 1,
591	'good' => 1,
592	'max' => 4,
593	'ok' => 4,
594	'skipped' => 0,
595	'sub_skipped' => 0,
596	'tests' => 1,
597	'todo' => 0
598	}
599	},
600	'switches' => {
f7c69158	601	'skip_if' => sub {
	602	( $ENV{PERL5OPT} \|\| '' ) =~ m{(?:^\|\s)-[dM]};
	603	},
b965d173	604	'failed' => {
27fc0087	605	"$TEST_DIR/switches" => {
b965d173	606	'canon' => 1,
	607	'estat' => '',
	608	'failed' => 1,
	609	'max' => 1,
27fc0087	610	'name' => "$TEST_DIR/switches",
b965d173	611	'wstat' => ''
	612	}
	613	},
	614	'todo' => {},
	615	'totals' => {
	616	'bad' => 1,
	617	'bonus' => 0,
	618	'files' => 1,
	619	'good' => 0,
	620	'max' => 1,
	621	'ok' => 0,
	622	'skipped' => 0,
	623	'sub_skipped' => 0,
	624	'tests' => 1,
	625	'todo' => 0
	626	}
	627	},
	628	'taint' => {
	629	'failed' => {},
	630	'todo' => {},
	631	'totals' => {
	632	'bad' => 0,
	633	'bonus' => 0,
	634	'files' => 1,
	635	'good' => 1,
	636	'max' => 1,
	637	'ok' => 1,
	638	'skipped' => 0,
	639	'sub_skipped' => 0,
	640	'tests' => 1,
	641	'todo' => 0
	642	}
	643	},
	644	'taint_warn' => {
	645	'failed' => {},
	646	'todo' => {},
	647	'totals' => {
	648	'bad' => 0,
	649	'bonus' => 0,
	650	'files' => 1,
	651	'good' => 1,
	652	'max' => 1,
	653	'ok' => 1,
	654	'skipped' => 0,
	655	'sub_skipped' => 0,
	656	'tests' => 1,
	657	'todo' => 0
	658	},
	659	'require' => 5.008001,
	660	},
	661	'todo_inline' => {
	662	'failed' => {},
	663	'todo' => {
27fc0087	664	"$TEST_DIR/todo_inline" => {
b965d173	665	'canon' => 2,
	666	'estat' => '',
	667	'failed' => 1,
	668	'max' => 2,
27fc0087	669	'name' => "$TEST_DIR/todo_inline",
b965d173	670	'wstat' => ''
	671	}
	672	},
	673	'totals' => {
	674	'bad' => 0,
	675	'bonus' => 1,
	676	'files' => 1,
	677	'good' => 1,
	678	'max' => 3,
	679	'ok' => 3,
	680	'skipped' => 0,
	681	'sub_skipped' => 0,
	682	'tests' => 1,
	683	'todo' => 2
	684	}
	685	},
	686	'todo_misparse' => {
	687	'failed' => {
27fc0087	688	"$TEST_DIR/todo_misparse" => {
b965d173	689	'canon' => 1,
	690	'estat' => '',
	691	'failed' => 1,
	692	'max' => 1,
27fc0087	693	'name' => "$TEST_DIR/todo_misparse",
b965d173	694	'wstat' => ''
	695	}
	696	},
	697	'todo' => {},
	698	'totals' => {
	699	'bad' => 1,
	700	'bonus' => 0,
	701	'files' => 1,
	702	'good' => 0,
	703	'max' => 1,
	704	'ok' => 0,
	705	'skipped' => 0,
	706	'sub_skipped' => 0,
	707	'tests' => 1,
	708	'todo' => 0
	709	}
	710	},
	711	'too_many' => {
	712	'failed' => {
27fc0087	713	"$TEST_DIR/too_many" => {
b965d173	714	'canon' => '4-7',
	715	'estat' => 4,
	716	'failed' => 4,
	717	'max' => 3,
27fc0087	718	'name' => "$TEST_DIR/too_many",
b965d173	719	'wstat' => '1024'
	720	}
	721	},
	722	'todo' => {},
	723	'totals' => {
	724	'bad' => 1,
	725	'bonus' => 0,
	726	'files' => 1,
	727	'good' => 0,
	728	'max' => 3,
	729	'ok' => 7,
	730	'skipped' => 0,
	731	'sub_skipped' => 0,
	732	'tests' => 1,
	733	'todo' => 0
	734	}
	735	},
	736	'vms_nit' => {
	737	'failed' => {
27fc0087	738	"$TEST_DIR/vms_nit" => {
b965d173	739	'canon' => 1,
	740	'estat' => '',
	741	'failed' => 1,
	742	'max' => 2,
27fc0087	743	'name' => "$TEST_DIR/vms_nit",
b965d173	744	'wstat' => ''
	745	}
	746	},
	747	'todo' => {},
	748	'totals' => {
	749	'bad' => 1,
	750	'bonus' => 0,
	751	'files' => 1,
	752	'good' => 0,
	753	'max' => 2,
	754	'ok' => 1,
	755	'skipped' => 0,
	756	'sub_skipped' => 0,
	757	'tests' => 1,
	758	'todo' => 0
	759	}
	760	}
	761	};
	762
	763	my $num_tests = ( keys %$results ) * $PER_LOOP;
	764
	765	plan tests => $num_tests;
	766
	767	sub local_name {
	768	my $name = shift;
	769	return File::Spec->catfile( split /\//, $name );
	770	}
	771
	772	sub local_result {
	773	my $hash = shift;
	774	my $new = {};
	775
	776	while ( my ( $file, $want ) = each %$hash ) {
	777	if ( exists $want->{name} ) {
	778	$want->{name} = local_name( $want->{name} );
	779	}
	780	$new->{ local_name($file) } = $want;
	781	}
	782	return $new;
	783	}
	784
	785	sub vague_status {
	786	my $hash = shift;
	787	return $hash unless $^O eq 'VMS';
	788
	789	while ( my ( $file, $want ) = each %$hash ) {
27fc0087	790	for (qw( estat wstat )) {
b965d173	791	if ( exists $want->{$_} ) {
	792	$want->{$_} = $want->{$_} ? 1 : 0;
	793	}
	794	}
	795	}
27fc0087	796	return $hash;
b965d173	797	}
	798
	799	{
	800	local $^W = 0;
	801
	802	# Silence harness output
	803	*TAP::Formatter::Console::_output = sub {
	804
	805	# do nothing
	806	};
	807	}
	808
	809	for my $test_key ( sort keys %$results ) {
	810	my $result = $results->{$test_key};
	811	SKIP: {
	812	if ( $result->{require} && $] < $result->{require} ) {
	813	skip "Test requires Perl $result->{require}, we have $]", 4;
	814	}
f7c69158	815
	816	if ( my $skip_if = $result->{skip_if} ) {
	817	skip
	818	"Test '$test_key' can't run properly in this environment", 4
	819	if $skip_if->();
	820	}
	821
b965d173	822	my @test_names = split( /,/, $test_key );
	823	my @test_files
	824	= map { File::Spec->catfile( $TEST_DIR, $_ ) } @test_names;
	825
	826	# For now we supress STDERR because it crufts up /our/ test
	827	# results. Should probably capture and analyse it.
	828	local ( OLDERR, OLDOUT );
	829	open OLDERR, '>&STDERR' or die $!;
	830	open OLDOUT, '>&STDOUT' or die $!;
	831	my $devnull = File::Spec->devnull;
	832	open STDERR, ">$devnull" or die $!;
	833	open STDOUT, ">$devnull" or die $!;
	834
	835	my ( $tot, $fail, $todo, $harness, $aggregate )
	836	= execute_tests( tests => \@test_files );
	837
	838	open STDERR, '>&OLDERR' or die $!;
	839	open STDOUT, '>&OLDOUT' or die $!;
	840
	841	my $bench = delete $tot->{bench};
	842	isa_ok $bench, 'Benchmark';
	843
	844	# Localise filenames in failed, todo
	845	my $lfailed = vague_status( local_result( $result->{failed} ) );
	846	my $ltodo = vague_status( local_result( $result->{todo} ) );
	847
	848	# use Data::Dumper;
	849	# diag Dumper( [ $lfailed, $ltodo ] );
	850
	851	is_deeply $tot, $result->{totals}, "totals match for $test_key";
	852	is_deeply vague_status($fail), $lfailed,
	853	"failure summary matches for $test_key";
	854	is_deeply vague_status($todo), $ltodo,
	855	"todo summary matches for $test_key";
	856	}
	857	}
	858	}