Better diagnostics of skipped non-essential tests

[dbsrgits/DBIx-Class.git] / maint / travis-ci_scripts / lib / TAP / Harness / IgnoreNonessentialDzilAutogeneratedTests.pm

package TAP::Harness::IgnoreNonessentialDzilAutogeneratedTests;

use warnings;
use strict;

use base 'TAP::Harness';
use File::Spec ();
use IPC::Open3 'open3';
use File::Temp ();

my $frivolous_test_map = {
# Test based on the extremely dep-heavy, *prone to failures* Test::CheckDeps
#
  't/00-check-deps.t' => [
    qr|^\Q# this test was generated with Dist::Zilla::Plugin::Test::CheckDeps|m,

    # older non-annotated versions
    qr|use \s+ Test::CheckDeps .*? ^\Qcheck_dependencies('suggests')\E .*? \QBAIL_OUT("Missing dependencies") if !Test::More->builder->is_passing|smx,
  ],

# "does everything compile" tests are useless by definition - this is what the
# rest of the test suite is for
#
  't/00-compile.t' => [
    qr|^\Q# this test was generated with Dist::Zilla::Plugin::Test::Compile|m,
  ],
};

sub aggregate_tests {
  my ($self, $aggregate, @all_tests) = @_;

  my ($run_tests, $skip_tests);

  TESTFILE:
  for (@all_tests) {
    my $fn = File::Spec::Unix->catpath( File::Spec->splitpath( $_ ) );

    if (my $REs = $frivolous_test_map->{$fn}) {
      my $slurptest = do { local (@ARGV, $/) = $fn; <> };
      $slurptest =~ $_ and push @$skip_tests, $fn and next TESTFILE for @$REs;
    }

    push @$run_tests, $fn;
  }

  if ($skip_tests) {

    for my $tfn (@$skip_tests) {

      (my $tfn_flattened = $tfn) =~ s|/|_|g;

      my $log_file = File::Temp->new(
        DIR => '/tmp',
        TEMPLATE => "AutoGenTest_${tfn_flattened}_XXXXX",
        SUFFIX => '.txt',
      );

      # FIXME I have no idea why the fileno dance is necessary - will investigate later
      # All I know is that if I pass in just $log_file - open3 ignores it >:(
      my $pid = open3(undef, '>&'.fileno($log_file), undef, $^X, qw(-I blib -I arch/lib), $tfn );
      waitpid ($pid, 0);
      my $ex = $?;

      if ($ex) {
        # use qx as opposed to another open3 until I figure out the above
        close $log_file or die "Unable to close $log_file: $!";
        chomp( my $url = `/usr/bin/nopaste -q -s Shadowcat -d $log_file < $log_file` );

        $tfn .= "[would NOT have passed: $ex / $url]";
      }
    }

    print STDERR "=== Skipping nonessential autogenerated tests: @$skip_tests\n";
  }

  return $self->SUPER::aggregate_tests($aggregate, @$run_tests);
}

1;