From: Peter Rabbitson <ribasushi@cpan.org>
Date: Sat, 5 Mar 2016 14:32:11 +0000 (+0100)
Subject: Restructure thread/fork tests to run with maximum concurrency
X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=10dd5c05;p=dbsrgits%2FDBIx-Class.git

Restructure thread/fork tests to run with maximum concurrency

Add better exit handling on older perls: the thread-related failures on
global destroy have nothing to do with what we want to test. Activated by
setting $ENV{DBICTEST_DIRTY_EXIT}

Also reduce the default amount of workers - 10 is too many
---

diff --git a/t/50fork.t b/t/50fork.t
index c3c60ec..244bf2a 100644
--- a/t/50fork.t
+++ b/t/50fork.t
@@ -5,17 +5,17 @@ use strict;
 use warnings;
 use Test::More;
 use Test::Exception;
-
+use Time::HiRes qw(time sleep);
 
 use DBICTest;
 
 my $main_pid = $$;
 
-# README: If you set the env var to a number greater than 10,
+# README: If you set the env var to a number greater than 5,
 #   we will use that many children
 my $num_children = $ENV{DBICTEST_FORK_STRESS} || 1;
-if($num_children !~ /^[0-9]+$/ || $num_children < 10) {
-   $num_children = 10;
+if($num_children !~ /^[0-9]+$/ || $num_children < 5) {
+   $num_children = 5;
 }
 
 my ($dsn, $user, $pass) = @ENV{map { "DBICTEST_PG_${_}" } qw/DSN USER PASS/};
@@ -88,6 +88,11 @@ ok(!$@) or diag "Creation eval failed: $@";
 }
 
 $parent_rs->reset;
+
+# sleep until this spot so everything starts simultaneously
+# add "until turn of second" for prettier display
+my $t = int( time() ) + 4;
+
 my @pids;
 while(@pids < $num_children) {
 
@@ -102,6 +107,9 @@ while(@pids < $num_children) {
 
     $pid = $$;
 
+    sleep ( $t - time );
+    note ("Child process $pid starting work at " . time() );
+
     my $work = sub {
       my $child_rs = $schema->resultset('CD')->search({ year => 1901 });
       my $row = $parent_rs->next;
@@ -122,7 +130,7 @@ while(@pids < $num_children) {
       $work->();
     }
 
-    sleep(3);
+    sleep(2);
     exit 0;
 }
 
diff --git a/t/51threadnodb.t b/t/51threadnodb.t
index 30e8aec..ab3683c 100644
--- a/t/51threadnodb.t
+++ b/t/51threadnodb.t
@@ -19,6 +19,7 @@ use warnings;
 use Test::More;
 use Errno ();
 use DBIx::Class::_Util 'sigwarn_silencer';
+use Time::HiRes qw(time sleep);
 
 use DBICTest;
 
@@ -28,16 +29,20 @@ plan skip_all => 'DBIC does not actively support threads before perl 5.8.5'
 plan skip_all => 'Potential problems on Win32 Perl < 5.14 and Variable::Magic - investigation pending'
   if $^O eq 'MSWin32' && "$]" < 5.014 && DBICTest::RunMode->is_plain;
 
-# README: If you set the env var to a number greater than 10,
+# README: If you set the env var to a number greater than 5,
 #   we will use that many children
 my $num_children = $ENV{DBICTEST_THREAD_STRESS} || 1;
-if($num_children !~ /^[0-9]+$/ || $num_children < 10) {
-   $num_children = 10;
+if($num_children !~ /^[0-9]+$/ || $num_children < 5) {
+   $num_children = 5;
 }
 
 my $schema = DBICTest->init_schema(no_deploy => 1);
 isa_ok ($schema, 'DBICTest::Schema');
 
+# sleep until this spot so everything starts simultaneously
+# add "until turn of second" for prettier display
+my $t = int( time() ) + 4;
+
 my @threads;
 SKIP: {
 
@@ -45,12 +50,17 @@ SKIP: {
 
   for (1.. $num_children) {
     push @threads, threads->create(sub {
+      my $tid = threads->tid;
+
+      sleep ($t - time);
+      note ("Thread $tid starting work at " . time() );
+
       my $rsrc = $schema->source('Artist');
       undef $schema;
       isa_ok ($rsrc->schema, 'DBICTest::Schema');
       my $s2 = $rsrc->schema->clone;
 
-      sleep 1;  # without this many tasty crashes
+      sleep (0.2); # without this many tasty crashes even on latest perls
     }) || do {
       skip "EAGAIN encountered, your system is likely bogged down: skipping rest of test", 1
         if $! == Errno::EAGAIN();
@@ -65,4 +75,9 @@ ok(1, "past spawning");
 $_->join for @threads;
 ok(1, "past joining");
 
+# Too many threading bugs on exit, none of which have anything to do with
+# the actual stuff we test
+$ENV{DBICTEST_DIRTY_EXIT} = 1
+  if "$]"< 5.012;
+
 done_testing;
diff --git a/t/51threads.t b/t/51threads.t
index 0f24f7e..be0b1d6 100644
--- a/t/51threads.t
+++ b/t/51threads.t
@@ -21,6 +21,7 @@ use warnings;
 
 use Test::More;
 use Test::Exception;
+use Time::HiRes qw(time sleep);
 
 plan skip_all => 'DBIC does not actively support threads before perl 5.8.5'
   if "$]" < 5.008005;
@@ -28,11 +29,11 @@ plan skip_all => 'DBIC does not actively support threads before perl 5.8.5'
 
 use DBICTest;
 
-# README: If you set the env var to a number greater than 10,
+# README: If you set the env var to a number greater than 5,
 #   we will use that many children
 my $num_children = $ENV{DBICTEST_THREAD_STRESS} || 1;
-if($num_children !~ /^[0-9]+$/ || $num_children < 10) {
-   $num_children = 10;
+if($num_children !~ /^[0-9]+$/ || $num_children < 5) {
+   $num_children = 5;
 }
 
 my ($dsn, $user, $pass) = @ENV{map { "DBICTEST_PG_${_}" } qw/DSN USER PASS/};
@@ -85,7 +86,7 @@ lives_ok (sub {
     done_testing;
 
     close $tb->$_ for (qw/output failure_output todo_output/);
-    sleep(1); # tasty crashes without this
+    sleep (0.2); # tasty crashes without this
 
     $out;
   };
@@ -103,18 +104,31 @@ lives_ok (sub {
 }
 
 $parent_rs->reset;
+
+# sleep until this spot so everything starts simultaneously
+# add "until turn of second" for prettier display
+my $t = int( time() ) + 4;
+
 my @children;
 while(@children < $num_children) {
 
     my $newthread = async {
         my $tid = threads->tid;
 
+        sleep ($t - time);
+
+        # FIXME if we do not stagger the threads, sparks fly due to CXSA
+        sleep ( $tid / 10 ) if "$]" < 5.012;
+
+        note ("Thread $tid starting work at " . time() );
+
         my $child_rs = $schema->resultset('CD')->search({ year => 1901 });
         my $row = $parent_rs->next;
         if($row && $row->get_column('artist') =~ /^(?:123|456)$/) {
             $schema->resultset('CD')->create({ title => "test success $tid", artist => $tid, year => scalar(@children) });
         }
-        sleep(1); # tasty crashes without this
+
+        sleep (0.2); # without this many tasty crashes even on latest perls
     };
     die "Thread creation failed: $! $@" if !defined $newthread;
     push(@children, $newthread);
@@ -122,16 +136,17 @@ while(@children < $num_children) {
 
 ok(1, "past spawning");
 
-{
-    $_->join for(@children);
+my @tids;
+for (@children) {
+  push @tids, $_->tid;
+  $_->join;
 }
 
 ok(1, "past joining");
 
-while(@children) {
-    my $child = pop(@children);
-    my $tid = $child->tid;
-    my $rs = $schema->resultset('CD')->search({ title => "test success $tid", artist => $tid, year => scalar(@children) });
+while (@tids) {
+    my $tid = pop @tids;
+    my $rs = $schema->resultset('CD')->search({ title => "test success $tid", artist => $tid, year => scalar(@tids) });
     is($rs->next->get_column('artist'), $tid, "Child $tid successful");
 }
 
@@ -140,4 +155,9 @@ undef $parent_rs;
 
 $schema->storage->dbh->do("DROP TABLE cd");
 
+# Too many threading bugs on exit, none of which have anything to do with
+# the actual stuff we test
+$ENV{DBICTEST_DIRTY_EXIT} = 1
+  if "$]" < 5.012;
+
 done_testing;
diff --git a/t/51threadtxn.t b/t/51threadtxn.t
index 3e285ca..52a6966 100644
--- a/t/51threadtxn.t
+++ b/t/51threadtxn.t
@@ -1,6 +1,6 @@
 BEGIN { do "./t/lib/ANFANG.pm" or die ( $@ || $! ) }
 
-# README: If you set the env var to a number greater than 10,
+# README: If you set the env var to a number greater than 5,
 #   we will use that many children
 
 use Config;
@@ -28,12 +28,13 @@ plan skip_all => 'DBIC does not actively support threads before perl 5.8.5'
   if "$]" < 5.008005;
 
 use Scalar::Util 'weaken';
+use Time::HiRes qw(time sleep);
 
 use DBICTest;
 
 my $num_children = $ENV{DBICTEST_THREAD_STRESS} || 1;
-if($num_children !~ /^[0-9]+$/ || $num_children < 10) {
-   $num_children = 10;
+if($num_children !~ /^[0-9]+$/ || $num_children < 5) {
+   $num_children = 5;
 }
 
 my ($dsn, $user, $pass) = @ENV{map { "DBICTEST_PG_${_}" } qw/DSN USER PASS/};
@@ -59,11 +60,23 @@ eval {
 };
 ok(!$@) or diag "Creation eval failed: $@";
 
+# sleep until this spot so everything starts simultaneously
+# add "until turn of second" for prettier display
+my $t = int( time() ) + 4;
+
 my @children;
 while(@children < $num_children) {
 
     my $newthread = async {
         my $tid = threads->tid;
+
+        sleep ($t - time);
+
+        # FIXME if we do not stagger the threads, sparks fly due to CXSA
+        sleep ( $tid / 10 ) if "$]" < 5.012;
+
+        note ("Thread $tid starting work at " . time() );
+
         weaken(my $weak_schema = $schema);
         weaken(my $weak_parent_rs = $parent_rs);
         $schema->txn_do(sub {
@@ -73,7 +86,8 @@ while(@children < $num_children) {
                 $weak_schema->resultset('CD')->create({ title => "test success $tid", artist => $tid, year => scalar(@children) });
             }
         });
-        sleep(1);  # tasty crashes without this
+
+        sleep (0.2); # without this many tasty crashes even on latest perls
     };
     die "Thread creation failed: $! $@" if !defined $newthread;
     push(@children, $newthread);
@@ -98,4 +112,9 @@ ok(1, "Made it to the end");
 
 $schema->storage->dbh->do("DROP TABLE cd");
 
+# Too many threading bugs on exit, none of which have anything to do with
+# the actual stuff we test
+$ENV{DBICTEST_DIRTY_EXIT} = 1
+  if "$]" < 5.012;
+
 done_testing;
diff --git a/t/lib/ANFANG.pm b/t/lib/ANFANG.pm
index 354bc01..bdae983 100644
--- a/t/lib/ANFANG.pm
+++ b/t/lib/ANFANG.pm
@@ -151,6 +151,36 @@ lib->import('t/lib');
 # dead. In order to reduce hair-pulling make sure that ./inc/ is always there
 -f 'Makefile.PL' and mkdir 'inc' and mkdir 'inc/.author';
 
+END {
+  if( my @finalest_tasks = (
+
+    ( !$ENV{DBICTEST_DIRTY_EXIT} ? () : sub {
+
+      my $exit = $?;
+      require POSIX;
+
+      # Crucial flushes in case we are piping things out (e.g. prove)
+      # Otherwise the last lines will never arrive at the receiver
+      select($_), $| = 1 for \*STDOUT, \*STDERR;
+
+      POSIX::_exit($exit);
+    } ),
+
+  )) {
+
+    # in the case of an early skip_all B may very well not have loaded
+    unless( $INC{"B.pm"} ) {
+      local ( $!, $^E, $?, $@ );
+      require B;
+    }
+
+    # Make sure we run after any cleanup in other END blocks
+    # ( push-to-end twice in a row )
+    push @{ B::end_av()->object_2svref }, sub {
+      push @{ B::end_av()->object_2svref }, @finalest_tasks;
+    }
+  }
+}
 
 # make absolutely sure this is last
 $anfang_loaded = 1;