Smoker-wide parallel testing of installed deps

diff --git a/maint/travis-ci_scripts/30_before_script.bash b/maint/travis-ci_scripts/30_before_script.bash
index 706c979..44218a6 100755 (executable)
--- a/maint/travis-ci_scripts/30_before_script.bash
+++ b/maint/travis-ci_scripts/30_before_script.bash
@@ -31,9 +31,6 @@ if [[ "$CLEANTEST" = "true" ]]; then
 
   mv ~/.cpanm/latest-build/DBIx-Class-*/inc .
 
-  # older perls do not have a CPAN which understands configure_requires
-  # properly and what is worse a `cpan Foo` run exits with 0 even if some
-  # modules failed to install
   # The first CPAN which is somewhat sane is around 1.94_56 (perl 5.12)
   # The problem is that the first sane version also brings a *lot* of
   # deps with it, notably things like YAML and HTTP::Tiny
@@ -41,35 +38,10 @@ if [[ "$CLEANTEST" = "true" ]]; then
   # possible, mainly to catch "but X is perl core" mistakes
   # So instead we still use our stock (possibly old) CPAN, and add some
   # handholding
-  if ! CPAN_is_sane ; then
-    for m in \
-ExtUtils::MakeMaker \
-ExtUtils::CBuilder \
-Module::Build \
-; do
-      run_or_err "Pre-installing $m" "cpan $m"
-      if ! perl -e '
-
-eval ( q{require } . (
-  $ARGV[0] =~ m{ \/ .*? ([^\/]+) $ }x
-    ? do { my @p = split (/\-/, $1); pop @p; join "::", @p }
-    : $ARGV[0]
-) ) or ( print $@ and exit 1)' "$m" 2> /dev/null ; then
-
-        echo_err -e "$m installation failed\n$LASTOUT"
-        exit 1
-      fi
-    done
-  fi
 
-  # DBI has by far the longest test runtime - run less tests
-  # FIXME horrible horrible hack, need to implement in DBI itself
-  run_or_err "Downloading latest DBI distdir from CPAN" \
-    "SHELL=/bin/true cpanm --look DBI"
-  cd ~/.cpanm/latest-build/DBI-*/
-  perl -p -i -e 's/(create_.+?_tests) => 1/$1 => 0/' Makefile.PL
-  run_or_err "Pre-installing DBI, but running less tests" "perl Makefile.PL && make && make test && make install"
-  cd - &>/dev/null
+  # no configure_requires - we will need the usual suspects anyway
+  # without pre-installign these in one pass things like extract_prereqs won't work
+  CPAN_is_sane || installdeps ExtUtils::MakeMaker ExtUtils::CBuilder Module::Build
 
 else
   # we will be running all dbic tests - preinstall lots of stuff, run basic tests
@@ -123,6 +95,9 @@ if [[ "$CLEANTEST" = "true" ]]; then
     # sanity check, T::H does not report sensible errors when the subclass fails to load
     perl -MTAP::Harness::IgnoreNonessentialDzilAutogeneratedTests -e1
 
+    # DBD::SQLite reasonably wants DBI at config time
+    HARD_DEPS="DBI $HARD_DEPS"
+
     # FIXME
     # parent is temporary due to Carp https://rt.cpan.org/Ticket/Display.html?id=88494
     HARD_DEPS="parent $HARD_DEPS"
@@ -142,10 +117,7 @@ if [[ "$CLEANTEST" = "true" ]]; then
   fi
 ##### END TEMPORARY WORKAROUNDS
 
-  run_or_err "Installing/testing dependencies (may take up to 5 minutes): $HARD_DEPS" "cpan $HARD_DEPS"
-
-  # save the log as we may need it
-  INSTALLDEPS_OUT="$LASTOUT"
+  installdeps $HARD_DEPS
 
 ### FIXME in case we set it earlier in a workaround
   if [[ -n "$HARNESS_SUBCLASS" ]] ; then
@@ -182,7 +154,7 @@ perl Makefile.PL
 if [[ -n "$(make listdeps)" ]] ; then
   echo_err "$(tstamp) Not all deps installed - something went wrong :("
   sleep 1 # without this the echo below confuses the console listener >.<
-  CPAN_is_sane || echo_err -e "Outdated CPAN.pm used - full logs follows\n$INSTALLDEPS_OUT\n\nSearch for 'NOT OK' in the text above\n\nDeps still missing:"
+  CPAN_is_sane || echo_err -e "Outdated CPAN.pm used - full installdep log follows\n$INSTALLDEPS_OUT\n\nSearch for 'NOT OK' in the text above\n\nDeps still missing:"
   sleep 3 # without this the above echo confuses the console listener >.<
   make listdeps
   exit 1

diff --git a/maint/travis-ci_scripts/common.bash b/maint/travis-ci_scripts/common.bash
index cfa4ffe..4fb0819 100755 (executable)
--- a/maint/travis-ci_scripts/common.bash
+++ b/maint/travis-ci_scripts/common.bash
@@ -95,6 +95,65 @@ parallel_installdeps_notest() {
     "
 }
 
+installdeps() {
+  if [[ -z "$@" ]] ; then return; fi
+
+  echo_err "$(tstamp) Processing dependencies: $@"
+
+  local -x HARNESS_OPTIONS
+
+  HARNESS_OPTIONS="j$NUMTHREADS"
+
+  echo_err -n "Attempting install of $# modules under parallel ($HARNESS_OPTIONS) testing ... "
+
+  LASTEXIT=0
+  START_TIME=$SECONDS
+  LASTOUT=$( cpan_inst "$@" ) || LASTEXIT=$?
+  DELTA_TIME=$(( $SECONDS - $START_TIME ))
+
+  if [[ "$LASTEXIT" = "0" ]] ; then
+    echo_err "done (took ${DELTA_TIME}s)"
+  else
+    echo_err -n "failed (Exit:$LASTEXIT Log:$(/usr/bin/nopaste -q -s Shadowcat -d "Parallel installfail" <<< "$LASTOUT")) retrying with sequential testing ... "
+
+    HARNESS_OPTIONS=""
+    LASTEXIT=0
+    START_TIME=$SECONDS
+    LASTOUT=$( cpan_inst "$@" ) || LASTEXIT=$?
+    DELTA_TIME=$(( $SECONDS - $START_TIME ))
+
+    if [[ "$LASTEXIT" = "0" ]] ; then
+      echo_err "done (took ${DELTA_TIME}s)"
+    else
+      echo_err "FAILED !!! (after ${DELTA_TIME}s)"
+      echo_err "STDOUT+STDERR:"
+      echo_err "$LASTOUT"
+      exit 1
+    fi
+  fi
+
+  INSTALLDEPS_OUT="${INSTALLDEPS_OUT}${LASTOUT}"
+}
+
+cpan_inst() {
+  cpan "$@" 2>&1
+
+  # older perls do not have a CPAN which can exit with error on failed install
+  for m in "$@"; do
+    if ! perl -e '
+
+eval ( q{require } . (
+  $ARGV[0] =~ m{ \/ .*? ([^\/]+) $ }x
+    ? do { my @p = split (/\-/, $1); pop @p; join "::", @p }
+    : $ARGV[0]
+) ) or ( print $@ and exit 1)' "$m" 2> /dev/null ; then
+
+      echo -e "$m installation seems to have failed"
+      return 1
+    fi
+  done
+}
+
 CPAN_is_sane() { perl -MCPAN\ 1.94_56 -e 1 &>/dev/null ; }
 
 CPAN_supports_BUILDPL() { perl -MCPAN\ 1.9205 -e1 &>/dev/null; }