From: Peter Rabbitson Date: Tue, 24 Sep 2013 13:59:48 +0000 (+0200) Subject: Smoker-wide parallel testing of installed deps X-Git-Tag: v0.08260~139 X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=579472dfc78ac84183a7f6b3a99b815e0a90b321;p=dbsrgits%2FDBIx-Class.git Smoker-wide parallel testing of installed deps --- diff --git a/maint/travis-ci_scripts/30_before_script.bash b/maint/travis-ci_scripts/30_before_script.bash index 706c979..44218a6 100755 --- a/maint/travis-ci_scripts/30_before_script.bash +++ b/maint/travis-ci_scripts/30_before_script.bash @@ -31,9 +31,6 @@ if [[ "$CLEANTEST" = "true" ]]; then mv ~/.cpanm/latest-build/DBIx-Class-*/inc . - # older perls do not have a CPAN which understands configure_requires - # properly and what is worse a `cpan Foo` run exits with 0 even if some - # modules failed to install # The first CPAN which is somewhat sane is around 1.94_56 (perl 5.12) # The problem is that the first sane version also brings a *lot* of # deps with it, notably things like YAML and HTTP::Tiny @@ -41,35 +38,10 @@ if [[ "$CLEANTEST" = "true" ]]; then # possible, mainly to catch "but X is perl core" mistakes # So instead we still use our stock (possibly old) CPAN, and add some # handholding - if ! CPAN_is_sane ; then - for m in \ -ExtUtils::MakeMaker \ -ExtUtils::CBuilder \ -Module::Build \ -; do - run_or_err "Pre-installing $m" "cpan $m" - if ! perl -e ' - -eval ( q{require } . ( - $ARGV[0] =~ m{ \/ .*? ([^\/]+) $ }x - ? do { my @p = split (/\-/, $1); pop @p; join "::", @p } - : $ARGV[0] -) ) or ( print $@ and exit 1)' "$m" 2> /dev/null ; then - - echo_err -e "$m installation failed\n$LASTOUT" - exit 1 - fi - done - fi - # DBI has by far the longest test runtime - run less tests - # FIXME horrible horrible hack, need to implement in DBI itself - run_or_err "Downloading latest DBI distdir from CPAN" \ - "SHELL=/bin/true cpanm --look DBI" - cd ~/.cpanm/latest-build/DBI-*/ - perl -p -i -e 's/(create_.+?_tests) => 1/$1 => 0/' Makefile.PL - run_or_err "Pre-installing DBI, but running less tests" "perl Makefile.PL && make && make test && make install" - cd - &>/dev/null + # no configure_requires - we will need the usual suspects anyway + # without pre-installign these in one pass things like extract_prereqs won't work + CPAN_is_sane || installdeps ExtUtils::MakeMaker ExtUtils::CBuilder Module::Build else # we will be running all dbic tests - preinstall lots of stuff, run basic tests @@ -123,6 +95,9 @@ if [[ "$CLEANTEST" = "true" ]]; then # sanity check, T::H does not report sensible errors when the subclass fails to load perl -MTAP::Harness::IgnoreNonessentialDzilAutogeneratedTests -e1 + # DBD::SQLite reasonably wants DBI at config time + HARD_DEPS="DBI $HARD_DEPS" + # FIXME # parent is temporary due to Carp https://rt.cpan.org/Ticket/Display.html?id=88494 HARD_DEPS="parent $HARD_DEPS" @@ -142,10 +117,7 @@ if [[ "$CLEANTEST" = "true" ]]; then fi ##### END TEMPORARY WORKAROUNDS - run_or_err "Installing/testing dependencies (may take up to 5 minutes): $HARD_DEPS" "cpan $HARD_DEPS" - - # save the log as we may need it - INSTALLDEPS_OUT="$LASTOUT" + installdeps $HARD_DEPS ### FIXME in case we set it earlier in a workaround if [[ -n "$HARNESS_SUBCLASS" ]] ; then @@ -182,7 +154,7 @@ perl Makefile.PL if [[ -n "$(make listdeps)" ]] ; then echo_err "$(tstamp) Not all deps installed - something went wrong :(" sleep 1 # without this the echo below confuses the console listener >.< - CPAN_is_sane || echo_err -e "Outdated CPAN.pm used - full logs follows\n$INSTALLDEPS_OUT\n\nSearch for 'NOT OK' in the text above\n\nDeps still missing:" + CPAN_is_sane || echo_err -e "Outdated CPAN.pm used - full installdep log follows\n$INSTALLDEPS_OUT\n\nSearch for 'NOT OK' in the text above\n\nDeps still missing:" sleep 3 # without this the above echo confuses the console listener >.< make listdeps exit 1 diff --git a/maint/travis-ci_scripts/common.bash b/maint/travis-ci_scripts/common.bash index cfa4ffe..4fb0819 100755 --- a/maint/travis-ci_scripts/common.bash +++ b/maint/travis-ci_scripts/common.bash @@ -95,6 +95,65 @@ parallel_installdeps_notest() { " } +installdeps() { + if [[ -z "$@" ]] ; then return; fi + + echo_err "$(tstamp) Processing dependencies: $@" + + local -x HARNESS_OPTIONS + + HARNESS_OPTIONS="j$NUMTHREADS" + + echo_err -n "Attempting install of $# modules under parallel ($HARNESS_OPTIONS) testing ... " + + LASTEXIT=0 + START_TIME=$SECONDS + LASTOUT=$( cpan_inst "$@" ) || LASTEXIT=$? + DELTA_TIME=$(( $SECONDS - $START_TIME )) + + if [[ "$LASTEXIT" = "0" ]] ; then + echo_err "done (took ${DELTA_TIME}s)" + else + echo_err -n "failed (Exit:$LASTEXIT Log:$(/usr/bin/nopaste -q -s Shadowcat -d "Parallel installfail" <<< "$LASTOUT")) retrying with sequential testing ... " + + HARNESS_OPTIONS="" + LASTEXIT=0 + START_TIME=$SECONDS + LASTOUT=$( cpan_inst "$@" ) || LASTEXIT=$? + DELTA_TIME=$(( $SECONDS - $START_TIME )) + + if [[ "$LASTEXIT" = "0" ]] ; then + echo_err "done (took ${DELTA_TIME}s)" + else + echo_err "FAILED !!! (after ${DELTA_TIME}s)" + echo_err "STDOUT+STDERR:" + echo_err "$LASTOUT" + exit 1 + fi + fi + + INSTALLDEPS_OUT="${INSTALLDEPS_OUT}${LASTOUT}" +} + +cpan_inst() { + cpan "$@" 2>&1 + + # older perls do not have a CPAN which can exit with error on failed install + for m in "$@"; do + if ! perl -e ' + +eval ( q{require } . ( + $ARGV[0] =~ m{ \/ .*? ([^\/]+) $ }x + ? do { my @p = split (/\-/, $1); pop @p; join "::", @p } + : $ARGV[0] +) ) or ( print $@ and exit 1)' "$m" 2> /dev/null ; then + + echo -e "$m installation seems to have failed" + return 1 + fi + done +} + CPAN_is_sane() { perl -MCPAN\ 1.94_56 -e 1 &>/dev/null ; } CPAN_supports_BUILDPL() { perl -MCPAN\ 1.9205 -e1 &>/dev/null; }