X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=blobdiff_plain;f=maint%2Ftravis-ci_prepare_env;h=56ec0195cf1772e1f02937d1849f38834508281b;hb=f16a86807fac55fb4a49a7b1de5d732380ce8954;hp=5d2337dcee39a72f6e1dd2b8145f8d944ec9420f;hpb=e352dd9afb6badc08be1ddb11e92cc8b29adb848;p=dbsrgits%2FDBIx-Class.git

diff --git a/maint/travis-ci_prepare_env b/maint/travis-ci_prepare_env
index 5d2337d..56ec019 100755
--- a/maint/travis-ci_prepare_env
+++ b/maint/travis-ci_prepare_env
@@ -8,8 +8,26 @@
 
 set -e
 
+echo_err() { echo "$@" 1>&2 ; }
+
+tstamp() { echo -n "[$(date '+%H:%M:%S')]" ; }
+
+run_or_err() {
+  echo_err -n "$(tstamp) $1 ... "
+
+  LASTEXIT=0
+  LASTOUT=$( bash -c "$2" 2>&1 ) || LASTEXIT=$?
+
+  if [[ "$LASTEXIT" != "0" ]] ; then
+    echo_err -e "FAILED !!!\nCommand executed:\n$2\nSTDOUT+STDERR:\n$LASTOUT"
+    return $LASTEXIT
+  else
+    echo_err "done."
+  fi
+}
+
 if [[ "$TRAVIS" != "true" ]] ; then
-  echo "Running this script makes no sense outside of travis-ci"
+  echo_err "Running this script makes no sense outside of travis-ci"
   exit 1
 fi
 
@@ -34,65 +52,90 @@ fi
 #   build options for perlbrew
 #
 
+echo_err "$(tstamp) Smoke preconfiguration starting $(date)"
+
+TRAVIS_CPAN_MIRROR=$(echo "$PERL_CPANM_OPT" | grep -oP -- '--mirror\s+\S+' | head -n 1 | cut -d ' ' -f 2)
+if ! [[ "$TRAVIS_CPAN_MIRROR" =~ "http://" ]] ; then
+  echo_err "Unable to extract primary cpan mirror from PERL_CPANM_OPT - something is wrong"
+  echo_err "PERL_CPANM_OPT: $PERL_CPANM_OPT"
+  exit 1
+fi
+
+export PERL_MM_USE_DEFAULT=1 PERL_MM_NONINTERACTIVE=1 PERL_AUTOINSTALL_PREFER_CPAN=1 PERLBREW_CPAN_MIRROR="$TRAVIS_CPAN_MIRROR"
+
+# Fixup CPANM_OPT to behave more like a traditional cpan client
+export PERL_CPANM_OPT="$( echo $PERL_CPANM_OPT | sed 's/--skip-satisfied//' )"
+
 if [[ -n "$BREWVER" ]] ; then
-  # if this is not master and not a smoke/ branch - cancel all testing
-  if [[ "$TRAVIS_BRANCH" =~ "topic/" ]]; then
+  # .travis.yml already restricts branches to master, topic/* and smoke/*
+  # do some extra short-circuiting here
+
+  # when smoking master do not attempt bleadperl (not release-critical)
+  if [[ "$TRAVIS_BRANCH" = "master" ]] && [[ "$BREWVER" = "blead" ]]; then
     export SHORT_CIRCUIT_SMOKE=1
+  # on topic/ branches test only with travis perls
+  elif [[ "$TRAVIS_BRANCH" =~ "topic/" ]]; then
+    export SHORT_CIRCUIT_SMOKE=1
+  fi
+
+  if [[ -n "$SHORT_CIRCUIT_SMOKE" ]]; then
+    echo_err "$(tstamp) non-smoke branch and custom perl compilation requested - bailing out"
     sleep 20  # give the console time to attach, otherwise it hangs
     return  # this is like an `exit 0` in sourcing
   fi
 
-  perlbrew install --as $BREWVER --notest $BREWOPTS -j $NUMTHREADS perl-$BREWVER
+  run_or_err "Compiling/installing Perl $BREWVER (without testing, may take up to 5 minutes)" \
+    "perlbrew install --as $BREWVER --notest --verbose $BREWOPTS -j $NUMTHREADS $BREWVER"
+
+  sync
   perlbrew use $BREWVER
 fi
 
-export PERL_MM_USE_DEFAULT=1 PERL_MM_NONINTERACTIVE=1 PERL_AUTOINSTALL_PREFER_CPAN=1
-export PERL_CPANM_OPT="$( echo $PERL_CPANM_OPT | sed 's/--skip-satisfied//' )"
+# configure CPAN.pm - older versions go into an endless loop
+# when trying to autoconf themselves
+CPAN_CFG_SCRIPT="
+  require CPAN;
+  require CPAN::FirstTime;
+  *CPAN::FirstTime::conf_sites = sub {};
+  CPAN::Config->load;
+  \$CPAN::Config->{urllist} = [qw{ $TRAVIS_CPAN_MIRROR }];
+  \$CPAN::Config->{halt_on_failure} = 1;
+  CPAN::Config->commit;
+"
+run_or_err "Configuring CPAN.pm" "perl -e '$CPAN_CFG_SCRIPT'"
+
+extract_prereqs() {
+  # hack-hack-hack
+  LASTEXIT=0
+  COMBINED_OUT="$( { stdout="$(cpanm --quiet --scandeps --format tree "$@")" ; } 2>&1; echo "!!!STDERRSTDOUTSEPARATOR!!!$stdout")" \
+    || LASTEXIT=$?
+
+  OUT=${COMBINED_OUT#*!!!STDERRSTDOUTSEPARATOR!!!}
+  ERR=$(grep -v " is up to date." <<< "${COMBINED_OUT%!!!STDERRSTDOUTSEPARATOR!!!*}")
+
+  if [[ "$LASTEXIT" != "0" ]] || [[ -n "$ERR" ]] ; then
+    echo_err "$(echo -e "Error occured (exit code $LASTEXIT) retrieving dependencies of $@:\n$ERR\n$OUT")"
+    exit 1
+  fi
+
+  # throw away non-children (what was in $@), throw away ascii art, convert to modnames
+  perl -p -e 's/^[a-z].+//i; s/^[^a-z]+//i; s/\-[^\-]+$/ /; s/\-/::/g' <<< "$OUT"
+}
 
-installdeps() {
+parallel_installdeps_notest() {
   if [[ -z "$@" ]] ; then return; fi
 
+  # flatten list into one string
+  MODLIST=$(echo "$@")
+
   # The reason we do things so "non-interactively" is that xargs -P will have the
   # latest cpanm instance overwrite the buildlog. There seems to be no way to
   # specify a custom buildlog, hence we just collect the verbose output
   # and display it in case of failure
-  echo -n "Installing $@... "
-  if ! OUT=$( echo "$@" | xargs -n 1 -P $NUMTHREADS cpanm --verbose --no-interactive --notest --no-man-pages 2>&1 ) ; then
-    EX=$?
-    echo "FAILED !!!"
-    echo "$OUT"
-    exit $?
-  else
-    echo "done."
-  fi
+  run_or_err "Installing (without testing) $MODLIST" \
+    "echo $MODLIST | xargs -n 1 -P $NUMTHREADS cpanm --verbose --no-interactive --notest --no-man-pages"
 }
 
-# Install *hard dep* modules that typically appear in more than one dep
-#
-# *NEVER* add optional depenencies here - will make CLEANTEST=true smokes useless
-#
-# do it in several passes to minimize amount of cross-deps installing multiple
-# times, and to avoid module re-architecture breaking another install
-# (e.g. once Carp is upgraded there's no more Carp::Heavy)
-#
-installdeps $(cpanm --notest --quiet --showdeps Module::Build)
-installdeps Test::Exception Test::Fatal Module::Runtime Carp
-installdeps Sub::Name multidimensional namespace::clean Class::XSAccessor MRO::Compat
-installdeps DBI Moo Class::Accessor::Grouped
-
-# configure CPAN.pm - older versions get tickled by M::AI and
-# go into an endless loop when trying to autoconf themselves
-# we are not *supposed* to actually use it, this is just a
-# precaution
-perl -e '
-  require CPAN;
-  require CPAN::FirstTime;
-  *CPAN::FirstTime::conf_sites = sub {};
-  CPAN::Config->load;
-  $CPAN::Config->{urllist} = [ "http://cpan.cpantesters.org/" ];
-  CPAN::Config->commit;
-' &> /dev/null
-
 if [[ "$CLEANTEST" = "true" ]]; then
   # get the last inc/ off cpan - we will get rid of MI
   # soon enough, but till then this will do
@@ -101,12 +144,29 @@ if [[ "$CLEANTEST" = "true" ]]; then
   # effects from travis preinstalls)
 
   # trick cpanm into executing true as shell - we just need the find+unpack
-  SHELL=/bin/true cpanm --look DBIx::Class
-  mv ~/.cpanm/latest-build/*/inc .
+  run_or_err "Downloading DBIC inc/ from CPAN" \
+    "SHELL=/bin/true cpanm --look DBIx::Class"
+
+  mv ~/.cpanm/latest-build/DBIx-Class-*/inc .
 else
-  # we will be running all tests - preinstall MOAR stuff, run basic tests using SQLT
-  # and set up whatever databases necessary
-  installdeps Module::Install DateTime::Format::Strptime MooseX::Types JSON::Any Class::DBI
+  # we will be running all dbic tests - preinstall lots of stuff, run basic tests
+  # using SQLT and set up whatever databases necessary
+
+  # do the preinstall in several passes to minimize amount of cross-deps installing
+  # multiple times, and to avoid module re-architecture breaking another install
+  # (e.g. once Carp is upgraded there's no more Carp::Heavy)
+  #
+  parallel_installdeps_notest ExtUtils::MakeMaker
+  parallel_installdeps_notest Carp
+  parallel_installdeps_notest Module::Build
+  parallel_installdeps_notest Module::Runtime ExtUtils::Depends File::Spec Data::Dumper
+  parallel_installdeps_notest Test::Exception LWP
+  parallel_installdeps_notest Test::Fatal Test::Warn bareword::filehandles
+  parallel_installdeps_notest namespace::clean Class::XSAccessor MRO::Compat
+  parallel_installdeps_notest DBD::SQLite Moo Class::Accessor::Grouped
+  parallel_installdeps_notest Module::Install DateTime::Format::Strptime
+  parallel_installdeps_notest JSON::DWIW JSON JSON::XS Test::Pod::Coverage Test::EOL
+  parallel_installdeps_notest MooseX::Types JSON::Any Class::DBI
 
   export DBICTEST_SQLT_DEPLOY=1
 
@@ -116,69 +176,76 @@ else
   sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/enabled\tboolean\ttrue" | debconf-set-selections'
   sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/sysdba_password/new_password\tpassword\t123" | debconf-set-selections'
 
-  sudo apt-get install -y memcached firebird2.5-super firebird2.5-dev expect
+  APT_PACKAGES="memcached firebird2.5-super firebird2.5-dev expect"
+  run_or_err "Installing packages ($APT_PACKAGES)" "sudo apt-get install -y $APT_PACKAGES"
 
 ### memcached
   export DBICTEST_MEMCACHED=127.0.0.1:11211
 
 ### mysql
-  mysql -e 'create database dbic_test;'
+  run_or_err "Creating MySQL TestDB" "mysql -e 'create database dbic_test;'"
   export DBICTEST_MYSQL_DSN='dbi:mysql:database=dbic_test;host=127.0.0.1'
   export DBICTEST_MYSQL_USER=root
 
 ### pg
-  psql -c 'create database dbic_test;' -U postgres
+  run_or_err "Creating PostgreSQL TestDB" "psql -c 'create database dbic_test;' -U postgres"
   export DBICTEST_PG_DSN='dbi:Pg:database=dbic_test;host=127.0.0.1'
   export DBICTEST_PG_USER=postgres
 
 ### firebird
   # poor man's deb config
-  echo -n "Re-configuring firebird... "
-  if ! OUT=$(
+  EXPECT_FB_SCRIPT='
+    spawn dpkg-reconfigure --frontend=text firebird2.5-super
+    expect "Enable Firebird server?"
+    send "\177\177\177\177yes\r"
+    expect "Password for SYSDBA"
+    send "123\r"
+    sleep 1
+    wait
+    sleep 1
+  '
+  run_or_err "Re-configuring Firebird" "
+    sync
+    DEBIAN_FRONTEND=text sudo expect -c '$EXPECT_FB_SCRIPT'
+    sleep 1
     sync
-    DEBIAN_FRONTEND=text sudo expect -c '
-      spawn dpkg-reconfigure --frontend=text firebird2.5-super
-      expect "Enable Firebird server?"
-      send "\177\177\177\177yes\r"
-      expect "Password for SYSDBA"
-      send "123\r"
-      sleep 1
-      wait
-      sleep 1
-    '
     # restart the server for good measure
     sudo /etc/init.d/firebird2.5-super stop || true
+    sleep 1
+    sync
     sudo /etc/init.d/firebird2.5-super start
-  ) ; then
-    EX=$?
-    echo "FAILED !!!"
-    echo "$OUT"
-    exit $?
-  else
-    echo "done."
-  fi
+    sleep 1
+    sync
+  "
 
   # creating testdb
-  sudo isql-fb -u sysdba -p 123 <<< "CREATE DATABASE '/var/lib/firebird/2.5/data/dbic_test.fdb';"
-
-  # the official version is full of 5.10-isms, but works perfectly fine on 5.8
-  # pull in our patched copy
-  git clone https://github.com/dbsrgits/perl-dbd-firebird-5.8.git ~/dbd-firebird
+  # FIXME - this step still fails from time to time >:(((
+  # has to do with the FB reconfiguration I suppose
+  # for now if it fails - simply skip FB testing
+  if run_or_err "Creating Firebird TestDB" \
+    "echo \"CREATE DATABASE '/var/lib/firebird/2.5/data/dbic_test.fdb';\" | sudo isql-fb -u sysdba -p 123"
+  then
+    # the official version is full of 5.10-isms, but works perfectly fine on 5.8
+    # pull in our patched copy
+    run_or_err "Fetching patched DBD::Firebird" \
+      "git clone https://github.com/dbsrgits/perl-dbd-firebird-5.8.git ~/dbd-firebird"
 
-  # the official version is very much outdated and does not compile on 5.14+
-  # use this rather updated source tree (needs to go to PAUSE):
-  # https://github.com/pilcrow/perl-dbd-interbase
-  git clone https://github.com/dbsrgits/perl-dbd-interbase ~/dbd-interbase
+    # the official version is very much outdated and does not compile on 5.14+
+    # use this rather updated source tree (needs to go to PAUSE):
+    # https://github.com/pilcrow/perl-dbd-interbase
+    run_or_err "Fetching patched DBD::InterBase" \
+      "git clone https://github.com/dbsrgits/perl-dbd-interbase ~/dbd-interbase"
 
-  installdeps ~/dbd-interbase/ ~/dbd-firebird/
+    parallel_installdeps_notest ~/dbd-interbase/ ~/dbd-firebird/
 
-  export DBICTEST_FIREBIRD_DSN=dbi:Firebird:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb
-  export DBICTEST_FIREBIRD_USER=SYSDBA
-  export DBICTEST_FIREBIRD_PASS=123
+    export DBICTEST_FIREBIRD_DSN=dbi:Firebird:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb
+    export DBICTEST_FIREBIRD_USER=SYSDBA
+    export DBICTEST_FIREBIRD_PASS=123
 
-  export DBICTEST_FIREBIRD_INTERBASE_DSN=dbi:InterBase:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb
-  export DBICTEST_FIREBIRD_INTERBASE_USER=SYSDBA
-  export DBICTEST_FIREBIRD_INTERBASE_PASS=123
+    export DBICTEST_FIREBIRD_INTERBASE_DSN=dbi:InterBase:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb
+    export DBICTEST_FIREBIRD_INTERBASE_USER=SYSDBA
+    export DBICTEST_FIREBIRD_INTERBASE_PASS=123
+  fi
 
 ### oracle
   # FIXME: todo
@@ -191,18 +258,108 @@ else
   #ORACLE_HOME=/usr/lib/oracle/xe/app/oracle/product/10.2.0/client
 fi
 
-# install the remaining dependencies
-perl Makefile.PL
-installdeps $(make listalldeps)
+CPAN_is_sane() { perl -MCPAN\ 1.94_56 -e 1 &>/dev/null ; }
 
-# announce what are we running
-perl -V
-echo "Using $NUMTHREADS Travis-CI concurrent processes"
+# install the rest
+if [[ "$CLEANTEST" = "true" ]]; then
+  # older perls do not have a CPAN which understands configure_requires
+  # properly and what is worse a `cpan Foo` run exits with 0 even if some
+  # modules failed to install
+  # The first CPAN which is somewhat sane is around 1.94_56 (perl 5.12)
+  # The problem is that the first sane version also brings a *lot* of
+  # deps with it, notably things like YAML and HTTP::Tiny
+  # The goal of CLEANTEST is to have as little extra stuff installed as
+  # possible, mainly to catch "but X is perl core" mistakes
+  # So instead we still use our stock (possibly old) CPAN, and add some
+  # handholding
+
+  CPAN_is_sane || \
+    run_or_err "Pre-installing ExtUtils::MakeMaker and Module::Build" \
+      "cpan ExtUtils::MakeMaker Module::Build"
+
+  if ! perl -MModule::Build -e 1 &> /dev/null ; then
+    echo_err -e "Module::Build installation failed\n$LASTOUT"
+    exit 1
+  fi
+
+  # DBI has by far the longest test runtime - run less tests
+  # FIXME horrible horrible hack, need to implement in DBI itself
+  run_or_err "Downloading latest DBI distdir from CPAN" \
+    "SHELL=/bin/true cpanm --look DBI"
+  cd ~/.cpanm/latest-build/DBI-*/
+  perl -p -i -e 's/(create_.+?_tests) => 1/$1 => 0/' Makefile.PL
+  run_or_err "Pre-installing DBI, but running less tests" "perl Makefile.PL && make && make test && make install"
+  cd - &>/dev/null
+
+  # generate the makefile which will have different deps depending on
+  # the runmode and envvars set above
+  run_or_err "Configure on current branch" "perl Makefile.PL"
+  HARD_DEPS="$(echo $(make listdeps))"
+
+  # this is a fucked CPAN - won't understand configure_requires of
+  # various pieces we may run into
+  CPAN_is_sane || HARD_DEPS="ExtUtils::Depends B::Hooks::OP::Check $HARD_DEPS"
+
+##### TEMPORARY WORKAROUNDS
+
+  # not sure what's going on here yet
+  perl -M5.008008 -e 1 &> /dev/null || \
+    parallel_installdeps_notest multidimensional bareword::filehandles
+
+  # work around Params::Validate not having a Makefile.PL so really old
+  # toolchains can not figure out what the prereqs are ;(
+  # Need to do more research before filing a bug requesting Makefile inclusion
+  perl -M5.008008 -e 1 &> /dev/null || \
+    HARD_DEPS="$(extract_prereqs Params::Validate) $HARD_DEPS"
+
+##### END TEMPORARY WORKAROUNDS
+
+  run_or_err "Installing/testing dependencies (may take up to 10 minutes): $HARD_DEPS" "cpan $HARD_DEPS"
+
+  # this is a fucked CPAN - save the log as we may need it
+  CPAN_is_sane || INSTALLDEPS_OUT="$LASTOUT"
+
+else
+  # generate the makefile which will have different deps depending on
+  # the runmode and envvars set above
+  run_or_err "Configure on current branch" "perl Makefile.PL"
+
+  # listalldeps is deliberate - will upgrade everything it can find
+  parallel_installdeps_notest $(make listalldeps)
+fi
+
+echo_err "$(tstamp) Module configuration finished"
+# this will display list of available versions
+perl Makefile.PL
 
 # make sure we got everything we need
-perl Makefile.PL &> /dev/null
 if [[ -n "$(make listdeps)" ]] ; then
-  echo "Not all deps installed - something went wrong :("
+  echo_err "$(tstamp) Not all deps installed - something went wrong :("
+  sleep 1 # without this the echo below confuses the console listener >.<
+  CPAN_is_sane || echo_err -e "Outdated CPAN.pm used - full logs follows\n$INSTALLDEPS_OUT\n\nSearch for 'NOT OK' in the text above\n\nDeps still missing:"
+  sleep 3 # without this the above echo confuses the console listener >.<
   make listdeps
   exit 1
 fi
+
+run_or_err "Prepare blib" "make pure_all"
+
+# announce what are we running
+echo_err "
+========================= CONFIGURATION COMPLETE ===========================
+$(tstamp) Configuration phase seems to have taken $(date -ud "@$SECONDS" '+%H:%M:%S') (@$SECONDS)
+
+= CPUinfo
+$(perl -0777 -p -e 's/.+\n\n(?!\z)//s' < /proc/cpuinfo)
+
+= Meminfo
+$(free -m -t)
+
+= Environment
+$(env | grep -P 'TEST|TRAVIS|PERL|DBIC' | LC_ALL=C sort | cat -v)
+
+= Perl in use
+$(perl -V)
+============================================================================
+
+$(tstamp) Starting tests using $NUMTHREADS concurrent processes"