From: Peter Rabbitson Date: Sun, 9 Dec 2012 05:25:04 +0000 (+0100) Subject: Reorganize travis preconfig script - better tracing and handling of failures X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=44020f08e0c3670eddb381c29168250b79fcf33c;p=dbsrgits%2FDBIx-Class-Historic.git Reorganize travis preconfig script - better tracing and handling of failures --- diff --git a/maint/travis-ci_prepare_env b/maint/travis-ci_prepare_env index 6ccaa7a..8b433ab 100755 --- a/maint/travis-ci_prepare_env +++ b/maint/travis-ci_prepare_env @@ -8,8 +8,25 @@ set -e +echo_err() { echo "$@" 1>&2 ; } + +tstamp() { echo -n "[$(date '+%H:%M:%S')]" ; } + +run_or_err() { + echo_err -n "$(tstamp) $1 ... " + + LASTOUT=$( bash -c "$2" 2>&1) + LASTEXIT=$? + if [[ "$LASTEXIT" != "0" ]] ; then + echo_err -e "FAILED !!!\nCommand executed:\n$2\nSTDOUT+STDERR:\n$LASTOUT" + return $LASTEXIT + else + echo_err "done." + fi +} + if [[ "$TRAVIS" != "true" ]] ; then - echo "Running this script makes no sense outside of travis-ci" + echo_err "Running this script makes no sense outside of travis-ci" exit 1 fi @@ -34,37 +51,78 @@ fi # build options for perlbrew # +echo_err "$(tstamp) Smoke preconfiguration starting $(date)" + +TRAVIS_CPAN_MIRROR=$(echo "$PERL_CPANM_OPT" | grep -oP -- '--mirror\s+\S+' | head -n 1 | cut -d ' ' -f 2) +if ! [[ "$TRAVIS_CPAN_MIRROR" =~ "http://" ]] ; then + echo_err "Unable to extract primary cpan mirror from PERL_CPANM_OPT - something is wrong" + echo_err "PERL_CPANM_OPT: $PERL_CPANM_OPT" + exit 1 +fi + +export PERL_MM_USE_DEFAULT=1 PERL_MM_NONINTERACTIVE=1 PERL_AUTOINSTALL_PREFER_CPAN=1 PERLBREW_CPAN_MIRROR="$TRAVIS_CPAN_MIRROR" + +# Fixup CPANM_OPT to behave more like a traditional cpan client +export PERL_CPANM_OPT="$( echo $PERL_CPANM_OPT | sed 's/--skip-satisfied//' )" + if [[ -n "$BREWVER" ]] ; then # if this is not master and not a smoke/ branch - cancel all testing if [[ "$TRAVIS_BRANCH" =~ "topic/" ]]; then export SHORT_CIRCUIT_SMOKE=1 + echo_err "$(tstamp) non-smoke branch and custom perl compilation requested - bailing out" sleep 20 # give the console time to attach, otherwise it hangs return # this is like an `exit 0` in sourcing fi - perlbrew install --as $BREWVER --notest $BREWOPTS -j $NUMTHREADS perl-$BREWVER + run_or_err "Compiling/installing Perl $BREWVER (without testing, may take up to 5 minutes)" \ + "perlbrew install --as $BREWVER --notest $BREWOPTS -j $NUMTHREADS perl-$BREWVER" + perlbrew use $BREWVER fi -export PERL_MM_USE_DEFAULT=1 PERL_MM_NONINTERACTIVE=1 PERL_AUTOINSTALL_PREFER_CPAN=1 -export PERL_CPANM_OPT="$( echo $PERL_CPANM_OPT | sed 's/--skip-satisfied//' )" +# configure CPAN.pm - older versions go into an endless loop +# when trying to autoconf themselves +CPAN_CFG_SCRIPT=" + require CPAN; + require CPAN::FirstTime; + *CPAN::FirstTime::conf_sites = sub {}; + CPAN::Config->load; + \$CPAN::Config->{urllist} = [qw{ $TRAVIS_CPAN_MIRROR }]; + \$CPAN::Config->{halt_on_failure} = 1; + CPAN::Config->commit; +" +run_or_err "Configuring CPAN.pm" "perl -e '$CPAN_CFG_SCRIPT'" + +extract_prereqs() { + # hack-hack-hack + COMBINED_OUT="$( { stdout="$(cpanm --quiet --scandeps --format tree "$@")" ; } 2>&1; echo "!!!STDERRSTDOUTSEPARATOR!!!$stdout")" \ + || LASTEXIT=$? + + OUT=${COMBINED_OUT#*!!!STDERRSTDOUTSEPARATOR!!!} + ERR=$(grep -v " is up to date." <<< "${COMBINED_OUT%!!!STDERRSTDOUTSEPARATOR!!!*}") + + [[ -n "$LASTEXIT" ]] || LASTEXIT=0 + if [[ "$LASTEXIT" != "0" ]] || [[ -n "$ERR" ]] ; then + echo_err "$(echo -e "Error occured (exit code $LASTEXIT) retrieving dependencies of $@:\n$ERR\n$OUT")" + exit 1 + fi + + # throw away non-children (what was in $@), throw away ascii art, convert to modnames + perl -p -e 's/^\s*[^\\].+//; s/^[^A-Za-z]+//; s/\-[^\-]+$/ /; s/\-/::/g' <<< "$OUT" +} -installdeps() { +parallel_installdeps_notest() { if [[ -z "$@" ]] ; then return; fi + # flatten list into one string + MODLIST=$(echo "$@") + # The reason we do things so "non-interactively" is that xargs -P will have the # latest cpanm instance overwrite the buildlog. There seems to be no way to # specify a custom buildlog, hence we just collect the verbose output # and display it in case of failure - echo -n "Installing $@... " - if ! OUT=$( echo "$@" | xargs -n 1 -P $NUMTHREADS cpanm --verbose --no-interactive --notest --no-man-pages 2>&1 ) ; then - EX=$? - echo "FAILED !!!" - echo "$OUT" - exit $? - else - echo "done." - fi + run_or_err "Installing (without testing) $MODLIST" \ + "echo $MODLIST | xargs -n 1 -P $NUMTHREADS cpanm --verbose --no-interactive --notest --no-man-pages" } # Install *hard dep* modules that typically appear in more than one dep @@ -75,23 +133,10 @@ installdeps() { # times, and to avoid module re-architecture breaking another install # (e.g. once Carp is upgraded there's no more Carp::Heavy) # -installdeps $(cpanm --notest --quiet --showdeps Module::Build) -installdeps Test::Exception Test::Fatal Module::Runtime Carp -installdeps Sub::Name multidimensional namespace::clean Class::XSAccessor MRO::Compat -installdeps DBI Moo Class::Accessor::Grouped - -# configure CPAN.pm - older versions get tickled by M::AI and -# go into an endless loop when trying to autoconf themselves -# we are not *supposed* to actually use it, this is just a -# precaution -perl -e ' - require CPAN; - require CPAN::FirstTime; - *CPAN::FirstTime::conf_sites = sub {}; - CPAN::Config->load; - $CPAN::Config->{urllist} = [ "http://cpan.cpantesters.org/" ]; - CPAN::Config->commit; -' &> /dev/null +parallel_installdeps_notest $(extract_prereqs Module::Build) +parallel_installdeps_notest Test::Exception Test::Fatal Module::Runtime Carp +parallel_installdeps_notest Sub::Name multidimensional namespace::clean Class::XSAccessor MRO::Compat +parallel_installdeps_notest DBI Moo Class::Accessor::Grouped if [[ "$CLEANTEST" = "true" ]]; then # get the last inc/ off cpan - we will get rid of MI @@ -101,12 +146,14 @@ if [[ "$CLEANTEST" = "true" ]]; then # effects from travis preinstalls) # trick cpanm into executing true as shell - we just need the find+unpack - SHELL=/bin/true cpanm --look DBIx::Class - mv ~/.cpanm/latest-build/*/inc . + run_or_err "Downloading DBIC inc/ from CPAN" \ + "SHELL=/bin/true cpanm --look DBIx::Class" + + mv ~/.cpanm/latest-build/DBIx-Class-*/inc . else # we will be running all tests - preinstall MOAR stuff, run basic tests using SQLT # and set up whatever databases necessary - installdeps Module::Install DateTime::Format::Strptime MooseX::Types JSON::Any Class::DBI + parallel_installdeps_notest Module::Install DateTime::Format::Strptime MooseX::Types JSON::Any Class::DBI export DBICTEST_SQLT_DEPLOY=1 @@ -116,63 +163,64 @@ else sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/enabled\tboolean\ttrue" | debconf-set-selections' sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/sysdba_password/new_password\tpassword\t123" | debconf-set-selections' - sudo apt-get install -y memcached firebird2.5-super firebird2.5-dev expect + APT_PACKAGES="memcached firebird2.5-super firebird2.5-dev expect" + run_or_err "Installing packages ($APT_PACKAGES)" "sudo apt-get install -y $APT_PACKAGES" ### memcached export DBICTEST_MEMCACHED=127.0.0.1:11211 ### mysql - mysql -e 'create database dbic_test;' + run_or_err "Creating MySQL TestDB" "mysql -e 'create database dbic_test;'" export DBICTEST_MYSQL_DSN='dbi:mysql:database=dbic_test;host=127.0.0.1' export DBICTEST_MYSQL_USER=root ### pg - psql -c 'create database dbic_test;' -U postgres + run_or_err "Creating PostgreSQL TestDB" "psql -c 'create database dbic_test;' -U postgres" export DBICTEST_PG_DSN='dbi:Pg:database=dbic_test;host=127.0.0.1' export DBICTEST_PG_USER=postgres ### firebird # poor man's deb config - echo -n "Re-configuring firebird... " - if ! OUT=$( + EXPECT_FB_SCRIPT=' + spawn dpkg-reconfigure --frontend=text firebird2.5-super + expect "Enable Firebird server?" + send "\177\177\177\177yes\r" + expect "Password for SYSDBA" + send "123\r" + sleep 1 + wait + sleep 1 + ' + run_or_err "Re-configuring Firebird" " sync - DEBIAN_FRONTEND=text sudo expect -c ' - spawn dpkg-reconfigure --frontend=text firebird2.5-super - expect "Enable Firebird server?" - send "\177\177\177\177yes\r" - expect "Password for SYSDBA" - send "123\r" - sleep 1 - wait - sleep 1 - ' + DEBIAN_FRONTEND=text sudo expect -c '$EXPECT_FB_SCRIPT' + sleep 1 sync # restart the server for good measure sudo /etc/init.d/firebird2.5-super stop || true - sleep 3 + sleep 1 + sync sudo /etc/init.d/firebird2.5-super start - ) ; then - EX=$? - echo "FAILED !!!" - echo "$OUT" - exit $? - else - echo "done." - fi + sleep 1 + sync + " # creating testdb - sudo isql-fb -u sysdba -p 123 <<< "CREATE DATABASE '/var/lib/firebird/2.5/data/dbic_test.fdb';" + run_or_err "Creating Firebird TestDB" \ + "echo \"CREATE DATABASE '/var/lib/firebird/2.5/data/dbic_test.fdb';\" | sudo isql-fb -u sysdba -p 123" # the official version is full of 5.10-isms, but works perfectly fine on 5.8 # pull in our patched copy - git clone https://github.com/dbsrgits/perl-dbd-firebird-5.8.git ~/dbd-firebird + run_or_err "Fetching patched DBD::Firebird" \ + "git clone https://github.com/dbsrgits/perl-dbd-firebird-5.8.git ~/dbd-firebird" # the official version is very much outdated and does not compile on 5.14+ # use this rather updated source tree (needs to go to PAUSE): # https://github.com/pilcrow/perl-dbd-interbase - git clone https://github.com/dbsrgits/perl-dbd-interbase ~/dbd-interbase + run_or_err "Fetching patched DBD::InterBase" \ + "git clone https://github.com/dbsrgits/perl-dbd-interbase ~/dbd-interbase" - installdeps ~/dbd-interbase/ ~/dbd-firebird/ + parallel_installdeps_notest ~/dbd-interbase/ ~/dbd-firebird/ export DBICTEST_FIREBIRD_DSN=dbi:Firebird:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb export DBICTEST_FIREBIRD_USER=SYSDBA @@ -193,18 +241,23 @@ else #ORACLE_HOME=/usr/lib/oracle/xe/app/oracle/product/10.2.0/client fi -# install the remaining dependencies -perl Makefile.PL -installdeps $(make listalldeps) +# generate the makefile which will have different deps depending on +# the runmode and envvars set above +run_or_err "Configure on current branch" "perl Makefile.PL" -# announce what are we running -perl -V -echo "Using $NUMTHREADS Travis-CI concurrent processes" +# install the remaining dependencies +parallel_installdeps_notest $(make listalldeps) # make sure we got everything we need perl Makefile.PL &> /dev/null if [[ -n "$(make listdeps)" ]] ; then - echo "Not all deps installed - something went wrong :(" + echo_err "$(tstamp) Not all deps installed - something went wrong :(" make listdeps exit 1 fi + +run_or_err "Prepare blib" "make pure_all" + +# announce what are we running +perl -V +echo_err "$(tstamp) Starting tests using $NUMTHREADS concurrent processes"