From: Peter Rabbitson Date: Thu, 27 Dec 2012 09:43:08 +0000 (+0100) Subject: Reorganize travis test script, I *think* no functional changes X-Git-Tag: v0.08205~28 X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=b58ecb01825f4afa0a13f76b799cd4608bb96ee2;p=dbsrgits%2FDBIx-Class.git Reorganize travis test script, I *think* no functional changes --- diff --git a/.travis.yml b/.travis.yml index db8fdfa..e6765dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ # * The way .travis.yml is fed to the command controller is idiotic - it # makes using multiline `bash -c` statements impossible. Therefore to # aid readability (our travis logic is rather complex), the bulk of -# functionality is moved to a script. More about the problem (and the +# functionality is moved to scripts. More about the problem (and the # WONTFIX "explanation") here: https://github.com/travis-ci/travis-ci/issues/497 # @@ -73,6 +73,13 @@ env: matrix: include: + # bleadperl + - perl: bleadperl_thr_mb + env: + - CLEANTEST=false + - BREWOPTS="-Duseithreads -Dusemorebits" + - BREWVER=blead + # minimum supported with threads - perl: 5.8.5_thr env: @@ -87,13 +94,6 @@ matrix: - BREWOPTS="" - BREWVER=5.8.3 - # bleadperl - - perl: bleadperl_thr_mb - env: - - CLEANTEST=false - - BREWOPTS="-Duseithreads -Dusemorebits" - - BREWVER=blead - # check CLEANTEST of minimum supported - perl: 5.8.3_nt_mb env: @@ -115,30 +115,60 @@ matrix: - BREWOPTS="-Duseithreads -Dusemorebits" - BREWVER=5.8.8 +# sourcing the files is *EXTREMELY* important - otherwise +# no envvars will survive + +# timeouts below taken from the base and overrides: +# https://github.com/travis-ci/travis-worker/blob/master/lib/travis/worker/config.rb#L32 +# https://github.com/travis-ci/travis-worker/blob/master/config/worker.php.yml + before_install: - # Do not make this part of the env-matrix - # different boxes we run on have different amount of hw threads - # hence why we need to query - # result is 1.5 times the physical threads - - export NUMTHREADS=$(( ( $(cut -f 2 -d '-' /sys/devices/system/cpu/online) + 1 ) * 15 / 10 )) + # Sets global envvars, downloads/configures debs based on CLEANTEST + # Sets extra DBICTEST_* envvars + # + # TIMEOUT: 900s + # + - source maint/travis-ci_scripts/10_before_install.bash install: # Build and switch to a custom perl if requested - # Set the environment based on CLEANTEST - # Preinstall/install deps + # Configure the perl env, preinstall some generic toolchain parts + # + # TIMEOUT: 900s # - # sourcing the file is *EXTREMELY* important - otherwise - # no envvars will survive - - source maint/travis-ci_prepare_env + - source maint/travis-ci_scripts/20_install.bash -script: - - export HARNESS_TIMER=1 HARNESS_OPTIONS=c:j$NUMTHREADS +before_script: + # Preinstall/install deps based on envvars/CLEANTEST + # + # TIMEOUT: 900s + # + - source maint/travis-ci_scripts/30_before_script.bash - # either a plain 'make test' OR a shuffled prove torture - # use the random order test plan unless CLEANTEST - # prepare_env may have short-circuited the test entirely - - test -n "$SHORT_CIRCUIT_SMOKE" || (test "$CLEANTEST" = "true" && make test || prove -lrswj$NUMTHREADS t xt) +script: + # Run actual tests + # + # TIMEOUT: 2100s + # + - source maint/travis-ci_scripts/40_script.bash after_success: - # check if we can assemble a dist properly given we have all deps - - test -n "$SHORT_CIRCUIT_SMOKE" || test "$CLEANTEST" = "true" || make dist + # Check if we can assemble a dist properly if not in CLEANTEST + # + # TIMEOUT: 300s + # + - source maint/travis-ci_scripts/50_after_success.bash + +after_failure: + # No tasks yet + # + # TIMEOUT: 300s + # + #- source maint/travis-ci_scripts/50_after_failure.bash + +after_script: + # No tasks yet + # + # TIMEOUT: 300s + # + #- source maint/travis-ci_scripts/60_after_script.bash diff --git a/maint/travis-ci_prepare_env b/maint/travis-ci_prepare_env deleted file mode 100755 index 56ec019..0000000 --- a/maint/travis-ci_prepare_env +++ /dev/null @@ -1,365 +0,0 @@ -#!/bin/bash - -# * The way .travis.yml is fed to the command controller is idiotic - it -# makes using multiline `bash -c` statements impossible. Therefore to -# aid readability (our travis logic is rather complex), the bulk of -# functionality is moved to a script. More about the problem here: -# https://github.com/travis-ci/travis-ci/issues/497 - -set -e - -echo_err() { echo "$@" 1>&2 ; } - -tstamp() { echo -n "[$(date '+%H:%M:%S')]" ; } - -run_or_err() { - echo_err -n "$(tstamp) $1 ... " - - LASTEXIT=0 - LASTOUT=$( bash -c "$2" 2>&1 ) || LASTEXIT=$? - - if [[ "$LASTEXIT" != "0" ]] ; then - echo_err -e "FAILED !!!\nCommand executed:\n$2\nSTDOUT+STDERR:\n$LASTOUT" - return $LASTEXIT - else - echo_err "done." - fi -} - -if [[ "$TRAVIS" != "true" ]] ; then - echo_err "Running this script makes no sense outside of travis-ci" - exit 1 -fi - -# -# Current dir is the root of the DBIC checkout (make sure to -# come back there if moving around before end of this script) -# -# envvars available for us: -# -# NUMTHREADS = { number} -# dynamically determined amount of threads we want to run on this -# smoker concurrently -# -# CLEANTEST = [ true | false ] -# controls whether we simulate a "user-side" install experience -# that is - no author deps and no M::I installation -# -# BREWVER = { tripple dotted perl version, e.g. 5.8.3 } -# brew a custom perl version such and such -# -# BREWOPTS = { string to be fed unquoted to perlbrew, e.g. -Duseithreads } -# build options for perlbrew -# - -echo_err "$(tstamp) Smoke preconfiguration starting $(date)" - -TRAVIS_CPAN_MIRROR=$(echo "$PERL_CPANM_OPT" | grep -oP -- '--mirror\s+\S+' | head -n 1 | cut -d ' ' -f 2) -if ! [[ "$TRAVIS_CPAN_MIRROR" =~ "http://" ]] ; then - echo_err "Unable to extract primary cpan mirror from PERL_CPANM_OPT - something is wrong" - echo_err "PERL_CPANM_OPT: $PERL_CPANM_OPT" - exit 1 -fi - -export PERL_MM_USE_DEFAULT=1 PERL_MM_NONINTERACTIVE=1 PERL_AUTOINSTALL_PREFER_CPAN=1 PERLBREW_CPAN_MIRROR="$TRAVIS_CPAN_MIRROR" - -# Fixup CPANM_OPT to behave more like a traditional cpan client -export PERL_CPANM_OPT="$( echo $PERL_CPANM_OPT | sed 's/--skip-satisfied//' )" - -if [[ -n "$BREWVER" ]] ; then - # .travis.yml already restricts branches to master, topic/* and smoke/* - # do some extra short-circuiting here - - # when smoking master do not attempt bleadperl (not release-critical) - if [[ "$TRAVIS_BRANCH" = "master" ]] && [[ "$BREWVER" = "blead" ]]; then - export SHORT_CIRCUIT_SMOKE=1 - # on topic/ branches test only with travis perls - elif [[ "$TRAVIS_BRANCH" =~ "topic/" ]]; then - export SHORT_CIRCUIT_SMOKE=1 - fi - - if [[ -n "$SHORT_CIRCUIT_SMOKE" ]]; then - echo_err "$(tstamp) non-smoke branch and custom perl compilation requested - bailing out" - sleep 20 # give the console time to attach, otherwise it hangs - return # this is like an `exit 0` in sourcing - fi - - run_or_err "Compiling/installing Perl $BREWVER (without testing, may take up to 5 minutes)" \ - "perlbrew install --as $BREWVER --notest --verbose $BREWOPTS -j $NUMTHREADS $BREWVER" - - sync - perlbrew use $BREWVER -fi - -# configure CPAN.pm - older versions go into an endless loop -# when trying to autoconf themselves -CPAN_CFG_SCRIPT=" - require CPAN; - require CPAN::FirstTime; - *CPAN::FirstTime::conf_sites = sub {}; - CPAN::Config->load; - \$CPAN::Config->{urllist} = [qw{ $TRAVIS_CPAN_MIRROR }]; - \$CPAN::Config->{halt_on_failure} = 1; - CPAN::Config->commit; -" -run_or_err "Configuring CPAN.pm" "perl -e '$CPAN_CFG_SCRIPT'" - -extract_prereqs() { - # hack-hack-hack - LASTEXIT=0 - COMBINED_OUT="$( { stdout="$(cpanm --quiet --scandeps --format tree "$@")" ; } 2>&1; echo "!!!STDERRSTDOUTSEPARATOR!!!$stdout")" \ - || LASTEXIT=$? - - OUT=${COMBINED_OUT#*!!!STDERRSTDOUTSEPARATOR!!!} - ERR=$(grep -v " is up to date." <<< "${COMBINED_OUT%!!!STDERRSTDOUTSEPARATOR!!!*}") - - if [[ "$LASTEXIT" != "0" ]] || [[ -n "$ERR" ]] ; then - echo_err "$(echo -e "Error occured (exit code $LASTEXIT) retrieving dependencies of $@:\n$ERR\n$OUT")" - exit 1 - fi - - # throw away non-children (what was in $@), throw away ascii art, convert to modnames - perl -p -e 's/^[a-z].+//i; s/^[^a-z]+//i; s/\-[^\-]+$/ /; s/\-/::/g' <<< "$OUT" -} - -parallel_installdeps_notest() { - if [[ -z "$@" ]] ; then return; fi - - # flatten list into one string - MODLIST=$(echo "$@") - - # The reason we do things so "non-interactively" is that xargs -P will have the - # latest cpanm instance overwrite the buildlog. There seems to be no way to - # specify a custom buildlog, hence we just collect the verbose output - # and display it in case of failure - run_or_err "Installing (without testing) $MODLIST" \ - "echo $MODLIST | xargs -n 1 -P $NUMTHREADS cpanm --verbose --no-interactive --notest --no-man-pages" -} - -if [[ "$CLEANTEST" = "true" ]]; then - # get the last inc/ off cpan - we will get rid of MI - # soon enough, but till then this will do - # the point is to have a *really* clean perl (the ones - # we build are guaranteed to be clean, without side - # effects from travis preinstalls) - - # trick cpanm into executing true as shell - we just need the find+unpack - run_or_err "Downloading DBIC inc/ from CPAN" \ - "SHELL=/bin/true cpanm --look DBIx::Class" - - mv ~/.cpanm/latest-build/DBIx-Class-*/inc . -else - # we will be running all dbic tests - preinstall lots of stuff, run basic tests - # using SQLT and set up whatever databases necessary - - # do the preinstall in several passes to minimize amount of cross-deps installing - # multiple times, and to avoid module re-architecture breaking another install - # (e.g. once Carp is upgraded there's no more Carp::Heavy) - # - parallel_installdeps_notest ExtUtils::MakeMaker - parallel_installdeps_notest Carp - parallel_installdeps_notest Module::Build - parallel_installdeps_notest Module::Runtime ExtUtils::Depends File::Spec Data::Dumper - parallel_installdeps_notest Test::Exception LWP - parallel_installdeps_notest Test::Fatal Test::Warn bareword::filehandles - parallel_installdeps_notest namespace::clean Class::XSAccessor MRO::Compat - parallel_installdeps_notest DBD::SQLite Moo Class::Accessor::Grouped - parallel_installdeps_notest Module::Install DateTime::Format::Strptime - parallel_installdeps_notest JSON::DWIW JSON JSON::XS Test::Pod::Coverage Test::EOL - parallel_installdeps_notest MooseX::Types JSON::Any Class::DBI - - export DBICTEST_SQLT_DEPLOY=1 - -### apt-get invocation - faster to grab everything at once - # - # FIXME these debconf lines should automate the firebird config but do not :((( - sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/enabled\tboolean\ttrue" | debconf-set-selections' - sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/sysdba_password/new_password\tpassword\t123" | debconf-set-selections' - - APT_PACKAGES="memcached firebird2.5-super firebird2.5-dev expect" - run_or_err "Installing packages ($APT_PACKAGES)" "sudo apt-get install -y $APT_PACKAGES" - -### memcached - export DBICTEST_MEMCACHED=127.0.0.1:11211 - -### mysql - run_or_err "Creating MySQL TestDB" "mysql -e 'create database dbic_test;'" - export DBICTEST_MYSQL_DSN='dbi:mysql:database=dbic_test;host=127.0.0.1' - export DBICTEST_MYSQL_USER=root - -### pg - run_or_err "Creating PostgreSQL TestDB" "psql -c 'create database dbic_test;' -U postgres" - export DBICTEST_PG_DSN='dbi:Pg:database=dbic_test;host=127.0.0.1' - export DBICTEST_PG_USER=postgres - -### firebird - # poor man's deb config - EXPECT_FB_SCRIPT=' - spawn dpkg-reconfigure --frontend=text firebird2.5-super - expect "Enable Firebird server?" - send "\177\177\177\177yes\r" - expect "Password for SYSDBA" - send "123\r" - sleep 1 - wait - sleep 1 - ' - run_or_err "Re-configuring Firebird" " - sync - DEBIAN_FRONTEND=text sudo expect -c '$EXPECT_FB_SCRIPT' - sleep 1 - sync - # restart the server for good measure - sudo /etc/init.d/firebird2.5-super stop || true - sleep 1 - sync - sudo /etc/init.d/firebird2.5-super start - sleep 1 - sync - " - - # creating testdb - # FIXME - this step still fails from time to time >:((( - # has to do with the FB reconfiguration I suppose - # for now if it fails - simply skip FB testing - if run_or_err "Creating Firebird TestDB" \ - "echo \"CREATE DATABASE '/var/lib/firebird/2.5/data/dbic_test.fdb';\" | sudo isql-fb -u sysdba -p 123" - then - # the official version is full of 5.10-isms, but works perfectly fine on 5.8 - # pull in our patched copy - run_or_err "Fetching patched DBD::Firebird" \ - "git clone https://github.com/dbsrgits/perl-dbd-firebird-5.8.git ~/dbd-firebird" - - # the official version is very much outdated and does not compile on 5.14+ - # use this rather updated source tree (needs to go to PAUSE): - # https://github.com/pilcrow/perl-dbd-interbase - run_or_err "Fetching patched DBD::InterBase" \ - "git clone https://github.com/dbsrgits/perl-dbd-interbase ~/dbd-interbase" - - parallel_installdeps_notest ~/dbd-interbase/ ~/dbd-firebird/ - - export DBICTEST_FIREBIRD_DSN=dbi:Firebird:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb - export DBICTEST_FIREBIRD_USER=SYSDBA - export DBICTEST_FIREBIRD_PASS=123 - - export DBICTEST_FIREBIRD_INTERBASE_DSN=dbi:InterBase:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb - export DBICTEST_FIREBIRD_INTERBASE_USER=SYSDBA - export DBICTEST_FIREBIRD_INTERBASE_PASS=123 - fi - -### oracle - # FIXME: todo - #DBICTEST_ORA_DSN=dbi:Oracle:host=localhost;sid=XE - #DBICTEST_ORA_USER=dbic_test - #DBICTEST_ORA_PASS=123 - #DBICTEST_ORA_EXTRAUSER_DSN=dbi:Oracle:host=localhost;sid=XE - #DBICTEST_ORA_EXTRAUSER_USER=dbic_test_extra - #DBICTEST_ORA_EXTRAUSER_PASS=123 - #ORACLE_HOME=/usr/lib/oracle/xe/app/oracle/product/10.2.0/client -fi - -CPAN_is_sane() { perl -MCPAN\ 1.94_56 -e 1 &>/dev/null ; } - -# install the rest -if [[ "$CLEANTEST" = "true" ]]; then - # older perls do not have a CPAN which understands configure_requires - # properly and what is worse a `cpan Foo` run exits with 0 even if some - # modules failed to install - # The first CPAN which is somewhat sane is around 1.94_56 (perl 5.12) - # The problem is that the first sane version also brings a *lot* of - # deps with it, notably things like YAML and HTTP::Tiny - # The goal of CLEANTEST is to have as little extra stuff installed as - # possible, mainly to catch "but X is perl core" mistakes - # So instead we still use our stock (possibly old) CPAN, and add some - # handholding - - CPAN_is_sane || \ - run_or_err "Pre-installing ExtUtils::MakeMaker and Module::Build" \ - "cpan ExtUtils::MakeMaker Module::Build" - - if ! perl -MModule::Build -e 1 &> /dev/null ; then - echo_err -e "Module::Build installation failed\n$LASTOUT" - exit 1 - fi - - # DBI has by far the longest test runtime - run less tests - # FIXME horrible horrible hack, need to implement in DBI itself - run_or_err "Downloading latest DBI distdir from CPAN" \ - "SHELL=/bin/true cpanm --look DBI" - cd ~/.cpanm/latest-build/DBI-*/ - perl -p -i -e 's/(create_.+?_tests) => 1/$1 => 0/' Makefile.PL - run_or_err "Pre-installing DBI, but running less tests" "perl Makefile.PL && make && make test && make install" - cd - &>/dev/null - - # generate the makefile which will have different deps depending on - # the runmode and envvars set above - run_or_err "Configure on current branch" "perl Makefile.PL" - HARD_DEPS="$(echo $(make listdeps))" - - # this is a fucked CPAN - won't understand configure_requires of - # various pieces we may run into - CPAN_is_sane || HARD_DEPS="ExtUtils::Depends B::Hooks::OP::Check $HARD_DEPS" - -##### TEMPORARY WORKAROUNDS - - # not sure what's going on here yet - perl -M5.008008 -e 1 &> /dev/null || \ - parallel_installdeps_notest multidimensional bareword::filehandles - - # work around Params::Validate not having a Makefile.PL so really old - # toolchains can not figure out what the prereqs are ;( - # Need to do more research before filing a bug requesting Makefile inclusion - perl -M5.008008 -e 1 &> /dev/null || \ - HARD_DEPS="$(extract_prereqs Params::Validate) $HARD_DEPS" - -##### END TEMPORARY WORKAROUNDS - - run_or_err "Installing/testing dependencies (may take up to 10 minutes): $HARD_DEPS" "cpan $HARD_DEPS" - - # this is a fucked CPAN - save the log as we may need it - CPAN_is_sane || INSTALLDEPS_OUT="$LASTOUT" - -else - # generate the makefile which will have different deps depending on - # the runmode and envvars set above - run_or_err "Configure on current branch" "perl Makefile.PL" - - # listalldeps is deliberate - will upgrade everything it can find - parallel_installdeps_notest $(make listalldeps) -fi - -echo_err "$(tstamp) Module configuration finished" -# this will display list of available versions -perl Makefile.PL - -# make sure we got everything we need -if [[ -n "$(make listdeps)" ]] ; then - echo_err "$(tstamp) Not all deps installed - something went wrong :(" - sleep 1 # without this the echo below confuses the console listener >.< - CPAN_is_sane || echo_err -e "Outdated CPAN.pm used - full logs follows\n$INSTALLDEPS_OUT\n\nSearch for 'NOT OK' in the text above\n\nDeps still missing:" - sleep 3 # without this the above echo confuses the console listener >.< - make listdeps - exit 1 -fi - -run_or_err "Prepare blib" "make pure_all" - -# announce what are we running -echo_err " -========================= CONFIGURATION COMPLETE =========================== -$(tstamp) Configuration phase seems to have taken $(date -ud "@$SECONDS" '+%H:%M:%S') (@$SECONDS) - -= CPUinfo -$(perl -0777 -p -e 's/.+\n\n(?!\z)//s' < /proc/cpuinfo) - -= Meminfo -$(free -m -t) - -= Environment -$(env | grep -P 'TEST|TRAVIS|PERL|DBIC' | LC_ALL=C sort | cat -v) - -= Perl in use -$(perl -V) -============================================================================ - -$(tstamp) Starting tests using $NUMTHREADS concurrent processes" diff --git a/maint/travis-ci_scripts/10_before_install.bash b/maint/travis-ci_scripts/10_before_install.bash new file mode 100755 index 0000000..e1649b9 --- /dev/null +++ b/maint/travis-ci_scripts/10_before_install.bash @@ -0,0 +1,105 @@ +#!/bin/bash + +source maint/travis-ci_scripts/common.bash +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi + +if [[ -n "$BREWVER" ]] ; then + # .travis.yml already restricts branches to master, topic/* and smoke/* + # do some extra short-circuiting here + + # when smoking master do not attempt bleadperl (not release-critical) + if [[ "$TRAVIS_BRANCH" = "master" ]] && [[ "$BREWVER" = "blead" ]]; then + echo_err "$(tstamp) master branch is not smoked with bleadperl - bailing out" + export SHORT_CIRCUIT_SMOKE=1 + # on topic/ branches test only with travis perls + elif [[ "$TRAVIS_BRANCH" =~ "topic/" ]]; then + echo_err "$(tstamp) non-smoke branch and custom perl compilation requested - bailing out" + export SHORT_CIRCUIT_SMOKE=1 + fi + + if [[ -n "$SHORT_CIRCUIT_SMOKE" ]]; then + sleep 20 # give the console time to attach, otherwise it hangs + return # this is like an `exit 0` in sourcing + fi +fi + +# different boxes we run on may have different amount of hw threads +# hence why we need to query +# result is 1.5 times the physical threads +export NUMTHREADS=$(( ( $(cut -f 2 -d '-' /sys/devices/system/cpu/online) + 1 ) * 15 / 10 )) + +if [[ "$CLEANTEST" != "true" ]]; then +### apt-get invocation - faster to grab everything at once + # + # FIXME these debconf lines should automate the firebird config but do not :((( + sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/enabled\tboolean\ttrue" | debconf-set-selections' + sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/sysdba_password/new_password\tpassword\t123" | debconf-set-selections' + + APT_PACKAGES="memcached firebird2.5-super firebird2.5-dev expect" + run_or_err "Installing packages ($APT_PACKAGES)" "sudo apt-get install --allow-unauthenticated -y $APT_PACKAGES" + +### config memcached + export DBICTEST_MEMCACHED=127.0.0.1:11211 + +### config mysql + run_or_err "Creating MySQL TestDB" "mysql -e 'create database dbic_test;'" + export DBICTEST_MYSQL_DSN='dbi:mysql:database=dbic_test;host=127.0.0.1' + export DBICTEST_MYSQL_USER=root + +### config pg + run_or_err "Creating PostgreSQL TestDB" "psql -c 'create database dbic_test;' -U postgres" + export DBICTEST_PG_DSN='dbi:Pg:database=dbic_test;host=127.0.0.1' + export DBICTEST_PG_USER=postgres + +### conig firebird + # poor man's deb config + EXPECT_FB_SCRIPT=' + spawn dpkg-reconfigure --frontend=text firebird2.5-super + expect "Enable Firebird server?" + send "\177\177\177\177yes\r" + expect "Password for SYSDBA" + send "123\r" + sleep 1 + wait + sleep 1 + ' + run_or_err "Re-configuring Firebird" " + sync + DEBIAN_FRONTEND=text sudo expect -c '$EXPECT_FB_SCRIPT' + sleep 1 + sync + # restart the server for good measure + sudo /etc/init.d/firebird2.5-super stop || true + sleep 1 + sync + sudo /etc/init.d/firebird2.5-super start + sleep 1 + sync + " + + # creating testdb + # FIXME - this step still fails from time to time >:((( + # has to do with the FB reconfiguration I suppose + # for now if it fails - simply skip FB testing + if run_or_err "Creating Firebird TestDB" \ + "echo \"CREATE DATABASE '/var/lib/firebird/2.5/data/dbic_test.fdb';\" | sudo isql-fb -u sysdba -p 123" + then + export DBICTEST_FIREBIRD_DSN=dbi:Firebird:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb + export DBICTEST_FIREBIRD_USER=SYSDBA + export DBICTEST_FIREBIRD_PASS=123 + + export DBICTEST_FIREBIRD_INTERBASE_DSN=dbi:InterBase:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb + export DBICTEST_FIREBIRD_INTERBASE_USER=SYSDBA + export DBICTEST_FIREBIRD_INTERBASE_PASS=123 + fi + +### oracle + # FIXME: todo + #DBICTEST_ORA_DSN=dbi:Oracle:host=localhost;sid=XE + #DBICTEST_ORA_USER=dbic_test + #DBICTEST_ORA_PASS=123 + #DBICTEST_ORA_EXTRAUSER_DSN=dbi:Oracle:host=localhost;sid=XE + #DBICTEST_ORA_EXTRAUSER_USER=dbic_test_extra + #DBICTEST_ORA_EXTRAUSER_PASS=123 + #ORACLE_HOME=/usr/lib/oracle/xe/app/oracle/product/10.2.0/client +fi diff --git a/maint/travis-ci_scripts/20_install.bash b/maint/travis-ci_scripts/20_install.bash new file mode 100755 index 0000000..f0e3d8e --- /dev/null +++ b/maint/travis-ci_scripts/20_install.bash @@ -0,0 +1,38 @@ +#!/bin/bash + +source maint/travis-ci_scripts/common.bash +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi + +TRAVIS_CPAN_MIRROR=$(echo "$PERL_CPANM_OPT" | grep -oP -- '--mirror\s+\S+' | head -n 1 | cut -d ' ' -f 2) +if ! [[ "$TRAVIS_CPAN_MIRROR" =~ "http://" ]] ; then + echo_err "Unable to extract primary cpan mirror from PERL_CPANM_OPT - something is wrong" + echo_err "PERL_CPANM_OPT: $PERL_CPANM_OPT" + exit 1 +fi + +export PERL_MM_USE_DEFAULT=1 PERL_MM_NONINTERACTIVE=1 PERL_AUTOINSTALL_PREFER_CPAN=1 PERLBREW_CPAN_MIRROR="$TRAVIS_CPAN_MIRROR" + +# Fixup CPANM_OPT to behave more like a traditional cpan client +export PERL_CPANM_OPT="$( echo $PERL_CPANM_OPT | sed 's/--skip-satisfied//' )" + +if [[ -n "$BREWVER" ]] ; then + run_or_err "Compiling/installing Perl $BREWVER (without testing, may take up to 5 minutes)" \ + "perlbrew install --as $BREWVER --notest --verbose $BREWOPTS -j $NUMTHREADS $BREWVER" + + # can not do 'perlbrew uss' in the run_or_err subshell above + perlbrew use $BREWVER || \ + ( echo_err -e "Unable to switch to $BREWVER - compillation failed?\n$LASTOUT"; exit 1 ) +fi + +# configure CPAN.pm - older versions go into an endless loop +# when trying to autoconf themselves +CPAN_CFG_SCRIPT=" + require CPAN; + require CPAN::FirstTime; + *CPAN::FirstTime::conf_sites = sub {}; + CPAN::Config->load; + \$CPAN::Config->{urllist} = [qw{ $TRAVIS_CPAN_MIRROR }]; + \$CPAN::Config->{halt_on_failure} = 1; + CPAN::Config->commit; +" +run_or_err "Configuring CPAN.pm" "perl -e '$CPAN_CFG_SCRIPT'" diff --git a/maint/travis-ci_scripts/30_before_script.bash b/maint/travis-ci_scripts/30_before_script.bash new file mode 100755 index 0000000..5351080 --- /dev/null +++ b/maint/travis-ci_scripts/30_before_script.bash @@ -0,0 +1,152 @@ +#!/bin/bash + +source maint/travis-ci_scripts/common.bash +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi + +if [[ "$CLEANTEST" = "true" ]]; then + # get the last inc/ off cpan - we will get rid of MI + # soon enough, but till then this will do + # the point is to have a *really* clean perl (the ones + # we build are guaranteed to be clean, without side + # effects from travis preinstalls) + + # trick cpanm into executing true as shell - we just need the find+unpack + run_or_err "Downloading DBIC inc/ from CPAN" \ + "SHELL=/bin/true cpanm --look DBIx::Class" + + mv ~/.cpanm/latest-build/DBIx-Class-*/inc . + + # older perls do not have a CPAN which understands configure_requires + # properly and what is worse a `cpan Foo` run exits with 0 even if some + # modules failed to install + # The first CPAN which is somewhat sane is around 1.94_56 (perl 5.12) + # The problem is that the first sane version also brings a *lot* of + # deps with it, notably things like YAML and HTTP::Tiny + # The goal of CLEANTEST is to have as little extra stuff installed as + # possible, mainly to catch "but X is perl core" mistakes + # So instead we still use our stock (possibly old) CPAN, and add some + # handholding + CPAN_is_sane || \ + run_or_err "Pre-installing ExtUtils::MakeMaker and Module::Build" \ + "cpan ExtUtils::MakeMaker Module::Build" + + if ! perl -MModule::Build -e 1 &> /dev/null ; then + echo_err -e "Module::Build installation failed\n$LASTOUT" + exit 1 + fi + + # DBI has by far the longest test runtime - run less tests + # FIXME horrible horrible hack, need to implement in DBI itself + run_or_err "Downloading latest DBI distdir from CPAN" \ + "SHELL=/bin/true cpanm --look DBI" + cd ~/.cpanm/latest-build/DBI-*/ + perl -p -i -e 's/(create_.+?_tests) => 1/$1 => 0/' Makefile.PL + run_or_err "Pre-installing DBI, but running less tests" "perl Makefile.PL && make && make test && make install" + cd - &>/dev/null + +else + # we will be running all dbic tests - preinstall lots of stuff, run basic tests + # using SQLT and set up whatever databases necessary + export DBICTEST_SQLT_DEPLOY=1 + + # do the preinstall in several passes to minimize amount of cross-deps installing + # multiple times, and to avoid module re-architecture breaking another install + # (e.g. once Carp is upgraded there's no more Carp::Heavy) + # + parallel_installdeps_notest ExtUtils::MakeMaker + parallel_installdeps_notest Carp + parallel_installdeps_notest Module::Build + parallel_installdeps_notest Module::Runtime ExtUtils::Depends File::Spec Data::Dumper + parallel_installdeps_notest Test::Exception LWP + parallel_installdeps_notest Test::Fatal Test::Warn bareword::filehandles + parallel_installdeps_notest namespace::clean Class::XSAccessor MRO::Compat + parallel_installdeps_notest DBD::SQLite Moo Class::Accessor::Grouped + parallel_installdeps_notest Module::Install DateTime::Format::Strptime + parallel_installdeps_notest JSON::DWIW JSON JSON::XS Test::Pod::Coverage Test::EOL + parallel_installdeps_notest MooseX::Types JSON::Any Class::DBI + + if [[ -n "DBICTEST_FIREBIRD_DSN" ]] ; then + # the official version is full of 5.10-isms, but works perfectly fine on 5.8 + # pull in our patched copy + run_or_err "Fetching patched DBD::Firebird" \ + "git clone https://github.com/dbsrgits/perl-dbd-firebird-5.8.git ~/dbd-firebird" + + # the official version is very much outdated and does not compile on 5.14+ + # use this rather updated source tree (needs to go to PAUSE): + # https://github.com/pilcrow/perl-dbd-interbase + run_or_err "Fetching patched DBD::InterBase" \ + "git clone https://github.com/dbsrgits/perl-dbd-interbase ~/dbd-interbase" + + parallel_installdeps_notest ~/dbd-interbase/ ~/dbd-firebird/ + fi + +fi + +# generate the makefile which will have different deps depending on +# the runmode and envvars set above +run_or_err "Configure on current branch" "perl Makefile.PL" + +# install (remaining) dependencies, sometimes with a gentle push +if [[ "$CLEANTEST" = "true" ]]; then + # we may need to prepend some stuff to that list + HARD_DEPS="$(echo $(make listdeps))" + + # this is a fucked CPAN - won't understand configure_requires of + # various pieces we may run into + CPAN_is_sane || HARD_DEPS="ExtUtils::Depends B::Hooks::OP::Check $HARD_DEPS" + +##### TEMPORARY WORKAROUNDS + + # not sure what's going on here yet + perl -M5.008008 -e 1 &> /dev/null || \ + parallel_installdeps_notest multidimensional bareword::filehandles + + # work around Params::Validate not having a Makefile.PL so really old + # toolchains can not figure out what the prereqs are ;( + # Need to do more research before filing a bug requesting Makefile inclusion + perl -M5.008008 -e 1 &> /dev/null || \ + HARD_DEPS="$(extract_prereqs Params::Validate) $HARD_DEPS" + +##### END TEMPORARY WORKAROUNDS + + run_or_err "Installing/testing dependencies (may take up to 10 minutes): $HARD_DEPS" "cpan $HARD_DEPS" + + # this is a fucked CPAN - save the log as we may need it + CPAN_is_sane || INSTALLDEPS_OUT="$LASTOUT" + +else + # listalldeps is deliberate - will upgrade everything it can find + parallel_installdeps_notest $(make listalldeps) +fi + +echo_err "$(tstamp) Dependency configuration finished" +# this will display list of available versions +perl Makefile.PL + +# make sure we got everything we need +if [[ -n "$(make listdeps)" ]] ; then + echo_err "$(tstamp) Not all deps installed - something went wrong :(" + sleep 1 # without this the echo below confuses the console listener >.< + CPAN_is_sane || echo_err -e "Outdated CPAN.pm used - full logs follows\n$INSTALLDEPS_OUT\n\nSearch for 'NOT OK' in the text above\n\nDeps still missing:" + sleep 3 # without this the above echo confuses the console listener >.< + make listdeps + exit 1 +fi + +# announce what are we running +echo_err " +===================== DEPENDENCY CONFIGURATION COMPLETE ===================== +$(tstamp) Configuration phase seems to have taken $(date -ud "@$SECONDS" '+%H:%M:%S') (@$SECONDS) + += CPUinfo +$(perl -0777 -p -e 's/.+\n\n(?!\z)//s' < /proc/cpuinfo) + += Meminfo +$(free -m -t) + += Environment +$(env | grep -P 'TEST|TRAVIS|PERL|DBIC' | LC_ALL=C sort | cat -v) + += Perl in use +$(perl -V) +=============================================================================" diff --git a/maint/travis-ci_scripts/40_script.bash b/maint/travis-ci_scripts/40_script.bash new file mode 100755 index 0000000..c044507 --- /dev/null +++ b/maint/travis-ci_scripts/40_script.bash @@ -0,0 +1,19 @@ +#!/bin/bash + +source maint/travis-ci_scripts/common.bash +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi + +export HARNESS_TIMER=1 HARNESS_OPTIONS=c:j$NUMTHREADS + +START_TIME=$SECONDS +if [[ "$CLEANTEST" = "true" ]] ; then + echo_err "$(tstamp) Running tests with plain \`make test\`" + run_or_err "Prepare blib" "make pure_all" + make test +else + PROVECMD="prove -lrswj$NUMTHREADS t xt" + echo_err "$(tstamp) running tests with \`$PROVECMD\`" + $PROVECMD +fi + +echo "$(tstamp) Testing took a total of $(( $SECONDS - $START_TIME ))s" diff --git a/maint/travis-ci_scripts/50_after_failure.bash b/maint/travis-ci_scripts/50_after_failure.bash new file mode 100755 index 0000000..4935763 --- /dev/null +++ b/maint/travis-ci_scripts/50_after_failure.bash @@ -0,0 +1,11 @@ +#!/bin/bash + +# !!! Nothing here will be executed !!! +# The source-line calling this script is commented out in .travis.yml + +source maint/travis-ci_scripts/common.bash +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi + +echo_err "Nothing to do" + +return 0 diff --git a/maint/travis-ci_scripts/50_after_success.bash b/maint/travis-ci_scripts/50_after_success.bash new file mode 100755 index 0000000..e25d702 --- /dev/null +++ b/maint/travis-ci_scripts/50_after_success.bash @@ -0,0 +1,6 @@ +#!/bin/bash + +source maint/travis-ci_scripts/common.bash +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi + +[[ "$CLEANTEST" = "true" ]] || run_or_err "Attempt to build a dist with all prereqs present" "make dist" diff --git a/maint/travis-ci_scripts/60_after_script.bash b/maint/travis-ci_scripts/60_after_script.bash new file mode 100755 index 0000000..4935763 --- /dev/null +++ b/maint/travis-ci_scripts/60_after_script.bash @@ -0,0 +1,11 @@ +#!/bin/bash + +# !!! Nothing here will be executed !!! +# The source-line calling this script is commented out in .travis.yml + +source maint/travis-ci_scripts/common.bash +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi + +echo_err "Nothing to do" + +return 0 diff --git a/maint/travis-ci_scripts/common.bash b/maint/travis-ci_scripts/common.bash new file mode 100755 index 0000000..73a8b3c --- /dev/null +++ b/maint/travis-ci_scripts/common.bash @@ -0,0 +1,63 @@ +#!/bin/bash + +set -e + +echo_err() { echo "$@" 1>&2 ; } + +if [[ "$TRAVIS" != "true" ]] ; then + echo_err "Running this script makes no sense outside of travis-ci" + exit 1 +fi + +tstamp() { echo -n "[$(date '+%H:%M:%S')]" ; } + +run_or_err() { + echo_err -n "$(tstamp) $1 ... " + + LASTEXIT=0 + START_TIME=$SECONDS + LASTOUT=$( bash -c "$2" 2>&1 ) || LASTEXIT=$? + DELTA_TIME=$(( $SECONDS - $START_TIME )) + + if [[ "$LASTEXIT" != "0" ]] ; then + echo_err -e "FAILED !!! (after ${DELTA_TIME}s)\nCommand executed:\n$2\nSTDOUT+STDERR:\n$LASTOUT" + return $LASTEXIT + else + echo_err "done (took ${DELTA_TIME}s)" + fi +} + +extract_prereqs() { + # hack-hack-hack + LASTEXIT=0 + COMBINED_OUT="$( { stdout="$(cpanm --quiet --scandeps --format tree "$@")" ; } 2>&1; echo "!!!STDERRSTDOUTSEPARATOR!!!$stdout")" \ + || LASTEXIT=$? + + OUT=${COMBINED_OUT#*!!!STDERRSTDOUTSEPARATOR!!!} + ERR=$(grep -v " is up to date." <<< "${COMBINED_OUT%!!!STDERRSTDOUTSEPARATOR!!!*}") + + if [[ "$LASTEXIT" != "0" ]] || [[ -n "$ERR" ]] ; then + echo_err "$(echo -e "Error occured (exit code $LASTEXIT) retrieving dependencies of $@:\n$ERR\n$OUT")" + exit 1 + fi + + # throw away non-children (what was in $@), throw away ascii art, convert to modnames + perl -p -e 's/^[a-z].+//i; s/^[^a-z]+//i; s/\-[^\-]+$/ /; s/\-/::/g' <<< "$OUT" +} + +parallel_installdeps_notest() { + if [[ -z "$@" ]] ; then return; fi + + # flatten list into one string + MODLIST=$(echo "$@") + + # The reason we do things so "non-interactively" is that xargs -P will have the + # latest cpanm instance overwrite the buildlog. There seems to be no way to + # specify a custom buildlog, hence we just collect the verbose output + # and display it in case of failure + run_or_err "Installing (without testing) $MODLIST" \ + "echo $MODLIST | xargs -n 1 -P $NUMTHREADS cpanm --verbose --no-interactive --notest --no-man-pages" +} + + +CPAN_is_sane() { perl -MCPAN\ 1.94_56 -e 1 &>/dev/null ; }