From: Peter Rabbitson Date: Thu, 23 Oct 2014 10:02:47 +0000 (+0200) Subject: (travis) Various TravisCI improvements lifted from blead X-Git-Tag: v0.082820~8 X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=c4c7254dc23f98ec5ea80fd44c37d8dbe6f6783d;p=dbsrgits%2FDBIx-Class.git (travis) Various TravisCI improvements lifted from blead --- diff --git a/.travis.yml b/.travis.yml index 047a582..d339eaa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,8 @@ # functionality is moved to scripts. More about the problem (and the # WONTFIX "explanation") here: https://github.com/travis-ci/travis-ci/issues/497 # +# the entire run times out after 50 minutes, or after 5 minutes without +# console output # # Smoke all branches except for blocked* and wip/* @@ -72,7 +74,7 @@ perl: env: - CLEANTEST=false - - CLEANTEST=true + - CLEANTEST=true NUMTHREADS=1 matrix: fast_finish: true @@ -216,52 +218,63 @@ matrix: - perl: devcpan_bleadperl_thr_mb -# sourcing the files is *EXTREMELY* important - otherwise -# no envvars will survive - -# the entire run times out after 50 minutes, or after 5 minutes without -# console output +### +### For the following two phases -e is *set* +### before_install: # common functions for all run phases below + # + # this is an exporter - sourcing it is crucial + # among other things it also sets -e + # - source maint/travis-ci_scripts/common.bash # Sets global envvars, downloads/configures debs based on CLEANTEST # Sets extra DBICTEST_* envvars # + # this is an exporter - sourcing it is crucial + # - source maint/travis-ci_scripts/10_before_install.bash install: # Build and switch to a custom perl if requested # Configure the perl env, preinstall some generic toolchain parts + # Possibly poison the environment + # + # this is an exporter - sourcing it is crucial # - source maint/travis-ci_scripts/20_install.bash +### +### From this point on -e is *unset*, rely on travis' error handling +### + - set +e + before_script: # Preinstall/install deps based on envvars/CLEANTEST # - - source maint/travis-ci_scripts/30_before_script.bash + # need to invoke the after_failure script manually + # because 'after_failure' runs only after 'script' fails + # + - maint/getstatus maint/travis-ci_scripts/30_before_script.bash || ( maint/travis-ci_scripts/50_after_failure.bash && /bin/false ) script: # Run actual tests # - - source maint/travis-ci_scripts/40_script.bash + - maint/getstatus maint/travis-ci_scripts/40_script.bash after_success: # Check if we can assemble a dist properly if not in CLEANTEST # - - source maint/travis-ci_scripts/50_after_success.bash + - maint/getstatus maint/travis-ci_scripts/50_after_success.bash after_failure: - # No tasks yet + # Final sysinfo printout on fail # - #- source maint/travis-ci_scripts/50_after_failure.bash + - maint/getstatus maint/travis-ci_scripts/50_after_failure.bash after_script: # No tasks yet # - #- source maint/travis-ci_scripts/60_after_script.bash - - # if we do not unset this before we terminate the travis teardown will - # mark the entire job as failed - - set +e + #- maint/getstatus maint/travis-ci_scripts/60_after_script.bash diff --git a/maint/travis-ci_scripts/10_before_install.bash b/maint/travis-ci_scripts/10_before_install.bash index ddc2589..356c17c 100755 --- a/maint/travis-ci_scripts/10_before_install.bash +++ b/maint/travis-ci_scripts/10_before_install.bash @@ -2,7 +2,7 @@ # Stop pre-started RDBMS and sync for some settle time run_or_err "Stopping MySQL" "sudo /etc/init.d/mysql stop" -run_or_err "Stopping PostgreSQL" "sudo /etc/init.d/postgresql stop" +run_or_err "Stopping PostgreSQL" "sudo /etc/init.d/postgresql stop || /bin/true" /bin/sync # Sanity check VM before continuing @@ -38,42 +38,40 @@ if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi # slurp the entire file and get the index off the last # `processor : XX` line # -# We also divide the result by two, otherwise the travis VM -# gets overloaded (the amount of available swap is just TOOOO -# damn small) -export NUMTHREADS="$(( ( $(perl -0777 -n -e 'print (/ (?: .+ ^ processor \s+ : \s+ (\d+) ) (?! ^ processor ) /smx)' < /proc/cpuinfo) + 1 ) / 2 ))" +# We also divide the result by a factor, otherwise the travis VM gets +# overloaded (the amount of available swap is just TOOOO damn small) +if [[ -z "$NUMTHREADS" ]] ; then + export NUMTHREADS="$(( ( $(perl -0777 -n -e 'print (/ (?: .+ ^ processor \s+ : \s+ (\d+) ) (?! ^ processor ) /smx)' < /proc/cpuinfo) + 1 ) / 3 ))" +fi export CACHE_DIR="/tmp/poormanscache" -# install some common tools from APT, more below unless CLEANTEST -apt_install libapp-nopaste-perl tree apt-transport-https +# these will be installed no matter what, also some extras unless CLEANTEST +common_packages="libapp-nopaste-perl tree" -# FIXME - the debian package is oddly broken - uses a bin/env based shebang -# so nothing works under a brew. Fix here until #debian-perl patches it up -sudo /usr/bin/perl -p -i -e 's|#!/usr/bin/env perl|#!/usr/bin/perl|' $(which nopaste) +if [[ "$CLEANTEST" = "true" ]]; then + + apt_install $common_packages + +else -if [[ "$CLEANTEST" != "true" ]]; then -### apt-get invocation - faster to grab everything at once # # FIXME these debconf lines should automate the firebird config but do not :((( sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/enabled\tboolean\ttrue" | debconf-set-selections' sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/sysdba_password/new_password\tpassword\t123" | debconf-set-selections' - # add extra APT repo for Oracle - # (https is critical - apt-get update can't seem to follow the 302) - sudo bash -c 'echo -e "\ndeb [arch=i386] https://oss.oracle.com/debian unstable main non-free" >> /etc/apt/sources.list' - - run_or_err "Cloning poor man's cache from github" "git clone --depth=1 --branch=poor_mans_travis_cache https://github.com/ribasushi/travis_futzing.git $CACHE_DIR && $CACHE_DIR/reassemble" - - run_or_err "Priming up the APT cache with $(echo $(ls -d $CACHE_DIR/apt_cache/*.deb))" "sudo cp $CACHE_DIR/apt_cache/*.deb /var/cache/apt/archives" + run_or_err "Updating APT sources" "sudo apt-get update" + apt_install $common_packages libmysqlclient-dev memcached firebird2.5-super firebird2.5-dev unixodbc-dev expect - apt_install libmysqlclient-dev memcached firebird2.5-super firebird2.5-dev unixodbc-dev expect oracle-xe + run_or_err "Cloning poor man's cache from github" "git clone --depth=1 --single-branch --branch=oracle/10.2.0 https://github.com/poortravis/poormanscache.git $CACHE_DIR && $CACHE_DIR/reassemble" + run_or_err "Installing OracleXE manually from deb" "sudo dpkg -i $CACHE_DIR/apt_cache/oracle-xe_10.2.0.1-1.1_i386.deb || sudo bash -c 'source maint/travis-ci_scripts/common.bash && apt_install -f'" ### config memcached run_or_err "Starting memcached" "sudo /etc/init.d/memcached start" export DBICTEST_MEMCACHED=127.0.0.1:11211 ### config mysql + run_or_err "Installing minimizing MySQL config" "sudo cp maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf /etc/mysql/conf.d/ && sudo chmod 644 /etc/mysql/conf.d/*.cnf" run_or_err "Starting MySQL" "sudo /etc/init.d/mysql start" run_or_err "Creating MySQL TestDB" "mysql -e 'create database dbic_test;'" export DBICTEST_MYSQL_DSN='dbi:mysql:database=dbic_test;host=127.0.0.1' @@ -120,24 +118,6 @@ if [[ "$CLEANTEST" != "true" ]]; then "echo \"CREATE DATABASE '/var/lib/firebird/2.5/data/dbic_test.fdb';\" | sudo isql-fb -u sysdba -p 123" then - run_or_err "Fetching and building Firebird ODBC driver" ' - cd "$(mktemp -d)" - wget -qO- http://sourceforge.net/projects/firebird/files/firebird-ODBC-driver/2.0.2-Release/OdbcFb-Source-2.0.2.153.gz/download | tar -zx - cd Builds/Gcc.lin - perl -p -i -e "s|/usr/lib64|/usr/lib/x86_64-linux-gnu|g" ../makefile.environ - make -f makefile.linux - sudo make -f makefile.linux install - ' - - sudo bash -c 'cat >> /etc/odbcinst.ini' <<< " -[Firebird] -Description = InterBase/Firebird ODBC Driver -Driver = /usr/lib/x86_64-linux-gnu/libOdbcFb.so -Setup = /usr/lib/x86_64-linux-gnu/libOdbcFb.so -Threading = 1 -FileUsage = 1 -" - export DBICTEST_FIREBIRD_DSN=dbi:Firebird:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb export DBICTEST_FIREBIRD_USER=SYSDBA export DBICTEST_FIREBIRD_PASS=123 @@ -146,10 +126,6 @@ FileUsage = 1 export DBICTEST_FIREBIRD_INTERBASE_USER=SYSDBA export DBICTEST_FIREBIRD_INTERBASE_PASS=123 - export DBICTEST_FIREBIRD_ODBC_DSN="dbi:ODBC:Driver=Firebird;Dbname=/var/lib/firebird/2.5/data/dbic_test.fdb" - export DBICTEST_FIREBIRD_ODBC_USER=SYSDBA - export DBICTEST_FIREBIRD_ODBC_PASS=123 - break fi @@ -218,3 +194,10 @@ FileUsage = 1 export ORACLE_HOME="$CACHE_DIR/ora_instaclient/x86-64/oracle_instaclient_10.2.0.5.0" fi + +# The debian package is oddly broken - uses a /bin/env based shebang +# so nothing works under a brew, fixed in libapp-nopaste-perl 0.92-3 +# http://changelogs.ubuntu.com/changelogs/pool/universe/liba/libapp-nopaste-perl/libapp-nopaste-perl_0.96-1/changelog +# +# Since the vm runs an old version of umbongo fix things ourselves +sudo /usr/bin/perl -p -i -e 's|#!/usr/bin/env perl|#!/usr/bin/perl|' $(which nopaste) diff --git a/maint/travis-ci_scripts/20_install.bash b/maint/travis-ci_scripts/20_install.bash index 23ac294..0279f78 100755 --- a/maint/travis-ci_scripts/20_install.bash +++ b/maint/travis-ci_scripts/20_install.bash @@ -2,14 +2,11 @@ if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi -CPAN_MIRROR=$(echo "$PERL_CPANM_OPT" | grep -oP -- '--mirror\s+\S+' | head -n 1 | cut -d ' ' -f 2) -if ! [[ "$CPAN_MIRROR" =~ "http://" ]] ; then - echo_err "Unable to extract primary cpan mirror from PERL_CPANM_OPT - something is wrong" - echo_err "PERL_CPANM_OPT: $PERL_CPANM_OPT" - CPAN_MIRROR="http://cpan.metacpan.org/" - PERL_CPANM_OPT="$PERL_CPANM_OPT --mirror $CPAN_MIRROR" - echo_err "Using $CPAN_MIRROR for the time being" -fi +# we need a mirror that both has the standard index and a backpan version rolled +# into one, due to MDV testing +CPAN_MIRROR="http://cpan.metacpan.org/" + +PERL_CPANM_OPT="$PERL_CPANM_OPT --mirror $CPAN_MIRROR" # do not set PERLBREW_CPAN_MIRROR - not all backpan-like mirrors have the perl tarballs export PERL_MM_USE_DEFAULT=1 PERL_MM_NONINTERACTIVE=1 PERL_AUTOINSTALL_PREFER_CPAN=1 HARNESS_TIMER=1 MAKEFLAGS="-j$NUMTHREADS" @@ -84,3 +81,45 @@ CPAN_CFG_SCRIPT=" CPAN::Config->commit; " run_or_err "Configuring CPAN.pm" "perl -e '$CPAN_CFG_SCRIPT'" + +# poison the environment +if [[ "$POISON_ENV" = "true" ]] ; then + + # in addition to making sure tests do not rely on implicid order of + # returned results, look through lib, find all mentioned ENVvars and + # set them to true and see if anything explodes + for var in \ + DBICTEST_SQLITE_REVERSE_DEFAULT_ORDER \ + $(grep -P '\$ENV\{' -r lib/ --exclude-dir Optional | grep -oP '\bDBIC\w+' | sort -u | grep -v DBIC_TRACE) + do + if [[ -z "${!var}" ]] ; then + export $var=1 + echo "POISON_ENV: setting $var to 1" + fi + done + + # bogus nonexisting DBI_* + export DBI_DSN="dbi:ODBC:server=NonexistentServerAddress" + export DBI_DRIVER="ADO" + + # some people do in fact set this - boggle!!! + export PERL_STRICTURES_EXTRA=1 + + # emulate a local::lib-like env + # trick cpanm into executing true as shell - we just need the find+unpack + run_or_err "Downloading latest stable DBIC from CPAN" \ + "SHELL=/bin/true cpanm --look DBIx::Class" + + export PERL5LIB="$( ls -d ~/.cpanm/latest-build/DBIx-Class-*/lib | tail -n1 ):$PERL5LIB" + + # perldoc -l searches $(pwd)/lib in addition to PERL5LIB etc, hence the cd / + echo_err "Latest stable DBIC (without deps) locatable via \$PERL5LIB at $(cd / && perldoc -l DBIx::Class)" + +fi + +if [[ "$CLEANTEST" != "true" ]] ; then + # using SQLT if will be available + # not doing later because we will be running in a subshell + export DBICTEST_SQLT_DEPLOY=1 + +fi diff --git a/maint/travis-ci_scripts/30_before_script.bash b/maint/travis-ci_scripts/30_before_script.bash index 3d0fae8..651acd2 100755 --- a/maint/travis-ci_scripts/30_before_script.bash +++ b/maint/travis-ci_scripts/30_before_script.bash @@ -1,44 +1,15 @@ #!/bin/bash -if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi +# this file is executed in a subshell - set up the common stuff +source maint/travis-ci_scripts/common.bash -# poison the environment -if [[ "$POISON_ENV" = "true" ]] ; then - - # in addition to making sure tests do not rely on implicid order of - # returned results, look through lib, find all mentioned ENVvars and - # set them to true and see if anything explodes - for var in \ - DBICTEST_SQLITE_REVERSE_DEFAULT_ORDER \ - $(grep -P '\$ENV\{' -r lib/ --exclude-dir Optional | grep -oP '\bDBIC\w+' | sort -u | grep -v DBIC_TRACE) - do - if [[ -z "${!var}" ]] ; then - export $var=1 - echo "POISON_ENV: setting $var to 1" - fi - done - - # bogus nonexisting DBI_* - export DBI_DSN="dbi:ODBC:server=NonexistentServerAddress" - export DBI_DRIVER="ADO" - - # some people do in fact set this - boggle!!! - export PERL_STRICTURES_EXTRA=1 +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then exit 0 ; fi - # emulate a local::lib-like env - # trick cpanm into executing true as shell - we just need the find+unpack - run_or_err "Downloading latest stable DBIC from CPAN" \ - "SHELL=/bin/true cpanm --look DBIx::Class" - - export PERL5LIB="$( ls -d ~/.cpanm/latest-build/DBIx-Class-*/lib | tail -n1 ):$PERL5LIB" - - # perldoc -l searches $(pwd)/lib in addition to PERL5LIB etc, hence the cd / - echo_err "Latest stable DBIC (without deps) locatable via \$PERL5LIB at $(cd / && perldoc -l DBIx::Class)" +# FIXME - this is a kludge in place of proper MDV testing. For the time +# being simply use the minimum versions of our DBI/DBDstack, to avoid +# fuckups like 0.08260 (went unnoticed for 5 months) +if [[ "$POISON_ENV" = "true" ]] ; then - # FIXME - this is a kludge in place of proper MDV testing. For the time - # being simply use the minimum versions of our DBI/DBDstack, to avoid - # fuckups like 0.08260 (went unnoticed for 5 months) - # # use url-spec for DBI due to https://github.com/miyagawa/cpanminus/issues/328 if perl -M5.013003 -e1 &>/dev/null ; then # earlier DBI will not compile without PERL_POLLUTE which was gone in 5.14 @@ -77,31 +48,14 @@ if [[ "$CLEANTEST" = "true" ]]; then # So instead we still use our stock (possibly old) CPAN, and add some # handholding - if [[ "$DEVREL_DEPS" == "true" ]] ; then - # We are not "quite ready" for SQLA 1.99, do not consider it - # - installdeps 'SQL::Abstract~<1.99' - - else - - if ! CPAN_is_sane ; then - # no configure_requires - we will need the usual suspects anyway - # without pre-installing these in one pass things like extract_prereqs won't work - installdeps ExtUtils::MakeMaker ExtUtils::CBuilder Module::Build - fi - - # FIXME - temporary until 1.46 comes out / RT#99747 is fixed - # insufficient testing of 5.8.3, ned older DBD::SQlite, ribasushi-- - if ! perl -M5.008004 -e 1 &>/dev/null ; then - installdeps DBI I/IS/ISHIGAKI/DBD-SQLite-1.42.tar.gz - fi - + if [[ "$DEVREL_DEPS" != "true" ]] && ! CPAN_is_sane ; then + # no configure_requires - we will need the usual suspects anyway + # without pre-installing these in one pass things like extract_prereqs won't work + installdeps ExtUtils::MakeMaker ExtUtils::CBuilder Module::Build fi else # we will be running all dbic tests - preinstall lots of stuff, run basic tests - # using SQLT and set up whatever databases necessary - export DBICTEST_SQLT_DEPLOY=1 # do the preinstall in several passes to minimize amount of cross-deps installing # multiple times, and to avoid module re-architecture breaking another install diff --git a/maint/travis-ci_scripts/40_script.bash b/maint/travis-ci_scripts/40_script.bash index cda7779..2edcafd 100755 --- a/maint/travis-ci_scripts/40_script.bash +++ b/maint/travis-ci_scripts/40_script.bash @@ -1,6 +1,9 @@ #!/bin/bash -if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi +# this file is executed in a subshell - set up the common stuff +source maint/travis-ci_scripts/common.bash + +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then exit 0 ; fi run_harness_tests() { local -x HARNESS_OPTIONS=c:j$NUMTHREADS diff --git a/maint/travis-ci_scripts/50_after_failure.bash b/maint/travis-ci_scripts/50_after_failure.bash index abfd2b9..f087d6f 100755 --- a/maint/travis-ci_scripts/50_after_failure.bash +++ b/maint/travis-ci_scripts/50_after_failure.bash @@ -1,10 +1,12 @@ #!/bin/bash -# !!! Nothing here will be executed !!! -# The source-line calling this script is commented out in .travis.yml +# this file is executed in a subshell - set up the common stuff +source maint/travis-ci_scripts/common.bash -if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then exit 0 ; fi -echo_err "Nothing to do" +echo_err " +$(ci_vm_state_text) -return 0 +=== dmesg ringbuffer +$(sudo dmesg)" diff --git a/maint/travis-ci_scripts/50_after_success.bash b/maint/travis-ci_scripts/50_after_success.bash index fc94fce..5571e2b 100755 --- a/maint/travis-ci_scripts/50_after_success.bash +++ b/maint/travis-ci_scripts/50_after_success.bash @@ -1,6 +1,9 @@ #!/bin/bash -if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi +# this file is executed in a subshell - set up the common stuff +source maint/travis-ci_scripts/common.bash + +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then exit 0 ; fi if [[ "$CLEANTEST" != "true" ]] ; then parallel_installdeps_notest $(perl -Ilib -MDBIx::Class -e 'print join " ", keys %{DBIx::Class::Optional::Dependencies->req_list_for("dist_dir")}') diff --git a/maint/travis-ci_scripts/60_after_script.bash b/maint/travis-ci_scripts/60_after_script.bash index abfd2b9..fb5b5fb 100755 --- a/maint/travis-ci_scripts/60_after_script.bash +++ b/maint/travis-ci_scripts/60_after_script.bash @@ -3,8 +3,9 @@ # !!! Nothing here will be executed !!! # The source-line calling this script is commented out in .travis.yml -if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi +# this file is executed in a subshell - set up the common stuff +source maint/travis-ci_scripts/common.bash -echo_err "Nothing to do" +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then exit 0 ; fi -return 0 +echo_err "Nothing to do" diff --git a/maint/travis-ci_scripts/common.bash b/maint/travis-ci_scripts/common.bash index 7e3a781..5af1818 100755 --- a/maint/travis-ci_scripts/common.bash +++ b/maint/travis-ci_scripts/common.bash @@ -1,9 +1,10 @@ #!/bin/bash +# "autodie" set -e TEST_STDERR_LOG=/tmp/dbictest.stderr -TIMEOUT_CMD="/usr/bin/timeout --kill-after=9.5m --signal=TERM 9m" +TIMEOUT_CMD="/usr/bin/timeout --kill-after=16m --signal=TERM 15m" echo_err() { echo "$@" 1>&2 ; } @@ -86,9 +87,6 @@ apt_install() { # flatten pkgs="$@" - # Need to do this at every step, the sources list may very well have changed - run_or_err "Updating APT available package list" "sudo apt-get update" - run_or_err "Installing Debian APT packages: $pkgs" "sudo apt-get install --allow-unauthenticated --no-install-recommends -y $pkgs" } diff --git a/maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf b/maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf new file mode 100644 index 0000000..770effa --- /dev/null +++ b/maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf @@ -0,0 +1,23 @@ +[mysqld] + +thread_cache_size = 0 + +# mysql >= 5.5.16 +#thread_pool_size = 1 + +bulk_insert_buffer_size = 0 +read_buffer_size = 32K +join_buffer_size = 128K +sort_buffer_size = 128K +table_definition_cache = 400 + +performance_schema = 0 + +query_cache_type = 0 +query_cache_size = 0 + +innodb_use_sys_malloc = 1 +innodb_buffer_pool_size = 1M + +key_buffer_size = 64K +myisam_sort_buffer_size = 128K diff --git a/t/lib/DBICTest/RunMode.pm b/t/lib/DBICTest/RunMode.pm index ab47d0c..f1e5544 100644 --- a/t/lib/DBICTest/RunMode.pm +++ b/t/lib/DBICTest/RunMode.pm @@ -198,7 +198,7 @@ sub is_author { sub is_smoker { return - ( ($ENV{TRAVIS}||'') eq 'true' ) + ( ($ENV{TRAVIS}||'') eq 'true' and ($ENV{TRAVIS_REPO_SLUG}||'') eq 'dbsrgits/dbix-class' ) || ( $ENV{AUTOMATED_TESTING} && ! $ENV{PERL5_CPANM_IS_RUNNING} && ! $ENV{RELEASE_TESTING} ) ;