From: Peter Rabbitson Date: Sat, 28 Mar 2015 07:22:48 +0000 (+0100) Subject: (travis) Multiple CI improvements lifted from blead X-Git-Tag: v0.082821~3 X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?p=dbsrgits%2FDBIx-Class.git;a=commitdiff_plain;h=cdda207bedb96c29134adbd3d6c63305d563c16f (travis) Multiple CI improvements lifted from blead --- diff --git a/.travis.yml b/.travis.yml index d339eaa..54ee2ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,27 +6,8 @@ # # * Minimum perl officially supported by DBIC is 5.8.3. This *includes* the # basic depchain. On failure either attempt to fix it or bring it to the -# attention of ribasushi. *DO NOT* disable 5.8 testing - it is here for a -# reason -# -# * The matrix is built from two main modes - CLEANTEST = [true|false]. -# - In the first case we test with minimal deps available, and skip everything -# listed in DBIC::OptDesps. The modules are installed with classic CPAN -# invocations and are *fully tested*. In other words we simulate what would -# happen if a user tried to install on a just-compiled virgin perl -# - Without CLEANTEST we bring the armada of RDBMS and install the maximum -# possible set of deps *without testing them*. This ensures we stay within -# a reasonable build-time and still run as many of our tests as possible -# -# * The perl builds and the DBIC tests run under NUMTHREADS number of threads. -# The testing of dependencies under CLEANTEST runs single-threaded, at least -# until we fix our entire dep-chain to safely pass under -j -# -# * The way .travis.yml is fed to the command controller is idiotic - it -# makes using multiline `bash -c` statements impossible. Therefore to -# aid readability (our travis logic is rather complex), the bulk of -# functionality is moved to scripts. More about the problem (and the -# WONTFIX "explanation") here: https://github.com/travis-ci/travis-ci/issues/497 +# attention of the maintainer. *DO NOT* disable 5.8 testing - it is here for +# a very good reason # # the entire run times out after 50 minutes, or after 5 minutes without # console output @@ -55,167 +36,291 @@ notifications: email: recipients: - ribasushi@cpan.org - # Temporary - if it proves to be too noisy, we'll shut it off - #- dbix-class-devel@lists.scsys.co.uk on_success: change on_failure: always -# FIXME - This stuff is not yet available for free OSS accounts, sadpanda -# First paragrah on http://about.travis-ci.org/docs/user/caching/ -#cache: -# apt: true -# directories: -# - /var/cache/apt/archives +addons: + apt: + packages: + - libapp-nopaste-perl + - net-tools +# This is probably a net-loss for setup etc - a bare 'C' will likely fare much better language: perl -perl: - - "5.20-extras" +# Currently not trying osx: https://github.com/travis-ci/travis-ci/issues/2314 +os: linux -env: - - CLEANTEST=false - - CLEANTEST=true NUMTHREADS=1 +# The defaults run under the more rapid container infra. The hardware is +# actually *much* slower, but the jobs start much faster, for more info see +# https://docs.travis-ci.com/user/ci-environment/#Virtualization-environments +# Combined with 'fast_finish' this will result in an "uh-oh" email as early +# as possible +dist: precise +sudo: false +env: CLEANTEST=true + +perl: + - "5.8" + - "5.22-extras" matrix: fast_finish: true + include: - # this particular perl is quite widespread - - perl: 5.8.8_thr_mb - env: - - CLEANTEST=true - - BREWOPTS="-Duseithreads -Dusemorebits" - - BREWVER=5.8.8 - # so is this one (test a sane CPAN.pm) - - perl: 5.12.4_thr_mb + # Same as the "master matrix" above, frozen under older dist/infrastructure + # In genereal it is strongly recommended to keep things on the older + # version indefinitely - there is little value in-depth smoking on + # more recent software stacks + - perl: "5.8" + sudo: required + dist: precise env: - - CLEANTEST=true - - BREWOPTS="-Duseithreads -Dusemorebits" - - BREWVER=5.12.4 + - CLEANTEST=false - # this is the perl suse ships - - perl: 5.10.0_thr_dbg + - perl: "5.22-extras" + sudo: required + dist: precise env: - - CLEANTEST=true - - BREWOPTS="-DDEBUGGING -Duseithreads" - - BREWVER=5.10.0 + - CLEANTEST=false - # CLEANTEST of minimum supported - - perl: 5.8.3_nt_mb + # CLEANTEST of minimum supported with non-tracing poisoning, single thread (hence the sudo) + - perl: "5.8.3_nt_mb" + sudo: required + dist: precise env: + - VCPU_USE=1 - CLEANTEST=true + - POISON_ENV=true + - DBIC_TRACE_PROFILE=console_monochrome + - BREWVER=5.8.3 - BREWOPTS="-Dusemorebits" + + # Full Test of minimum supported without threads with plain poisoned trace + - perl: "5.8.3_nt" + sudo: required + dist: precise + # run these under legacy - great simulation of low memory env + group: legacy + env: + - CLEANTEST=false + - POISON_ENV=true + - DBIC_TRACE=1 - BREWVER=5.8.3 - # Full Test of minimum supported with threads - - perl: 5.8.5_thr + # Full Test of minimum supported with threads with non-tracing poisoning + - perl: "5.8.5_thr" + sudo: required + dist: precise + # run these under legacy - great simulation of low memory env + group: legacy env: - CLEANTEST=false - - BREWOPTS="-Duseithreads" - - BREWVER=5.8.5 + - POISON_ENV=true - DBIC_TRACE_PROFILE=console + - BREWVER=5.8.5 + - BREWOPTS="-Duseithreads" - # Full Test of minimum supported without threads - - perl: 5.8.3_nt + # CLEANTEST of solaris-like perl with non-tracing poisoning + - perl: "5.8.4_nt" + sudo: false + dist: precise env: - - CLEANTEST=false - - BREWOPTS="" - - BREWVER=5.8.3 - - DBIC_TRACE_PROFILE=console_monochrome + - CLEANTEST=true + - POISON_ENV=true + - DBIC_TRACE_PROFILE=console + - BREWVER=5.8.4 - ### - # some permutations of tracing and envvar poisoning + # CLEANTEST: this particular perl is quite widespread + - perl: "5.8.8_thr" + sudo: false + dist: precise + env: + - CLEANTEST=true + - BREWVER=5.8.8 + - BREWOPTS="-Duseithreads" - - perl: 5.16.3_thr_mb + # CLEANTEST: this is the perl suse ships, with env poisoning + - perl: "5.10.0_thr_dbg" + sudo: false + dist: precise env: - - CLEANTEST=false + - CLEANTEST=true - POISON_ENV=true - - DBIC_TRACE=1 - - DBIC_MULTICREATE_DEBUG=0 + - BREWVER=5.10.0 + - BREWOPTS="-DDEBUGGING -Duseithreads" + + # CLEANTEST: this one is in a number of debian-based LTS (test a sane CPAN.pm, single thread) + - perl: "5.14.2_thr_mb" + sudo: required + dist: precise + env: + - VCPU_USE=1 + - CLEANTEST=true + - BREWVER=5.14.2 - BREWOPTS="-Duseithreads -Dusemorebits" - - BREWVER=5.16.3 - - perl: 5.20-extras + ### + # some permutations of tracing and envvar poisoning + + - perl: "5.12.3_thr" + sudo: false + dist: precise env: - - CLEANTEST=false + - CLEANTEST=true - POISON_ENV=true + - DBIC_TRACE=1 + - DBIC_MULTICREATE_DEBUG=1 + - DBIC_STORAGE_RETRY_DEBUG=1 - DBIC_TRACE_PROFILE=console + - BREWVER=5.12.3 + - BREWOPTS="-Duseithreads" - - perl: 5.8 + - perl: "5.16.3_thr_mb" + sudo: required + dist: precise env: - - CLEANTEST=true + - CLEANTEST=false - POISON_ENV=true - DBIC_TRACE=1 - - DBIC_TRACE_PROFILE=console + - BREWVER=5.16.3 + - BREWOPTS="-Duseithreads -Dusemorebits" - - perl: 5.18-extras + - perl: "5.18-extras" + sudo: required + # explicit new infra spec preparing for a future forced upgrade + dist: trusty env: - CLEANTEST=false - POISON_ENV=true - DBIC_TRACE=1 - DBIC_TRACE_PROFILE=console_monochrome - - DBIC_MULTICREATE_DEBUG=0 + - DBICTEST_VIA_REPLICATED=0 + - DBICTEST_VERSION_WARNS_INDISCRIMINATELY=1 ### # Start of the allow_failures block - # old threaded with blead CPAN - - perl: devcpan_5.8.7_thr + # threaded oldest possible with blead CPAN + - perl: "devcpan_5.8.1_thr_mb" + sudo: false + dist: precise env: - CLEANTEST=true - - BREWOPTS="-Duseithreads" - - BREWVER=5.8.7 - DEVREL_DEPS=true + - BREWVER=5.8.1 + - BREWOPTS="-Duseithreads -Dusemorebits" - # 5.10.0 threaded with blead CPAN - - perl: devcpan_5.10.0_thr_mb + # oldest possible with blead CPAN with poisoning and plain trace + - perl: "devcpan_5.8.1" + sudo: false + dist: precise env: - CLEANTEST=true - - BREWOPTS="-Duseithreads -Dusemorebits" - - BREWVER=5.10.0 - DEVREL_DEPS=true + - POISON_ENV=true + - DBIC_TRACE=1 + - DBICTEST_VERSION_WARNS_INDISCRIMINATELY=1 + - BREWVER=5.8.1 + + # 5.8.3 with blead CPAN + - perl: "devcpan_5.8.3_mb" + sudo: required + # explicit new infra spec preparing for a future forced upgrade + dist: trusty + env: + - CLEANTEST=false + - DEVREL_DEPS=true + - BREWVER=5.8.3 + - BREWOPTS="-Dusemorebits" - # 5.12.2 with blead CPAN - - perl: devcpan_5.12.2_thr + # 5.8.7 threaded with blead CPAN with non-tracing poisoning + - perl: "devcpan_5.8.7_thr" + sudo: false + dist: precise env: - CLEANTEST=true + - DEVREL_DEPS=true + - POISON_ENV=true + - BREWVER=5.8.7 - BREWOPTS="-Duseithreads" - - BREWVER=5.12.2 + + # 5.8.8 threaded MB (exercises P5#72210) + - perl: "devcpan_5.8.8_thr_mb" + sudo: false + dist: precise + env: + - CLEANTEST=true + - DBICTEST_VERSION_WARNS_INDISCRIMINATELY=1 - DEVREL_DEPS=true + - BREWVER=5.8.8 + - BREWOPTS="-Duseithreads -Dusemorebits" - # recentish threaded stable with blead CPAN - - perl: devcpan_5.18.2_thr_mb + # 5.10.0 threaded with blead CPAN + - perl: "devcpan_5.10.0_thr_mb" + sudo: false + dist: precise env: - - CLEANTEST=false + - CLEANTEST=true + - DEVREL_DEPS=true + - BREWVER=5.10.0 - BREWOPTS="-Duseithreads -Dusemorebits" - - BREWVER=5.18.2 + + # 5.12.1 with blead CPAN + - perl: "devcpan_5.12.1_thr" + sudo: false + dist: precise + env: + - CLEANTEST=true - DEVREL_DEPS=true + - BREWVER=5.12.1 + - BREWOPTS="-Duseithreads" - # bleadperl with stock CPAN, full depchain test - - perl: bleadperl + # bleadperl with stock CPAN, full depchain test with non-tracing poisoning, single thread + - perl: "bleadperl" + sudo: required + dist: precise env: + - VCPU_USE=1 - CLEANTEST=true + - POISON_ENV=true - BREWVER=blead # bleadperl with blead CPAN - - perl: devcpan_bleadperl_thr_mb + - perl: "devcpan_bleadperl_thr_mb" + sudo: required + # explicitly do not specify dist - see what the default does env: - CLEANTEST=false - - BREWOPTS="-Duseithreads -Dusemorebits" - - BREWVER=blead - DEVREL_DEPS=true + - BREWVER=blead + - BREWOPTS="-Duseithreads -Dusemorebits" + # CLEANTEST of http://schplog.schmorp.de/2015-06-06-a-stable-perl.html with non-tracing poisoning + - perl: "schmorp_stableperl_thr_mb" + sudo: false + dist: precise + env: + - CLEANTEST=true + - POISON_ENV=true + - BREWVER=schmorp_stableperl + - BREWOPTS="-Duseithreads -Dusemorebits" # which ones of the above can fail allow_failures: # these run with various dev snapshots - allowed to fail + - perl: devcpan_5.8.1_thr_mb + - perl: devcpan_5.8.1 + - perl: devcpan_5.8.3_mb - perl: devcpan_5.8.7_thr + - perl: devcpan_5.8.8_thr_mb - perl: devcpan_5.10.0_thr_mb - - perl: devcpan_5.12.2_thr - - perl: devcpan_5.18.2_thr_mb + - perl: devcpan_5.12.1_thr - perl: bleadperl - perl: devcpan_bleadperl_thr_mb + - perl: schmorp_stableperl_thr_mb ### @@ -257,13 +362,18 @@ before_script: # need to invoke the after_failure script manually # because 'after_failure' runs only after 'script' fails # - - maint/getstatus maint/travis-ci_scripts/30_before_script.bash || ( maint/travis-ci_scripts/50_after_failure.bash && /bin/false ) + - maint/getstatus maint/travis-ci_scripts/30_before_script.bash script: # Run actual tests # - maint/getstatus maint/travis-ci_scripts/40_script.bash +### +### Set -e back, work around https://github.com/travis-ci/travis-ci/issues/3533 +### + - set -e + after_success: # Check if we can assemble a dist properly if not in CLEANTEST # diff --git a/AUTHORS b/AUTHORS index 88b0663..9e2bbff 100644 --- a/AUTHORS +++ b/AUTHORS @@ -75,6 +75,7 @@ felliott: Fitz Elliott freetime: Bill Moseley frew: Arthur Axel "fREW" Schmidt gbjk: Gareth Kirwan +geotheve: Georgina Thevenet Getty: Torsten Raudssus goraxe: Gordon Irving gphat: Cory G Watson diff --git a/maint/travis-ci_scripts/10_before_install.bash b/maint/travis-ci_scripts/10_before_install.bash index 356c17c..80315a0 100755 --- a/maint/travis-ci_scripts/10_before_install.bash +++ b/maint/travis-ci_scripts/10_before_install.bash @@ -1,9 +1,24 @@ #!/bin/bash -# Stop pre-started RDBMS and sync for some settle time -run_or_err "Stopping MySQL" "sudo /etc/init.d/mysql stop" -run_or_err "Stopping PostgreSQL" "sudo /etc/init.d/postgresql stop || /bin/true" -/bin/sync +export SHORT_CIRCUIT_SMOKE + +if have_sudo ; then + + # Stop pre-started RDBMS, move their data back to disk (save RAM) + # sync for some settle time (not available on all platforms) + for d in mysql postgresql ; do + # maybe not even running + run_or_err "Stopping $d" "sudo /etc/init.d/$d stop || /bin/true" + + # no longer available on newer build systems + if [[ -d /var/ramfs/$d ]] ; then + sudo rm -rf /var/lib/$d + sudo mv /var/ramfs/$d /var/lib/ + sudo ln -s /var/lib/$d /var/ramfs/$d + fi + done + /bin/sync +fi # Sanity check VM before continuing echo " @@ -25,55 +40,82 @@ Under Travis this state usually results in a failed build. Short-circuiting buildjob to avoid false negatives, please restart it manually. =============================================================================" + +# pull requests are always scrutinized after the fact anyway - run a +# a simpler matrix +elif [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then + if [[ -n "$BREWVER" ]]; then + # just don't brew anything + SHORT_CIRCUIT_SMOKE=1 + else + # running PRs with 1 thread is non-sensical + VCPU_USE="" + fi fi if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi -# Different boxes we run on may have different amount of hw threads -# Hence why we need to query -# Originally we used to read /sys/devices/system/cpu/online -# but it is not available these days (odd). Thus we fall to -# the alwas-present /proc/cpuinfo -# The oneliner is a tad convoluted - basicaly what we do is -# slurp the entire file and get the index off the last -# `processor : XX` line -# -# We also divide the result by a factor, otherwise the travis VM gets -# overloaded (the amount of available swap is just TOOOO damn small) -if [[ -z "$NUMTHREADS" ]] ; then - export NUMTHREADS="$(( ( $(perl -0777 -n -e 'print (/ (?: .+ ^ processor \s+ : \s+ (\d+) ) (?! ^ processor ) /smx)' < /proc/cpuinfo) + 1 ) / 3 ))" -fi +# Previously we were going off the OpenVZ vcpu count and dividing by 3 +# With the new infrastructure, somply go with "something high" +export VCPU_AVAILABLE=10 -export CACHE_DIR="/tmp/poormanscache" +if [[ -z "$VCPU_USE" ]] ; then + export VCPU_USE="$VCPU_AVAILABLE" +fi -# these will be installed no matter what, also some extras unless CLEANTEST -common_packages="libapp-nopaste-perl tree" -if [[ "$CLEANTEST" = "true" ]]; then +if [[ "$CLEANTEST" != "true" ]]; then - apt_install $common_packages + if [[ -z "$(tail -n +2 /proc/swaps)" ]] ; then + run_or_err "Configuring swap (for Oracle)" \ + "sudo bash -c 'dd if=/dev/zero of=/swap.img bs=256M count=5 && chmod 600 /swap.img && mkswap /swap.img && swapon /swap.img'" + fi -else + export CACHE_DIR="/tmp/poormanscache" # # FIXME these debconf lines should automate the firebird config but do not :((( sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/enabled\tboolean\ttrue" | debconf-set-selections' sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/sysdba_password/new_password\tpassword\t123" | debconf-set-selections' + # these APT sources do not mean anything to us anyway + sudo rm -rf /etc/apt/sources.list.d/* + + # the actual package is built for lucid, installs fine on both precise and trusty + sudo bash -c 'echo "deb http://archive.canonical.com/ubuntu precise partner" >> /etc/apt/sources.list' + + # never installed, this looks like trusty + if [[ ! -d /var/lib/mysql ]] ; then + sudo dpkg --add-architecture i386 + extra_debs="$extra_debs postgresql mysql-server" + fi + run_or_err "Updating APT sources" "sudo apt-get update" - apt_install $common_packages libmysqlclient-dev memcached firebird2.5-super firebird2.5-dev unixodbc-dev expect - run_or_err "Cloning poor man's cache from github" "git clone --depth=1 --single-branch --branch=oracle/10.2.0 https://github.com/poortravis/poormanscache.git $CACHE_DIR && $CACHE_DIR/reassemble" - run_or_err "Installing OracleXE manually from deb" "sudo dpkg -i $CACHE_DIR/apt_cache/oracle-xe_10.2.0.1-1.1_i386.deb || sudo bash -c 'source maint/travis-ci_scripts/common.bash && apt_install -f'" + apt_install $extra_debs libmysqlclient-dev memcached firebird2.5-super firebird2.5-dev unixodbc-dev expect + + # need to stop them again, in case we installed them above (trusty) + for d in mysql postgresql ; do + run_or_err "Stopping $d" "sudo /etc/init.d/$d stop || /bin/true" + done + + run_or_err "Cloning poor man's cache from github" "git clone --depth=1 --single-branch --branch=oracle/10.2.0 https://github.com/poormanscache/poormanscache.git $CACHE_DIR && $CACHE_DIR/reassemble" + run_or_err "Installing OracleXE manually from deb" \ + "sudo dpkg -i $CACHE_DIR/apt_cache/bc-multiarch-travis_1.0_all.deb $CACHE_DIR/apt_cache/oracle-xe_10.2.0.1-1.1_i386.deb || sudo bash -c 'source maint/travis-ci_scripts/common.bash && apt_install -f'" ### config memcached run_or_err "Starting memcached" "sudo /etc/init.d/memcached start" export DBICTEST_MEMCACHED=127.0.0.1:11211 ### config mysql - run_or_err "Installing minimizing MySQL config" "sudo cp maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf /etc/mysql/conf.d/ && sudo chmod 644 /etc/mysql/conf.d/*.cnf" + run_or_err "Installing minimizing MySQL config" "\ + sudo bash -c 'rm /var/lib/mysql/ib*' \ + && sudo cp maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf /etc/mysql/conf.d/ \ + && sudo chmod 644 /etc/mysql/conf.d/*.cnf \ + " + run_or_err "Starting MySQL" "sudo /etc/init.d/mysql start" - run_or_err "Creating MySQL TestDB" "mysql -e 'create database dbic_test;'" + run_or_err "Creating MySQL TestDB" "mysql -u root -e 'create database dbic_test;'" export DBICTEST_MYSQL_DSN='dbi:mysql:database=dbic_test;host=127.0.0.1' export DBICTEST_MYSQL_USER=root @@ -91,33 +133,43 @@ else send "\177\177\177\177yes\r" expect "Password for SYSDBA" send "123\r" - sleep 1 + sleep 2 expect eof ' # creating testdb # FIXME - this step still fails from time to time >:((( # has to do with the FB reconfiguration I suppose # for now if it fails twice - simply skip FB testing - for i in 1 2 ; do + for i in 1 2 3 ; do run_or_err "Re-configuring Firebird" " sync + sleep 5 DEBIAN_FRONTEND=text sudo expect -c '$EXPECT_FB_SCRIPT' - sleep 1 - sync - # restart the server for good measure - sudo /etc/init.d/firebird2.5-super stop || true - sleep 1 - sync - sudo /etc/init.d/firebird2.5-super start - sleep 1 - sync " if run_or_err "Creating Firebird TestDB" \ "echo \"CREATE DATABASE '/var/lib/firebird/2.5/data/dbic_test.fdb';\" | sudo isql-fb -u sysdba -p 123" then + run_or_err "Fetching and building Firebird ODBC driver" ' + cd "$(mktemp -d)" + wget -qO- http://sourceforge.net/projects/firebird/files/firebird-ODBC-driver/2.0.2-Release/OdbcFb-Source-2.0.2.153.gz/download | tar -zx + cd Builds/Gcc.lin + perl -p -i -e "s|/usr/lib64|/usr/lib/x86_64-linux-gnu|g" ../makefile.environ + make -f makefile.linux + sudo make -f makefile.linux install + ' + + sudo bash -c 'cat >> /etc/odbcinst.ini' <<< " +[Firebird] +Description = InterBase/Firebird ODBC Driver +Driver = /usr/lib/x86_64-linux-gnu/libOdbcFb.so +Setup = /usr/lib/x86_64-linux-gnu/libOdbcFb.so +Threading = 1 +FileUsage = 1 +" + export DBICTEST_FIREBIRD_DSN=dbi:Firebird:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb export DBICTEST_FIREBIRD_USER=SYSDBA export DBICTEST_FIREBIRD_PASS=123 @@ -126,6 +178,10 @@ else export DBICTEST_FIREBIRD_INTERBASE_USER=SYSDBA export DBICTEST_FIREBIRD_INTERBASE_PASS=123 + export DBICTEST_FIREBIRD_ODBC_DSN="dbi:ODBC:Driver=Firebird;Dbname=/var/lib/firebird/2.5/data/dbic_test.fdb" + export DBICTEST_FIREBIRD_ODBC_USER=SYSDBA + export DBICTEST_FIREBIRD_ODBC_PASS=123 + break fi @@ -194,10 +250,3 @@ else export ORACLE_HOME="$CACHE_DIR/ora_instaclient/x86-64/oracle_instaclient_10.2.0.5.0" fi - -# The debian package is oddly broken - uses a /bin/env based shebang -# so nothing works under a brew, fixed in libapp-nopaste-perl 0.92-3 -# http://changelogs.ubuntu.com/changelogs/pool/universe/liba/libapp-nopaste-perl/libapp-nopaste-perl_0.96-1/changelog -# -# Since the vm runs an old version of umbongo fix things ourselves -sudo /usr/bin/perl -p -i -e 's|#!/usr/bin/env perl|#!/usr/bin/perl|' $(which nopaste) diff --git a/maint/travis-ci_scripts/20_install.bash b/maint/travis-ci_scripts/20_install.bash index 3535407..2d6fbf2 100755 --- a/maint/travis-ci_scripts/20_install.bash +++ b/maint/travis-ci_scripts/20_install.bash @@ -9,31 +9,34 @@ CPAN_MIRROR="http://cpan.metacpan.org/" PERL_CPANM_OPT="$PERL_CPANM_OPT --mirror $CPAN_MIRROR" # do not set PERLBREW_CPAN_MIRROR - not all backpan-like mirrors have the perl tarballs -export PERL_MM_USE_DEFAULT=1 PERL_MM_NONINTERACTIVE=1 PERL_AUTOINSTALL_PREFER_CPAN=1 HARNESS_TIMER=1 MAKEFLAGS="-j$NUMTHREADS" +export PERL_MM_USE_DEFAULT=1 PERL_MM_NONINTERACTIVE=1 PERL_AUTOINSTALL_PREFER_CPAN=1 HARNESS_TIMER=1 MAKEFLAGS="-j$VCPU_USE" # try CPAN's latest offering if requested if [[ "$DEVREL_DEPS" == "true" ]] ; then PERL_CPANM_OPT="$PERL_CPANM_OPT --dev" - # FIXME inline-upgrade cpanm, work around https://github.com/travis-ci/travis-ci/issues/1477 - cpanm_loc="$(which cpanm)" - run_or_err "Upgrading cpanm ($cpanm_loc) to latest stable" \ - "wget -q -O $cpanm_loc cpanmin.us && chmod a+x $cpanm_loc" fi # Fixup CPANM_OPT to behave more like a traditional cpan client export PERL_CPANM_OPT="--verbose --no-interactive --no-man-pages $( echo $PERL_CPANM_OPT | sed 's/--skip-satisfied//' )" if [[ -n "$BREWVER" ]] ; then + # since perl 5.14 a perl can safely be built concurrently with -j$large # (according to brute force testing and my power bill) - if [[ "$BREWVER" == "blead" ]] || perl -Mversion -e "exit !!(version->new(q($BREWVER)) < 5.014)" ; then - perlbrew_jopt="$NUMTHREADS" + if [[ "$BREWVER" =~ [A-Za-z] ]] || perl -Mversion -e "exit !!(version->new(q($BREWVER)) < 5.014)" ; then + perlbrew_jopt="$VCPU_USE" + fi + + BREWSRC="$BREWVER" + + if [[ "$BREWVER" == "schmorp_stableperl" ]] ; then + BREWSRC="http://stableperl.schmorp.de/dist/stableperl-5.22.0-1.001.tar.gz" fi run_or_err "Compiling/installing Perl $BREWVER (without testing, using ${perlbrew_jopt:-1} threads, may take up to 5 minutes)" \ - "perlbrew install --as $BREWVER --notest --noman --verbose $BREWOPTS -j${perlbrew_jopt:-1} $BREWVER" + "perlbrew install --as $BREWVER --notest --noman --verbose $BREWOPTS -j${perlbrew_jopt:-1} $BREWSRC" # can not do 'perlbrew uss' in the run_or_err subshell above, or a $() # furthermore `perlbrew use` returns 0 regardless of whether the perl is @@ -48,25 +51,8 @@ if [[ -n "$BREWVER" ]] ; then # no brewver - this means a travis perl, which means we want to clean up # the presently installed libs -# Idea stolen from -# https://github.com/kentfredric/Dist-Zilla-Plugin-Prereqs-MatchInstalled-All/blob/master/maint-travis-ci/sterilize_env.pl elif [[ "$CLEANTEST" == "true" ]] && [[ "$POISON_ENV" != "true" ]] ; then - - echo_err "$(tstamp) Cleaning precompiled Travis-Perl" - perl -MConfig -MFile::Find -e ' - my $sitedirs = { - map { $Config{$_} => 1 } - grep { $_ =~ /site(lib|arch)exp$/ } - keys %Config - }; - find({ bydepth => 1, no_chdir => 1, follow_fast => 1, wanted => sub { - ! $sitedirs->{$_} and ( -d _ ? rmdir : unlink ) - } }, keys %$sitedirs ) - ' - - echo_err "Post-cleanup contents of sitelib of the pre-compiled Travis-Perl $TRAVIS_PERL_VERSION:" - echo_err "$(tree $(perl -MConfig -e 'print $Config{sitelib_stem}'))" - echo_err + purge_sitelib fi # configure CPAN.pm - older versions go into an endless loop @@ -82,16 +68,42 @@ CPAN_CFG_SCRIPT=" " run_or_err "Configuring CPAN.pm" "perl -e '$CPAN_CFG_SCRIPT'" + +# These envvars are always set, more *maybe* below +export DBIC_SHUFFLE_UNORDERED_RESULTSETS=1 + +# bogus nonexisting DBI_* +export DBI_DSN="dbi:ODBC:server=NonexistentServerAddress" +export DBI_DRIVER="ADO" + +# some people do in fact set this - boggle!!! +# it of course won't work before 5.8.4 +if perl -M5.008004 -e 1 &>/dev/null ; then + export PERL_STRICTURES_EXTRA=1 +fi + + # poison the environment if [[ "$POISON_ENV" = "true" ]] ; then - # in addition to making sure tests do not rely on implicid order of - # returned results, look through lib, find all mentioned ENVvars and - # set them to true and see if anything explodes - for var in \ + # look through lib, find all mentioned DBIC* ENVvars and set them to true and see if anything explodes + toggle_booleans=( $( grep -ohP '\bDBIC_[0-9_A-Z]+' -r lib/ --exclude-dir Optional | sort -u | grep -vP '^(DBIC_TRACE(_PROFILE)?|DBIC_.+_DEBUG)$' ) ) + + # some extra pollutants + toggle_booleans+=( \ + DBICTEST_ASSERT_NO_SPURIOUS_EXCEPTION_ACTION \ DBICTEST_SQLITE_USE_FILE \ + DBICTEST_RUN_ALL_TESTS \ DBICTEST_SQLITE_REVERSE_DEFAULT_ORDER \ - $(grep -P '\$ENV\{' -r lib/ --exclude-dir Optional | grep -oP '\bDBIC\w+' | sort -u | grep -v DBIC_TRACE) + ) + + # if we have Moose - try to run everything under replicated + # FIXME - when switching to Moo kill this + if [[ "$CLEANTEST" != "true" ]] && perl -M5.008003 -e 1 &>/dev/null ; then + toggle_booleans+=( DBICTEST_VIA_REPLICATED ) + fi + + for var in "${toggle_booleans[@]}" do if [[ -z "${!var}" ]] ; then export $var=1 @@ -99,13 +111,6 @@ if [[ "$POISON_ENV" = "true" ]] ; then fi done - # bogus nonexisting DBI_* - export DBI_DSN="dbi:ODBC:server=NonexistentServerAddress" - export DBI_DRIVER="ADO" - - # some people do in fact set this - boggle!!! - export PERL_STRICTURES_EXTRA=1 - # emulate a local::lib-like env # trick cpanm into executing true as shell - we just need the find+unpack run_or_err "Downloading latest stable DBIC from CPAN" \ @@ -124,3 +129,7 @@ if [[ "$CLEANTEST" != "true" ]] ; then export DBICTEST_SQLT_DEPLOY=1 fi + +# FIXME - work around https://github.com/miyagawa/cpanminus/issues/462 +# seriously... +perl -p -i -e 's/\blocal\$self->\{notest\}=1;//' $(which cpanm) diff --git a/maint/travis-ci_scripts/30_before_script.bash b/maint/travis-ci_scripts/30_before_script.bash index 651acd2..fae39e6 100755 --- a/maint/travis-ci_scripts/30_before_script.bash +++ b/maint/travis-ci_scripts/30_before_script.bash @@ -5,13 +5,20 @@ source maint/travis-ci_scripts/common.bash if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then exit 0 ; fi +# The prereq-install stage will not work with both POISON and DEVREL +# DEVREL wins +if [[ "$DEVREL_DEPS" = "true" ]] ; then + export POISON_ENV="" +fi + # FIXME - this is a kludge in place of proper MDV testing. For the time # being simply use the minimum versions of our DBI/DBDstack, to avoid # fuckups like 0.08260 (went unnoticed for 5 months) if [[ "$POISON_ENV" = "true" ]] ; then # use url-spec for DBI due to https://github.com/miyagawa/cpanminus/issues/328 - if perl -M5.013003 -e1 &>/dev/null ; then + if [[ "$CLEANTEST" != "true" ]] || perl -M5.013003 -e1 &>/dev/null ; then + # the fulltest may re-upgrade DBI, be conservative only on cleantests # earlier DBI will not compile without PERL_POLLUTE which was gone in 5.14 parallel_installdeps_notest T/TI/TIMB/DBI-1.614.tar.gz else @@ -19,12 +26,29 @@ if [[ "$POISON_ENV" = "true" ]] ; then fi # Test both minimum DBD::SQLite and minimum BigInt SQLite + # reverse the logic from above for this (low on full, higher on clean) if [[ "$CLEANTEST" = "true" ]]; then parallel_installdeps_notest DBD::SQLite@1.37 else parallel_installdeps_notest DBD::SQLite@1.29 fi + # also try minimal tested installs *without* a compiler + if [[ "$CLEANTEST" = "true" ]]; then + + # Clone and P::S::XS are both bugs + # File::Spec can go away as soon as I dump Path::Class + # File::Path is there because of RT#107392 (sigh) + # List::Util can be excised after that as well (need to make my own max() routine for older perls) + + installdeps Sub::Name Clone Package::Stash::XS \ + $( perl -MFile::Spec\ 3.26 -e1 &>/dev/null || echo "File::Path File::Spec" ) \ + $( perl -MList::Util\ 1.16 -e1 &>/dev/null || echo "List::Util" ) + + mkdir -p "$HOME/bin" # this is already in $PATH, just doesn't exist + run_or_err "Linking ~/bin/cc to /bin/false - thus essentially BREAKING the C compiler" \ + "ln -s /bin/false $HOME/bin/cc" + fi fi if [[ "$CLEANTEST" = "true" ]]; then @@ -48,10 +72,13 @@ if [[ "$CLEANTEST" = "true" ]]; then # So instead we still use our stock (possibly old) CPAN, and add some # handholding - if [[ "$DEVREL_DEPS" != "true" ]] && ! CPAN_is_sane ; then + if [[ "$DEVREL_DEPS" = "true" ]] ; then + # FIXME - work around RT#110882, sigh... + perl -Mversion\ 0.87 -e 1 &>/dev/null || installdeps version@0.9912 + elif ! CPAN_is_sane ; then # no configure_requires - we will need the usual suspects anyway - # without pre-installing these in one pass things like extract_prereqs won't work - installdeps ExtUtils::MakeMaker ExtUtils::CBuilder Module::Build + # without pre-installing these in one pass things won't yet work + installdeps Module::Build fi else @@ -62,24 +89,29 @@ else # (e.g. once Carp is upgraded there's no more Carp::Heavy, # while a File::Path upgrade may cause a parallel EUMM run to fail) # - parallel_installdeps_notest ExtUtils::MakeMaker parallel_installdeps_notest File::Path parallel_installdeps_notest Carp parallel_installdeps_notest Module::Build - parallel_installdeps_notest File::Spec Data::Dumper Module::Runtime + parallel_installdeps_notest File::Spec Module::Runtime parallel_installdeps_notest Test::Exception Encode::Locale Test::Fatal parallel_installdeps_notest Test::Warn B::Hooks::EndOfScope Test::Differences HTTP::Status parallel_installdeps_notest Test::Pod::Coverage Test::EOL Devel::GlobalDestruction Sub::Name MRO::Compat Class::XSAccessor URI::Escape HTML::Entities - parallel_installdeps_notest YAML LWP Class::Trigger JSON::XS DateTime::Format::Builder Class::Accessor::Grouped Package::Variant - parallel_installdeps_notest SQL::Abstract Moose Module::Install JSON SQL::Translator File::Which + parallel_installdeps_notest YAML LWP Class::Trigger DateTime::Format::Builder Class::Accessor::Grouped Package::Variant + parallel_installdeps_notest SQL::Abstract Moose Module::Install@1.15 JSON SQL::Translator File::Which Class::DBI::Plugin git://github.com/dbsrgits/perl-pperl.git + # the official version is very much outdated and does not compile on 5.14+ + # use this rather updated source tree (needs to go to PAUSE): + # https://github.com/pilcrow/perl-dbd-interbase if [[ -n "$DBICTEST_FIREBIRD_INTERBASE_DSN" ]] ; then - # the official version is very much outdated and does not compile on 5.14+ - # use this rather updated source tree (needs to go to PAUSE): - # https://github.com/pilcrow/perl-dbd-interbase parallel_installdeps_notest git://github.com/dbsrgits/perl-dbd-interbase.git fi + # SCGI does not install under < 5.8.8 perls nor under parallel make + # FIXME: The 5.8.8 thing is likely fixable, something to do with + # #define speedy_new(s,n,t) Newx(s,n,t) + if perl -M5.008008 -e 1 &>/dev/null ; then + MAKEFLAGS="" bash -c "parallel_installdeps_notest git://github.com/dbsrgits/cgi-speedycgi.git" + fi fi # generate the makefile which will have different deps depending on @@ -88,94 +120,38 @@ run_or_err "Configure on current branch" "perl Makefile.PL" # install (remaining) dependencies, sometimes with a gentle push if [[ "$CLEANTEST" = "true" ]]; then - # we may need to prepend some stuff to that list - HARD_DEPS="$(echo $(make listdeps))" - -##### TEMPORARY WORKAROUNDS needed in case we will be using CPAN.pm - if [[ "$DEVREL_DEPS" != "true" ]] && ! CPAN_is_sane ; then - # combat dzillirium on harness-wide level, otherwise breakage happens weekly - echo_err "$(tstamp) Ancient CPAN.pm: engaging TAP::Harness::IgnoreNonessentialDzilAutogeneratedTests during dep install" - perl -MTAP::Harness\ 3.18 -e1 &>/dev/null || run_or_err "Upgrading TAP::Harness for HARNESS_SUBCLASS support" "cpan TAP::Harness" - export PERL5LIB="$(pwd)/maint/travis-ci_scripts/lib:$PERL5LIB" - export HARNESS_SUBCLASS="TAP::Harness::IgnoreNonessentialDzilAutogeneratedTests" - # sanity check, T::H does not report sensible errors when the subclass fails to load - perl -MTAP::Harness::IgnoreNonessentialDzilAutogeneratedTests -e1 - - # DBD::SQLite reasonably wants DBI at config time - perl -MDBI -e1 &>/dev/null || HARD_DEPS="DBI $HARD_DEPS" - - # this is a fucked CPAN - won't understand configure_requires of - # various pieces we may run into - # FIXME - need to get these off metacpan or something instead - HARD_DEPS="ExtUtils::Depends B::Hooks::OP::Check $HARD_DEPS" - - # FIXME - # parent is temporary due to Carp https://rt.cpan.org/Ticket/Display.html?id=88494 - HARD_DEPS="parent $HARD_DEPS" - - if CPAN_supports_BUILDPL ; then - # We will invoke a posibly MBT based BUILD-file, but we do not support - # configure requires. So we not only need to install MBT but its prereqs - # FIXME This is madness - HARD_DEPS="$(extract_prereqs Module::Build::Tiny) Module::Build::Tiny $HARD_DEPS" - else - # FIXME - # work around Params::Validate not having a Makefile.PL so really old - # toolchains can not figure out what the prereqs are ;( - # Need to do more research before filing a bug requesting Makefile inclusion - HARD_DEPS="$(extract_prereqs Params::Validate) $HARD_DEPS" - fi - fi -##### END TEMPORARY WORKAROUNDS - installdeps $HARD_DEPS + # we are doing a devrel pass - try to upgrade *everything* (we will be using cpanm so safe-ish) + if [[ "$DEVREL_DEPS" == "true" ]] ; then + + HARD_DEPS="$(make listalldeps | sort -R)" + + else + + HARD_DEPS="$(make listdeps | sort -R)" + +##### TEMPORARY WORKAROUNDS needed in case we will be using a fucked CPAN.pm + if ! CPAN_is_sane ; then + + # DBD::SQLite reasonably wants DBI at config time + perl -MDBI -e1 &>/dev/null || HARD_DEPS="DBI $HARD_DEPS" -### FIXME in case we set it earlier in a workaround - if [[ -n "$HARNESS_SUBCLASS" ]] ; then - - INSTALLDEPS_SKIPPED_TESTLIST=$(perl -0777 -e ' -my $curmod_re = qr{ -^ - (?: - \QBuilding and testing\E - | - [\x20\t]* CPAN\.pm: [^\n]*? (?i:build)\S* - ) - - [\x20\t]+ (\S+) -$}mx; - -my $curskip_re = qr{^ === \x20 \QSkipping nonessential autogenerated tests: \E([^\n]+) }mx; - -my (undef, @chunks) = (split qr/$curmod_re/, <>); -while (@chunks) { - my ($mod, $log) = splice @chunks, 0, 2; - print "!!! Skipped nonessential tests while installing $mod:\n\t$1\n" - if $log =~ $curskip_re; -} -' <<< "$LASTOUT") - - if [[ -n "$INSTALLDEPS_SKIPPED_TESTLIST" ]] ; then - POSTMORTEM="$POSTMORTEM$( - echo - echo "The following non-essential tests were skipped during deps installation" - echo "=============================================================" - echo "$INSTALLDEPS_SKIPPED_TESTLIST" - echo "=============================================================" - echo - )" fi - unset HARNESS_SUBCLASS +##### END TEMPORARY WORKAROUNDS fi + installdeps $HARD_DEPS + else - parallel_installdeps_notest "$(make listdeps)" + + parallel_installdeps_notest "$(make listdeps | sort -R)" + fi echo_err "$(tstamp) Dependency installation finished" -# this will display list of available versions -perl Makefile.PL + +run_or_err "Re-configure" "perl Makefile.PL" # make sure we got everything we need if [[ -n "$(make listdeps)" ]] ; then @@ -193,6 +169,10 @@ if [[ "$POISON_ENV" = "true" ]] && ( perl -MDBD::SQLite\ 1.38 -e1 || perl -MDBI\ exit 1 fi +if [[ "$CLEANTEST" = "true" ]] && perl -MModule::Build::Tiny -e1 &>/dev/null ; then + echo_err "Module::Build::Tiny pulled in during the basic depchain install - this must not happen" + exit 1 +fi # announce what are we running echo_err " diff --git a/maint/travis-ci_scripts/40_script.bash b/maint/travis-ci_scripts/40_script.bash index 2edcafd..ddbaab4 100755 --- a/maint/travis-ci_scripts/40_script.bash +++ b/maint/travis-ci_scripts/40_script.bash @@ -6,7 +6,16 @@ source maint/travis-ci_scripts/common.bash if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then exit 0 ; fi run_harness_tests() { - local -x HARNESS_OPTIONS=c:j$NUMTHREADS + local -x HARNESS_OPTIONS=c:j$VCPU_USE + # if we run under docker (! have_sudo) the logic below won't work + # it seems as if ulimit acts globally, across the entire OS + # and is thus not served properly by a localised `ps xH` + if [[ "$VCPU_USE" == 1 ]] && have_sudo ; then + ulim=$(( ( $(ps xH | wc -l) - 3 ) + 4 )) # (real count excluding header + ps + wc) + space for ( make + tee + harness + ) + echo_err "$(tstamp) Setting process/thread limit to $ulim" + ulimit -u $ulim + sleep 10 # needed to settle things down a bit + fi make test 2> >(tee "$TEST_STDERR_LOG") } @@ -16,7 +25,7 @@ if [[ "$CLEANTEST" = "true" ]] ; then run_or_err "Prepare blib" "make pure_all" run_harness_tests else - PROVECMD="prove -lrswj$NUMTHREADS xt t" + PROVECMD="prove -lrswj$VCPU_USE xt t" # FIXME - temporary, until Package::Stash is fixed if perl -M5.010 -e 1 &>/dev/null ; then @@ -28,7 +37,12 @@ else fi TEST_T1=$SECONDS -if [[ -z "$DBICTRACE" ]] && [[ -z "$POISON_ENV" ]] && [[ -s "$TEST_STDERR_LOG" ]] ; then +if \ + [[ -z "$DBIC_TRACE" ]] \ +&& [[ -z "$DBIC_MULTICREATE_DEBUG" ]] \ +&& [[ -z "$DBICTEST_DEBUG_CONCURRENCY_LOCKS" ]] \ +&& [[ -z "$DBICTEST_VERSION_WARNS_INDISCRIMINATELY" ]] \ +&& [[ -s "$TEST_STDERR_LOG" ]] ; then STDERR_LOG_SIZE=$(wc -l < "$TEST_STDERR_LOG") # prepend STDERR log @@ -49,6 +63,6 @@ echo "${POSTMORTEM:- \o/ No notable smoke run issues \o/ }" echo echo "$(tstamp) Testing took a total of $(( $TEST_T1 - $TEST_T0 ))s" if [[ -n "$INSTALLDEPS_OUT" ]] ; then - echo "$(tstamp) Full dep install log at $(/usr/bin/nopaste -q -s Shadowcat -d DepInstall <<< "$INSTALLDEPS_OUT")" + echo "$(tstamp) Full dep install log at $(/usr/bin/perl /usr/bin/nopaste -q -s Shadowcat -d DepInstall <<< "$INSTALLDEPS_OUT")" fi echo diff --git a/maint/travis-ci_scripts/50_after_failure.bash b/maint/travis-ci_scripts/50_after_failure.bash index f087d6f..ba92421 100755 --- a/maint/travis-ci_scripts/50_after_failure.bash +++ b/maint/travis-ci_scripts/50_after_failure.bash @@ -5,8 +5,7 @@ source maint/travis-ci_scripts/common.bash if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then exit 0 ; fi -echo_err " -$(ci_vm_state_text) - -=== dmesg ringbuffer -$(sudo dmesg)" +if [[ "$(dmesg)" =~ $( echo "\\bOOM\\b" ) ]] ; then + echo_err "=== dmesg ringbuffer" + echo_err "$(dmesg)" +fi diff --git a/maint/travis-ci_scripts/50_after_success.bash b/maint/travis-ci_scripts/50_after_success.bash index 5571e2b..fdb6a64 100755 --- a/maint/travis-ci_scripts/50_after_success.bash +++ b/maint/travis-ci_scripts/50_after_success.bash @@ -3,15 +3,98 @@ # this file is executed in a subshell - set up the common stuff source maint/travis-ci_scripts/common.bash -if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then exit 0 ; fi +if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] || [[ "$TRAVIS_PULL_REQUEST" != "false" ]] ; then exit 0 ; fi + +# this part needs to run in parallel unconditionally +export VCPU_USE="$VCPU_AVAILABLE" +export HARNESS_OPTIONS="j$VCPU_USE" + +if [[ "$DEVREL_DEPS" == "true" ]] && perl -M5.008003 -e1 &>/dev/null ; then + # FIXME - Devel::Cover (brought by Test::Strict, but soon needed anyway) + # does not test cleanly on 5.8.7 - just get it directly + if perl -M5.008007 -e1 &>/dev/null && ! perl -M5.008008 -e1 &>/dev/null; then + parallel_installdeps_notest Devel::Cover + fi + + # FIXME - workaround for YAML/RT#81120 and L::SRH/RT#107681 + # We don't actually need these modules, only there because of SQLT (which will be fixed) + # does not test cleanly on 5.8.7 - just get them directly + if ! perl -M5.008008 -e1 &>/dev/null; then + parallel_installdeps_notest YAML Lexical::SealRequireHints + fi + + parallel_installdeps_notest "Module::Install@1.15" + + # FIXME Change when Moose goes away + installdeps Moose $(perl -Ilib -MDBIx::Class -e 'print join " ", keys %{DBIx::Class::Optional::Dependencies->req_list_for("dist_dir")}') + + run_or_err "Attempt to build a dist" "rm -rf inc/ && perl Makefile.PL --skip-author-deps && make dist" + tarball_assembled=1 + +elif [[ "$CLEANTEST" != "true" ]] ; then + parallel_installdeps_notest "Module::Install@1.15" $(perl -Ilib -MDBIx::Class -e 'print join " ", keys %{DBIx::Class::Optional::Dependencies->req_list_for("dist_dir")}') + + run_or_err "Attempt to build a dist from original checkout" "make dist" + tarball_assembled=1 +fi + +if [[ -n "$tarball_assembled" ]] ; then -if [[ "$CLEANTEST" != "true" ]] ; then - parallel_installdeps_notest $(perl -Ilib -MDBIx::Class -e 'print join " ", keys %{DBIx::Class::Optional::Dependencies->req_list_for("dist_dir")}') - run_or_err "Attempt to build a dist with all prereqs present" "make dist" echo "Contents of the resulting dist tarball:" echo "===========================================" tar -vzxf DBIx-Class-*.tar.gz echo "===========================================" - run_or_err 'Attempt to configure from re-extracted distdir' \ - 'bash -c "cd \$(find DBIx-Class-* -maxdepth 0 -type d | head -n 1) && perl Makefile.PL"' + + # kill as much as possible with fire + purge_sitelib + + + # undo some of the pollution (if any) affecting the plain install deps + # FIXME - this will go away once we move off Moose, and a new SQLT + # with much less recommends ships + export DBICTEST_SQLT_DEPLOY="" + export DBICTEST_VIA_REPLICATED="" + + + # make sure we are retrying with newest CPAN possible + # + # not running tests on CPAN.pm - they are not terribly slow, + # but https://rt.cpan.org/Ticket/Display.html?id=96437 sucks + parallel_installdeps_notest CPAN + run_or_err "Make sure CPAN was upgraded to at least 2.10" "perl -M'CPAN 2.010' -e1" + + run_or_err "Re-Configuring CPAN.pm" "perl -MCPAN -e '\ + CPAN::Config->load; + + # For the time being smoking with this setting is not realistic + # https://rt.cpan.org/Ticket/Display.html?id=103280 + # https://rt.cpan.org/Ticket/Display.html?id=37641 + # https://rt.cpan.org/Ticket/Display.html?id=77708 + # https://rt.cpan.org/Ticket/Display.html?id=87474 + #\$CPAN::Config->{build_requires_install_policy} = q{no}; + + \$CPAN::Config->{recommends_policy} = q{yes}; + CPAN::Config->commit; + '" + + cd "$(find DBIx-Class-* -maxdepth 0 -type d | head -n 1)" + + # only run a full test cycle on devrel_deps, as they are all marked + # as "allow fails" in the travis matrix + if [[ "$DEVREL_DEPS" == "true" ]] ; then + + for e in $( env | grep 'DBICTEST.*DSN' | cut -f 1 -d '=' ) ; do + echo "Unsetting $e" + export $e="" + done + + run_or_err \ + "Attempt to configure/test/build/install dist using latest CPAN@$(perl -MCPAN -e 'print CPAN->VERSION')" \ + "cpan ." + + else + run_or_err \ + "Attempt to configure/build/install dist using latest CPAN@$(perl -MCPAN -e 'print CPAN->VERSION')" \ + "perl -MCPAN -e 'notest( install => q{.} )'" + fi fi diff --git a/maint/travis-ci_scripts/60_after_script.bash b/maint/travis-ci_scripts/60_after_script.bash index fb5b5fb..102291d 100755 --- a/maint/travis-ci_scripts/60_after_script.bash +++ b/maint/travis-ci_scripts/60_after_script.bash @@ -1,7 +1,7 @@ #!/bin/bash # !!! Nothing here will be executed !!! -# The source-line calling this script is commented out in .travis.yml +# The line calling this script is commented out in .travis.yml # this file is executed in a subshell - set up the common stuff source maint/travis-ci_scripts/common.bash diff --git a/maint/travis-ci_scripts/common.bash b/maint/travis-ci_scripts/common.bash index 5af1818..de89f94 100755 --- a/maint/travis-ci_scripts/common.bash +++ b/maint/travis-ci_scripts/common.bash @@ -26,7 +26,7 @@ $(perl -0777 -p -e 's/.+\n\n(?!\z)//s' < /proc/cpuinfo) $(free -m -t) = Diskinfo -$(sudo df -h) +$(df -h) $(mount | grep '^/') @@ -37,13 +37,13 @@ $(uname -a) $(ip addr) = Network Sockets Status -$(sudo netstat -an46p | grep -Pv '\s(CLOSING|(FIN|TIME|CLOSE)_WAIT.?|LAST_ACK)\s') +$( (sudo netstat -an46p || netstat -an46p) | grep -Pv '\s(CLOSING|(FIN|TIME|CLOSE)_WAIT.?|LAST_ACK)\s') = Processlist -$(sudo ps fuxa) +$(ps fuxa) = Environment -$(env | grep -P 'TEST|HARNESS|MAKE|TRAVIS|PERL|DBIC' | LC_ALL=C sort | cat -v) +$(env | grep -P 'TEST|HARNESS|MAKE|TRAVIS|PERL|DBIC|PATH|SHELL' | LC_ALL=C sort | cat -v) = Perl in use $(perl -V) @@ -75,6 +75,10 @@ run_or_err() { echo_err "$LASTCMD" echo_err "STDOUT+STDERR:" echo_err "$LASTOUT" + if [[ "$(dmesg)" =~ $( echo "\\bOOM\\b" ) ]] ; then + echo_err "=== dmesg ringbuffer" + echo_err "$(dmesg)" + fi fi return $LASTEXIT @@ -155,12 +159,14 @@ parallel_installdeps_notest() { run_or_err "Installing (without testing) $(echo $MODLIST)" \ "echo \\ \"$MODLIST\" \\ - | xargs -d '\\n' -n 1 -P $NUMTHREADS bash -c \\ + | xargs -d '\\n' -n 1 -P $VCPU_USE bash -c \\ 'OUT=\$(maint/getstatus $TIMEOUT_CMD cpanm --notest \"\$@\" 2>&1 ) || (LASTEXIT=\$?; echo \"\$OUT\"; exit \$LASTEXIT)' \\ 'giant space monkey penises' " } +export -f parallel_installdeps_notest run_or_err echo_err tstamp + installdeps() { if [[ -z "$@" ]] ; then return; fi @@ -168,10 +174,10 @@ installdeps() { local -x HARNESS_OPTIONS - HARNESS_OPTIONS="j$NUMTHREADS" + HARNESS_OPTIONS="j$VCPU_USE" if ! run_or_err "Attempting install of $# modules under parallel ($HARNESS_OPTIONS) testing ($MODLIST)" "_dep_inst_with_test $MODLIST" quiet_fail ; then - local errlog="failed after ${DELTA_TIME}s Exit:$LASTEXIT Log:$(/usr/bin/nopaste -q -s Shadowcat -d "Parallel testfail" <<< "$LASTOUT")" + local errlog="failed after ${DELTA_TIME}s Exit:$LASTEXIT Log:$(/usr/bin/perl /usr/bin/nopaste -q -s Shadowcat -d "Parallel testfail" <<< "$LASTOUT")" echo "$errlog" POSTMORTEM="$POSTMORTEM$( @@ -190,24 +196,60 @@ _dep_inst_with_test() { if [[ "$DEVREL_DEPS" == "true" ]] ; then # --dev is already part of CPANM_OPT LASTCMD="$TIMEOUT_CMD cpanm $@" - $LASTCMD 2>&1 + $LASTCMD 2>&1 || return 1 else LASTCMD="$TIMEOUT_CMD cpan $@" - $LASTCMD 2>&1 + $LASTCMD 2>&1 || return 1 # older perls do not have a CPAN which can exit with error on failed install for m in "$@"; do if ! perl -e ' +$ARGV[0] =~ s/-TRIAL\.//; + my $mod = ( - $ARGV[0] =~ m{ \/ .*? ([^\/]+) $ }x + # abuse backtrack + $ARGV[0] =~ m{ / .*? ( [^/]+ ) $ }x ? do { my @p = split (/\-/, $1); pop @p; join "::", @p } : $ARGV[0] ); -$mod = q{List::Util} if $mod eq q{Scalar::List::Utils}; +# map some install-names to a module/version combo +# serves both as a grandfathered title-less tarball, and +# as a minimum version check for upgraded core modules +my $eval_map = { + + # this is temporary, will need something more robust down the road + # (perhaps by then Module::CoreList will be dep-free) + "Module::Build" => { ver => "0.4214" }, + "podlators" => { mod => "Pod::Man", ver => "2.17" }, + + "File::Spec" => { ver => "3.47" }, + "Cwd" => { ver => "3.47" }, + + "List::Util" => { ver => "1.42" }, + "Scalar::Util" => { ver => "1.42" }, + "Scalar::List::Utils" => { mod => "List::Util", ver => "1.42" }, +}; + +my $m = $eval_map->{$mod}{mod} || $mod; + +eval( + "require $m" + + . + + ($eval_map->{$mod}{ver} + ? "; $m->VERSION(\$eval_map->{\$mod}{ver}) " + : "" + ) -eval qq{require($mod)} or ( print $@ and exit 1) + . + + "; 1" +) + or +( print $@ and exit 1) ' "$m" 2> /dev/null ; then echo -e "$m installation seems to have failed" @@ -217,6 +259,77 @@ eval qq{require($mod)} or ( print $@ and exit 1) fi } +# Idea stolen from +# https://github.com/kentfredric/Dist-Zilla-Plugin-Prereqs-MatchInstalled-All/blob/master/maint-travis-ci/sterilize_env.pl +# Only works on 5.12+ (where sitelib was finally properly fixed) +purge_sitelib() { + echo_err "$(tstamp) Sterilizing the Perl installation (cleaning up sitelib)" + + if perl -M5.012 -e1 &>/dev/null ; then + + perl -M5.012 -MConfig -MFile::Find -e ' + my $sitedirs = { + map { $Config{$_} => 1 } + grep { $_ =~ /site(lib|arch)exp$/ } + keys %Config + }; + find({ bydepth => 1, no_chdir => 1, follow_fast => 1, wanted => sub { + ! $sitedirs->{$_} and ( -d _ ? rmdir : unlink ) + } }, keys %$sitedirs ) + ' + else + + cl_fn="/tmp/${TRAVIS_BUILD_ID}_Module_CoreList.pm"; + + [[ -s "$cl_fn" ]] || run_or_err \ + "Downloading latest Module::CoreList" \ + "curl -s --compress -o '$cl_fn' https://api.metacpan.org/source/Module::CoreList" + + perl -0777 -Ilib -MDBIx::Class::Optional::Dependencies -e ' + + # this is horrible, but really all we want is "has this ever been used" + # so a grep without a load is quite legit (and horrible) + my $mcl_source = <>; + + my %reqs_for_group = %{DBIx::Class::Optional::Dependencies->req_group_list}; + + my @all_possible_never_been_core_modpaths = map + { (my $mp = $_ . ".pm" ) =~ s|::|/|g; $mp } + grep + { $mcl_source !~ / ^ \s+ \x27 $_ \x27 \s* \=\> /mx } + ( + qw( + Module::Build::Tiny + ), + (map + { keys %{$reqs_for_group{$_}} } + grep + { !/^rdbms_|^dist_/ } + keys %reqs_for_group + ), + ) + ; + + # now that we have the list we can go ahead and destroy every single one + # of these modules without being concerned about breaking the base ability + # to install things + for my $mp ( sort { lc($a) cmp lc($b) } @all_possible_never_been_core_modpaths ) { + for my $incdir (@INC) { + -e "$incdir/$mp" + and + unlink "$incdir/$mp" + and + print "Nuking $incdir/$mp\n" + } + } + ' "$cl_fn" + + fi +} + + CPAN_is_sane() { perl -MCPAN\ 1.94_56 -e 1 &>/dev/null ; } CPAN_supports_BUILDPL() { perl -MCPAN\ 1.9205 -e1 &>/dev/null; } + +have_sudo() { sudo /bin/true &>/dev/null ; } diff --git a/maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf b/maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf index 770effa..e7ac7a6 100644 --- a/maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf +++ b/maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf @@ -1,23 +1,47 @@ [mysqld] +# the DBIC test suite does hold up to 3 concurrent connections +# (t/94versioning.t), otherwise this could conceivably go lower (~50MB vsize +# per connection) +max_connections = 3 + thread_cache_size = 0 +thread_stack = 128K # mysql >= 5.5.16 #thread_pool_size = 1 -bulk_insert_buffer_size = 0 +net_buffer_length = 4K read_buffer_size = 32K join_buffer_size = 128K sort_buffer_size = 128K -table_definition_cache = 400 +bulk_insert_buffer_size = 0 +table_definition_cache = 256 performance_schema = 0 query_cache_type = 0 query_cache_size = 0 +query_cache_limit = 16K -innodb_use_sys_malloc = 1 -innodb_buffer_pool_size = 1M - +myisam_sort_buffer_size = 16K +tmp_table_size = 1M key_buffer_size = 64K -myisam_sort_buffer_size = 128K + +innodb_data_file_path = ibdata1:10M:autoextend +innodb_autoextend_increment = 1 +innodb_buffer_pool_size = 512K + +innodb_stats_on_metadata = 0 +innodb_file_per_table = 0 + +innodb_log_file_size = 1M +innodb_log_buffer_size = 512K +innodb_buffer_pool_size = 512K + +innodb_use_sys_malloc = 0 +innodb_additional_mem_pool_size = 256K +innodb_flush_method = O_DIRECT + +innodb_read_io_threads = 1 +innodb_write_io_threads = 1 diff --git a/t/52leaks.t b/t/52leaks.t index b3b9b9c..bce0878 100644 --- a/t/52leaks.t +++ b/t/52leaks.t @@ -520,12 +520,6 @@ assert_empty_weakregistry ($weak_registry); # this is ugly and dirty but we do not yet have a Test::Embedded or # similar -# set up -I -require Config; -$ENV{PERL5LIB} = join ($Config::Config{path_sep}, @INC); -($ENV{PATH}) = $ENV{PATH} =~ /(.+)/; - - my $persistence_tests; SKIP: { skip 'Test already in a persistent loop', 1 @@ -556,7 +550,17 @@ SKIP: { @{$persistence_tests->{PPerl}{cmd}}[ 1 .. $#{$persistence_tests->{PPerl}{cmd}} ], ]; - require IPC::Open2; + # set up -I + require Config; + $ENV{PERL5LIB} = join ($Config::Config{path_sep}, @INC); + + # adjust PATH for -T + if (length $ENV{PATH}) { + ( $ENV{PATH} ) = join ( $Config::Config{path_sep}, + map { length($_) ? File::Spec->rel2abs($_) : () } + split /\Q$Config::Config{path_sep}/, $ENV{PATH} + ) =~ /\A(.+)\z/; + } for my $type (keys %$persistence_tests) { SKIP: { unless (eval "require $type") { @@ -578,6 +582,8 @@ SKIP: { if system(@cmd); } + require IPC::Open2; + for (1,2,3) { note ("Starting run in persistent env ($type pass $_)"); IPC::Open2::open2(my $out, undef, @cmd); diff --git a/t/lib/DBICTest.pm b/t/lib/DBICTest.pm index 47b62a9..35a9e0c 100644 --- a/t/lib/DBICTest.pm +++ b/t/lib/DBICTest.pm @@ -13,6 +13,7 @@ use Path::Class::File (); use File::Spec; use Fcntl qw/:DEFAULT :flock/; use Config; +use Scope::Guard (); =head1 NAME @@ -386,8 +387,11 @@ sub deploy_schema { my $schema = shift; my $args = shift || {}; - local $schema->storage->{debug} - if ($ENV{TRAVIS}||'') eq 'true'; + my $guard; + if ( ($ENV{TRAVIS}||'') eq 'true' and my $old_dbg = $schema->storage->debug ) { + $guard = Scope::Guard->new(sub { $schema->storage->debug($old_dbg) }); + $schema->storage->debug(0); + } if ($ENV{"DBICTEST_SQLT_DEPLOY"}) { $schema->deploy($args); @@ -417,8 +421,11 @@ sub populate_schema { my $self = shift; my $schema = shift; - local $schema->storage->{debug} - if ($ENV{TRAVIS}||'') eq 'true'; + my $guard; + if ( ($ENV{TRAVIS}||'') eq 'true' and my $old_dbg = $schema->storage->debug ) { + $guard = Scope::Guard->new(sub { $schema->storage->debug($old_dbg) }); + $schema->storage->debug(0); + } $schema->populate('Genre', [ [qw/genreid name/], diff --git a/t/lib/DBICTest/RunMode.pm b/t/lib/DBICTest/RunMode.pm index 1e31436..dc4d17f 100644 --- a/t/lib/DBICTest/RunMode.pm +++ b/t/lib/DBICTest/RunMode.pm @@ -216,6 +216,14 @@ sub is_smoker { ; } +sub is_ci { + return ( + ($ENV{TRAVIS}||'') eq 'true' + and + ($ENV{TRAVIS_REPO_SLUG}||'') =~ m|\w+/dbix-class$| + ) +} + sub is_plain { return (! __PACKAGE__->is_smoker && ! __PACKAGE__->is_author && ! $ENV{RELEASE_TESTING} ) } diff --git a/t/storage/quote_names.t b/t/storage/quote_names.t index bf46122..e667ad3 100644 --- a/t/storage/quote_names.t +++ b/t/storage/quote_names.t @@ -89,7 +89,9 @@ my %dbs = ( ); # lie that we already locked stuff - the tests below do not touch anything -$ENV{DBICTEST_LOCK_HOLDER} = -1; +# unless we are under travis, where the OOM killers reign and things are rough +$ENV{DBICTEST_LOCK_HOLDER} = -1 + unless DBICTest::RunMode->is_ci; # Make sure oracle is tried last - some clients (e.g. 10.2) have symbol # clashes with libssl, and will segfault everything coming after them diff --git a/xt/authors.t b/xt/authors.t index e5bc2cd..8ee1bf3 100644 --- a/xt/authors.t +++ b/xt/authors.t @@ -58,16 +58,29 @@ if (length $ENV{PATH}) { ) =~ /\A(.+)\z/; } -# this may fail - not every system has git -if (my @git_authors = map - { my ($gitname) = m/^ \s* \d+ \s* (.+?) \s* $/mx; utf8::decode($gitname); $gitname } - qx( git shortlog -e -s ) +# no git-check when smoking a PR +if ( + ( + ! $ENV{TRAVIS_PULL_REQUEST} + or + $ENV{TRAVIS_PULL_REQUEST} eq "false" + ) + and + -d '.git' ) { - for (@git_authors) { + + binmode (Test::More->builder->$_, ':utf8') for qw/output failure_output todo_output/; + + # this may fail - not every system has git + for ( + map + { my ($gitname) = m/^ \s* \d+ \s* (.+?) \s* $/mx; utf8::decode($gitname); $gitname } + qx( git shortlog -e -s ) + ) { my ($eml) = $_ =~ $email_re; - fail "Commit author '$_' (from git) not reflected in AUTHORS, perhaps a missing .mailmap entry?" - unless $known_authors{$eml}; + ok $known_authors{$eml}, + "Commit author '$_' (from .mailmap-aware `git shortlog -e -s`) reflected in ./AUTHORS"; } }