From: Dagfinn Ilmari Mannsåker <ilmari@ilmari.org>
Date: Tue, 20 Jan 2015 13:46:48 +0000 (+0000)
Subject: Nick Travis script improvements from DBIC
X-Git-Tag: 0.07043~20
X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=a61771df2487b674cc036f4647ebe532e6f1f471;p=dbsrgits%2FDBIx-Class-Schema-Loader.git

Nick Travis script improvements from DBIC
---

diff --git a/.travis.yml b/.travis.yml
index c83f52a..52f1eaa 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -235,42 +235,57 @@ matrix:
 # console output
 
 before_install:
+  # common functions for all run phases below
+  #
+  # this is an exporter - sourcing it is crucial
+  # among other things it also sets -e
+  #
+  - source maint/travis-ci_scripts/common.bash
+
   # Sets global envvars, downloads/configures debs based on CLEANTEST
   # Sets extra DBICTEST_* envvars
   #
+  # this is an exporter - sourcing it is crucial
+  #
   - source maint/travis-ci_scripts/10_before_install.bash
 
 install:
   # Build and switch to a custom perl if requested
   # Configure the perl env, preinstall some generic toolchain parts
   #
+  # this is an exporter - sourcing it is crucial
+  #
   - source maint/travis-ci_scripts/20_install.bash
 
+###
+### From this point on -e is *unset*, rely on travis' error handling
+###
+  - set +e
+
 before_script:
   # Preinstall/install deps based on envvars/CLEANTEST
   #
-  - source maint/travis-ci_scripts/30_before_script.bash
+  # need to invoke the after_failure script manually
+  # because 'after_failure' runs only after 'script' fails
+  #
+  - maint/getstatus maint/travis-ci_scripts/30_before_script.bash || ( maint/travis-ci_scripts/50_after_failure.bash && /bin/false )
 
 script:
   # Run actual tests
   #
-  - source maint/travis-ci_scripts/40_script.bash
+  - maint/getstatus maint/travis-ci_scripts/40_script.bash
 
 after_success:
-  # Check if we can assemble a dist properly if not in CLEANTEST
-  # skipped for now
-  # - source maint/travis-ci_scripts/50_after_success.bash
+  # No tasks yet
+  #
+  # - maint/getstatus maint/travis-ci_scripts/50_after_success.bash
 
 after_failure:
-  # No tasks yet
+  # Final sysinfo printout on fail
   #
-  #- source maint/travis-ci_scripts/50_after_failure.bash
+  - maint/getstatus maint/travis-ci_scripts/50_after_failure.bash
 
 after_script:
   # No tasks yet
   #
-  #- source maint/travis-ci_scripts/60_after_script.bash
-
-  # if we do not unset this before we terminate the travis teardown will
-  # mark the entire job as failed
-  - set +e
+  #- maint/getstatus maint/travis-ci_scripts/60_after_script.bash
diff --git a/maint/getstatus b/maint/getstatus
new file mode 100755
index 0000000..f49c410
--- /dev/null
+++ b/maint/getstatus
@@ -0,0 +1,52 @@
+#!/usr/bin/perl
+
+use warnings;
+use strict;
+
+use Config;
+use Term::ANSIColor ':constants';
+my $CRST = RESET;
+my $CCODE = BOLD;
+my $CSTAT = BOLD . GREEN;
+my $CCORE = BOLD . CYAN;
+my $CSIG = CYAN;
+
+if (@ARGV) {
+  my $code = system (@ARGV);
+
+  if ($code < 0) {
+    exit 127;
+  }
+  elsif ($code > 0) {
+
+    my $status = $code >> 8;
+    my $signum = $code & 127;
+    my $core = $code & 128;
+
+    my %sig_idx;
+    @sig_idx{split /\s+/, $Config{sig_num}} = split /\s/, $Config{sig_name};
+
+    printf STDERR (
+<<EOF
+
+Results of execution: `%s`
+----------------------
+System exit code:$CCODE %d $CRST$CSIG %s $CRST
+ ($CSTAT%08b$CRST$CCORE%b$CRST$CSIG%07b$CRST)
+
+Status: %3s ($CSTAT%08b$CRST)
+Signal: %3s ($CSIG%08b$CRST)
+Core:   %3s
+----------------------
+EOF
+    , (join ' ', @ARGV),
+      $code, ($signum ? "(SIG-$sig_idx{$signum})" : ''),
+      $status, $core, $signum,
+      ($status) x 2,
+      ($signum) x 2,
+      ($core ? 'Yes': 'No')
+    );
+
+    exit ($status);
+  }
+}
diff --git a/maint/travis-ci_scripts/10_before_install.bash b/maint/travis-ci_scripts/10_before_install.bash
index 7d8b78b..c5b6fee 100755
--- a/maint/travis-ci_scripts/10_before_install.bash
+++ b/maint/travis-ci_scripts/10_before_install.bash
@@ -1,6 +1,32 @@
 #!/bin/bash
 
-source maint/travis-ci_scripts/common.bash
+# Stop pre-started RDBMS and sync for some settle time
+run_or_err "Stopping MySQL"       "sudo /etc/init.d/mysql stop"
+run_or_err "Stopping PostgreSQL"  "sudo /etc/init.d/postgresql stop"
+/bin/sync
+
+# Sanity check VM before continuing
+echo "
+=============================================================================
+
+= Startup Meminfo
+$(free -m -t)
+
+============================================================================="
+
+CI_VM_MIN_FREE_MB=2000
+if [[ "$(free -m | grep 'buffers/cache:' | perl -p -e '$_ = (split /\s+/, $_)[3]')" -lt "$CI_VM_MIN_FREE_MB" ]]; then
+  SHORT_CIRCUIT_SMOKE=1
+  echo_err "
+=============================================================================
+
+CI virtual machine stuck in a state with a lot of memory locked for no reason.
+Under Travis this state usually results in a failed build.
+Short-circuiting buildjob to avoid false negatives, please restart it manually.
+
+============================================================================="
+fi
+
 if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi
 
 # Different boxes we run on may have different amount of hw threads
@@ -11,21 +37,24 @@ if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi
 # The oneliner is a tad convoluted - basicaly what we do is
 # slurp the entire file and get the index off the last
 # `processor    : XX` line
-export NUMTHREADS="$(( $(perl -0777 -n -e 'print (/ (?: .+ ^ processor \s+ : \s+ (\d+) ) (?! ^ processor ) /smx)' < /proc/cpuinfo) + 1 ))"
+#
+# We also divide the result by a factor, otherwise the travis VM gets
+# overloaded (the amount of available swap is just TOOOO damn small)
+export NUMTHREADS="$(( ( $(perl -0777 -n -e 'print (/ (?: .+ ^ processor \s+ : \s+ (\d+) ) (?! ^ processor ) /smx)' < /proc/cpuinfo) + 1 ) / 3 ))"
 
 export CACHE_DIR="/tmp/poormanscache"
 
 export SCHEMA_LOADER_TESTS_BACKCOMPAT=1
 
-# install some common tools from APT, more below unless CLEANTEST
-apt_install libapp-nopaste-perl tree apt-transport-https
+# these will be installed no matter what, also some extras unless CLEANTEST
+common_packages="libapp-nopaste-perl tree"
 
-# FIXME - the debian package is oddly broken - uses a bin/env based shebang
-# so nothing works under a brew. Fix here until #debian-perl patches it up
-sudo /usr/bin/perl -p -i -e 's|#!/usr/bin/env perl|#!/usr/bin/perl|' $(which nopaste)
+if [[ "$CLEANTEST" = "true" ]]; then
+
+  apt_install $common_packages
+
+else
 
-if [[ "$CLEANTEST" != "true" ]]; then
-### apt-get invocation - faster to grab everything at once
   #
   # FIXME these debconf lines should automate the firebird config but do not :(((
   sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/enabled\tboolean\ttrue" | debconf-set-selections'
@@ -39,18 +68,21 @@ if [[ "$CLEANTEST" != "true" ]]; then
 
   run_or_err "Priming up the APT cache with $(echo $(ls -d $CACHE_DIR/apt_cache/*.deb))" "sudo cp $CACHE_DIR/apt_cache/*.deb /var/cache/apt/archives"
 
-  apt_install memcached firebird2.5-super firebird2.5-dev unixodbc-dev expect oracle-xe
+  apt_install $common_packages firebird2.5-super firebird2.5-dev unixodbc-dev expect oracle-xe
 
 ### config memcached
   run_or_err "Starting memcached" "sudo /etc/init.d/memcached start"
   export DBICTEST_MEMCACHED=127.0.0.1:11211
 
 ### config mysql
+  run_or_err "Installing minimizing MySQL config" "sudo cp maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf /etc/mysql/conf.d/ && sudo chmod 644 /etc/mysql/conf.d/*.cnf"
+  run_or_err "Starting MySQL" "sudo /etc/init.d/mysql start"
   run_or_err "Creating MySQL TestDB" "mysql -e 'create database dbic_test;'"
   export DBICTEST_MYSQL_DSN='dbi:mysql:database=dbic_test;host=127.0.0.1'
   export DBICTEST_MYSQL_USER=root
 
 ### config pg
+  run_or_err "Starting PostgreSQL" "sudo /etc/init.d/postgresql start"
   run_or_err "Creating PostgreSQL TestDB" "psql -c 'create database dbic_test;' -U postgres"
   export DBICTEST_PG_DSN='dbi:Pg:database=dbic_test;host=127.0.0.1'
   export DBICTEST_PG_USER=postgres
@@ -188,3 +220,10 @@ FileUsage       = 1
 
   export ORACLE_HOME="$CACHE_DIR/ora_instaclient/x86-64/oracle_instaclient_10.2.0.5.0"
 fi
+
+# The debian package is oddly broken - uses a /bin/env based shebang
+# so nothing works under a brew, fixed in libapp-nopaste-perl 0.92-3
+# http://changelogs.ubuntu.com/changelogs/pool/universe/liba/libapp-nopaste-perl/libapp-nopaste-perl_0.96-1/changelog
+#
+# Since the vm runs an old version of umbongo fix things ourselves
+sudo /usr/bin/perl -p -i -e 's|#!/usr/bin/env perl|#!/usr/bin/perl|' $(which nopaste)
diff --git a/maint/travis-ci_scripts/20_install.bash b/maint/travis-ci_scripts/20_install.bash
index 34ebc14..1576028 100755
--- a/maint/travis-ci_scripts/20_install.bash
+++ b/maint/travis-ci_scripts/20_install.bash
@@ -1,16 +1,12 @@
 #!/bin/bash
 
-source maint/travis-ci_scripts/common.bash
 if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi
 
-CPAN_MIRROR=$(echo "$PERL_CPANM_OPT" | grep -oP -- '--mirror\s+\S+' | head -n 1 | cut -d ' ' -f 2)
-if ! [[ "$CPAN_MIRROR" =~ "http://" ]] ; then
-  echo_err "Unable to extract primary cpan mirror from PERL_CPANM_OPT - something is wrong"
-  echo_err "PERL_CPANM_OPT: $PERL_CPANM_OPT"
-  CPAN_MIRROR="https://cpan.metacpan.org/"
-  PERL_CPANM_OPT="$PERL_CPANM_OPT --mirror $CPAN_MIRROR"
-  echo_err "Using $CPAN_MIRROR for the time being"
-fi
+# we need a mirror that both has the standard index and a backpan version rolled
+# into one, due to MDV testing
+CPAN_MIRROR="http://cpan.metacpan.org/"
+
+PERL_CPANM_OPT="$PERL_CPANM_OPT --mirror $CPAN_MIRROR"
 
 export PERL_MM_USE_DEFAULT=1 PERL_MM_NONINTERACTIVE=1 PERL_AUTOINSTALL_PREFER_CPAN=1 PERLBREW_CPAN_MIRROR="$CPAN_MIRROR" HARNESS_TIMER=1 MAKEFLAGS="-j$NUMTHREADS"
 
diff --git a/maint/travis-ci_scripts/30_before_script.bash b/maint/travis-ci_scripts/30_before_script.bash
index 2029e90..99cebe6 100755
--- a/maint/travis-ci_scripts/30_before_script.bash
+++ b/maint/travis-ci_scripts/30_before_script.bash
@@ -1,6 +1,8 @@
 #!/bin/bash
 
+# this file is executed in a subshell - set up the common stuff
 source maint/travis-ci_scripts/common.bash
+
 if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi
 
 # poison the environment
@@ -256,24 +258,4 @@ echo_err "
 ===================== DEPENDENCY CONFIGURATION COMPLETE =====================
 $(tstamp) Configuration phase seems to have taken $(date -ud "@$SECONDS" '+%H:%M:%S') (@$SECONDS)
 
-= CPUinfo
-$(perl -0777 -p -e 's/.+\n\n(?!\z)//s' < /proc/cpuinfo)
-
-= Meminfo
-$(free -m -t)
-
-= Kernel info
-$(uname -a)
-
-= Network Configuration
-$(ip addr)
-
-= Network Sockets Status
-$(sudo netstat -an46p | grep -Pv '\s(CLOSING|(FIN|TIME|CLOSE)_WAIT.?|LAST_ACK)\s')
-
-= Environment
-$(env | grep -P 'TEST|HARNESS|MAKE|TRAVIS|PERL|DBIC' | LC_ALL=C sort | cat -v)
-
-= Perl in use
-$(perl -V)
-============================================================================="
+$(ci_vm_state_text)"
diff --git a/maint/travis-ci_scripts/40_script.bash b/maint/travis-ci_scripts/40_script.bash
index 602f2df..f7488d5 100755
--- a/maint/travis-ci_scripts/40_script.bash
+++ b/maint/travis-ci_scripts/40_script.bash
@@ -1,6 +1,8 @@
 #!/bin/bash
 
+# this file is executed in a subshell - set up the common stuff
 source maint/travis-ci_scripts/common.bash
+
 if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi
 
 run_harness_tests() {
diff --git a/maint/travis-ci_scripts/50_after_failure.bash b/maint/travis-ci_scripts/50_after_failure.bash
index 4935763..f087d6f 100755
--- a/maint/travis-ci_scripts/50_after_failure.bash
+++ b/maint/travis-ci_scripts/50_after_failure.bash
@@ -1,11 +1,12 @@
 #!/bin/bash
 
-# !!! Nothing here will be executed !!!
-# The source-line calling this script is commented out in .travis.yml
-
+# this file is executed in a subshell - set up the common stuff
 source maint/travis-ci_scripts/common.bash
-if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi
 
-echo_err "Nothing to do"
+if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then exit 0 ; fi
+
+echo_err "
+$(ci_vm_state_text)
 
-return 0
+=== dmesg ringbuffer
+$(sudo dmesg)"
diff --git a/maint/travis-ci_scripts/50_after_success.bash b/maint/travis-ci_scripts/50_after_success.bash
index c8d2bac..3c0e4bd 100755
--- a/maint/travis-ci_scripts/50_after_success.bash
+++ b/maint/travis-ci_scripts/50_after_success.bash
@@ -1,15 +1,13 @@
 #!/bin/bash
 
+# !!! Nothing here will be executed !!!
+# The line calling this script is commented out in .travis.yml
+
+# this file is executed in a subshell - set up the common stuff
 source maint/travis-ci_scripts/common.bash
+
 if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi
 
-if [[ "$CLEANTEST" != "true" ]] ; then
-  parallel_installdeps_notest $(perl -Ilib -MDBIx::Class -e 'print join " ", keys %{DBIx::Class::Optional::Dependencies->req_list_for("dist_dir")}')
-  run_or_err "Attempt to build a dist with all prereqs present" "make dist"
-  echo "Contents of the resulting dist tarball:"
-  echo "==========================================="
-  tar -vzxf DBIx-Class-*.tar.gz
-  echo "==========================================="
-  run_or_err 'Attempt to configure from re-extracted distdir' \
-    'bash -c "cd \$(find DBIx-Class-* -maxdepth 0 -type d | head -n 1) && perl Makefile.PL"'
-fi
+echo_err "Nothing to do"
+
+return 0
diff --git a/maint/travis-ci_scripts/common.bash b/maint/travis-ci_scripts/common.bash
index da6ce33..2588621 100755
--- a/maint/travis-ci_scripts/common.bash
+++ b/maint/travis-ci_scripts/common.bash
@@ -14,6 +14,41 @@ fi
 
 tstamp() { echo -n "[$(date '+%H:%M:%S')]" ; }
 
+ci_vm_state_text() {
+  echo "
+========================== CI System information ============================
+
+= CPUinfo
+$(perl -0777 -p -e 's/.+\n\n(?!\z)//s' < /proc/cpuinfo)
+
+= Meminfo
+$(free -m -t)
+
+= Diskinfo
+$(sudo df -h)
+
+$(mount | grep '^/')
+
+= Kernel info
+$(uname -a)
+
+= Network Configuration
+$(ip addr)
+
+= Network Sockets Status
+$(sudo netstat -an46p | grep -Pv '\s(CLOSING|(FIN|TIME|CLOSE)_WAIT.?|LAST_ACK)\s')
+
+= Processlist
+$(sudo ps fuxa)
+
+= Environment
+$(env | grep -P 'TEST|HARNESS|MAKE|TRAVIS|PERL|DBIC' | LC_ALL=C sort | cat -v)
+
+= Perl in use
+$(perl -V)
+============================================================================="
+}
+
 run_or_err() {
   echo_err -n "$(tstamp) $1 ... "
 
@@ -51,9 +86,6 @@ apt_install() {
   # flatten
   pkgs="$@"
 
-  # Need to do this at every step, the sources list may very well have changed
-  run_or_err "Updating APT available package list" "sudo apt-get update"
-
   run_or_err "Installing Debian APT packages: $pkgs" "sudo apt-get install --allow-unauthenticated  --no-install-recommends -y $pkgs"
 }
 
diff --git a/maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf b/maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf
new file mode 100644
index 0000000..770effa
--- /dev/null
+++ b/maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf
@@ -0,0 +1,23 @@
+[mysqld]
+
+thread_cache_size = 0
+
+# mysql >= 5.5.16
+#thread_pool_size = 1
+
+bulk_insert_buffer_size = 0
+read_buffer_size = 32K
+join_buffer_size = 128K
+sort_buffer_size = 128K
+table_definition_cache = 400
+
+performance_schema = 0
+
+query_cache_type = 0
+query_cache_size = 0
+
+innodb_use_sys_malloc = 1
+innodb_buffer_pool_size = 1M
+
+key_buffer_size = 64K
+myisam_sort_buffer_size = 128K