From: Peter Rabbitson <ribasushi@cpan.org>
Date: Thu, 29 Nov 2012 13:57:30 +0000 (+0100)
Subject: More extensive travis testing plan
X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=d70070c9e7e2a2ec3306f6b95f770c0faff61cbb;p=dbsrgits%2FDBIx-Class-Historic.git

More extensive travis testing plan
---

diff --git a/.travis.yml b/.travis.yml
index 199b5a5..3abc1d7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,60 +1,109 @@
+# Some overall notes on how this works
+#
+# * We smoke using the system provided latest, and custom built "oddball perls"
+# The reason for not having a blanket matrix is to conserve travis resources
+# as a full DBIC depchain isn't cheap
+#
+# * Minimum perl officially supported by DBIC is 5.8.3. This *includes* the
+# basic depchain. On failure either attempt to fix it or bring it to the
+# attention of ribasushi. *DO NOT* disable 5.8 testing - it is here for a
+# reason
+#
+# * The matrix is built from two main modes - CLEANTEST = [true|false]. In
+# the first case we test with minimal deps available, and skip everything
+# listed in DBIC::OptDesps. Without CLEANTEST we bring the armada of RDBMS
+# and install the maximum possible set of deps and configs
+#
+# * All builds/tests run under NUMTHREADS number of threads.
+#
+# * The way .travis.yml is fed to the command controller is idiotic - it
+# makes using multiline `bash -c` statements impossible. Therefore to
+# aid readability (our travis logic is rather complex), the bulk of
+# functionality is moved to a script. More about the problem (and the
+# WONTFIX "explanation") here: https://github.com/travis-ci/travis-ci/issues/497
+#
+
+# smoke only specific branches to a) not overload the queue and b) not
+# overspam the notification channels
+branches:
+  only:
+    - master
+    - /^topic\//
+    - /^smoke\//
+
+notifications:
+  irc:
+    channels:
+      - "irc.perl.org#dbic-smoke"
+    template:
+      - "%{branch}#%{build_number} by %{author}: %{message} (%{build_url})"
+    on_success: change
+    on_failure: always
+    use_notice: true
+
+  email:
+    - ribasushi@cpan.org
+    # Temporary - if it proves to be too noisy, we'll shut it off
+    - dbix-class-devel@lists.scsys.co.uk
+
 language: perl
+
 perl:
-   - "5.16"
-   - "5.14"
-   - "5.12"
-   - "5.10"
+  - "5.16"
 
-# Deal with all of the dependancies, quickly and quietly
-install:
-   - export RELEASE_TESTING=1 AUTOMATED_TESTING=1 AUTHOR_TESTING=1 HARNESS_OPTIONS=j10:c HARNESS_TIMER=1
-   - perl Makefile.PL --alldeps
-
-   # Install modules that typically appear in more than one dep
-   - echo -e 'DBI\nMoo\nDateTime::Format::Strptime' | xargs -n 1 -P 10 cpanm --quiet --notest
-
-   # Superfast parallelized dependancy installs
-   - make listalldeps | xargs -n 6 -P 10 cpanm --quiet --notest
-
-# DBICTEST extra database tests
-before_script:
-   # Install required DBDs
-   - echo -e 'DBD::Pg\nDBD::mysql\nSys::SigAction' | xargs -n 1 -P 10 cpanm --quiet --notest
-
-   # Set up DBICTEST vars
-   - export DBICTEST_PG_DSN='dbi:Pg:database=dbic_test;host=127.0.0.1'    DBICTEST_PG_USER=postgres DBICTEST_PG_PASS=''
-   - export DBICTEST_MYSQL_DSN='dbi:mysql:database=dbic_test;host=127.0.0.1' DBICTEST_MYSQL_USER=root  DBICTEST_MYSQL_PASS=''
-   ### Disabling for now until we figure out the weird Firebird test failure, and why DBD::Firebird doesn't install ###
-   #- export DBICTEST_FIREBIRD_DSN='dbi:Firebird:db=/var/lib/firebird/2.5/data/employee.fdb' DBICTEST_FIREBIRD_USER=SYSDBA DBICTEST_FIREBIRD_PASS=masterkey
-
-   # Set up databases for already-installed platforms
-   - mysql -e 'create database dbic_test;'
-   - psql -c 'create database dbic_test;' -U postgres
-
-   # Database install for Firebird
-   #- sudo apt-get -y install firebird2.5-super firebird2.5-examples
-   #- sudo perl -pi -e 's/\=no/=yes/;' /etc/default/firebird2.5
-   #- sudo /etc/init.d/firebird2.5-super start
-   #- cd /usr/share/doc/firebird2.5-examples/examples/empbuild/
-   #- sudo gunzip employee.fdb.gz
-   #- sudo chown firebird.firebird employee.fdb
-   #- sudo mv employee.fdb /var/lib/firebird/2.5/data/
-   #- cd -
+env:
+  - CLEANTEST=false
+  - CLEANTEST=true
 
-script:
-   # Override default to make sure we don't run Makefile.PL without --alldeps
-   - make test
+matrix:
+  include:
+    # minimum supported with threads
+    - perl: 5.8.5_thr
+      env:
+        - CLEANTEST=false
+        - BREWOPTS="-Duseithreads"
+        - BREWVER=5.8.5
 
-# Uncomment for cpanm install failures
-#after_failure:
-#   - tail -100 /home/travis/.cpanm/build.log
+    # minimum supported without threads
+    - perl: 5.8.3_nt
+      env:
+        - CLEANTEST=false
+        - BREWOPTS=""
+        - BREWVER=5.8.3
 
-notifications:
-   irc:
-      channels:
-         - "irc.perl.org#dbic-smoke"
-      template:
-         - "%{branch}#%{build_number} by %{author}: %{message} (%{build_url})"
-      on_success: change
-      on_failure: always
-      use_notice: true
+    # this is the perl suse ships
+    - perl: 5.10.0_thr_dbg
+      env:
+        - CLEANTEST=true
+        - BREWOPTS="-DDEBUGGING -Duseithreads"
+        - BREWVER=5.10.0
+
+    # this particular perl is quite widespread
+    - perl: 5.8.8_thr_mb
+      env:
+        - CLEANTEST=true
+        - BREWOPTS="-Duseithreads -Dusemorebits"
+        - BREWVER=5.8.8
+
+
+before_install:
+  # Do not make this part of the env-matrix
+  # different boxes we run on have different amount of hw threads
+  # hence why we need to query
+  # result is 1.5 times the physical threads
+  - export NUMTHREADS=$(( ( $(cut -f 2 -d '-' /sys/devices/system/cpu/online) + 1 ) * 15 / 10  ))
+
+install:
+  # Build and switch to a custom perl if requested
+  # Set the environment based on CLEANTEST
+  # Preinstall/install deps
+  #
+  # sourcing the file is *EXTREMELY* important - otherwise
+  # no envvars will survive
+  - source maint/travis-ci_prepare_env
+
+script:
+  # either a plain 'make test' OR a shuffled prove torture
+  - export HARNESS_TIMER=1 HARNESS_OPTIONS=c:j$NUMTHREADS
+  # use the random order test plan unless CLEANTEST
+  - test "$CLEANTEST" = "true" && make test || prove -lrswj$NUMTHREADS t xt
diff --git a/maint/travis-ci_prepare_env b/maint/travis-ci_prepare_env
new file mode 100755
index 0000000..b336ac7
--- /dev/null
+++ b/maint/travis-ci_prepare_env
@@ -0,0 +1,168 @@
+#!/bin/bash
+
+# * The way .travis.yml is fed to the command controller is idiotic - it
+# makes using multiline `bash -c` statements impossible. Therefore to
+# aid readability (our travis logic is rather complex), the bulk of
+# functionality is moved to a script. More about the problem here:
+# https://github.com/travis-ci/travis-ci/issues/497
+
+set -e
+
+if [[ "$TRAVIS" != "true" ]] ; then
+  echo "Running this script makes no sense outside of travis-ci"
+  exit 1
+fi
+
+#
+# Current dir is the root of the DBIC checkout (make sure to
+# come back there if moving around before end of this script)
+#
+# envvars available for us:
+#
+# NUMTHREADS = { number}
+#   dynamically determined amount of threads we want to run on this
+#   smoker concurrently
+#
+# CLEANTEST = [ true | false ]
+#   controls whether we simulate a "user-side" install experience
+#   that is - no author deps and no M::I installation
+#
+# BREWVER = { tripple dotted perl version, e.g. 5.8.3 }
+#   brew a custom perl version such and such
+#
+# BREWOPTS = { string to be fed unquoted to perlbrew, e.g. -Duseithreads }
+#   build options for perlbrew
+#
+
+if [[ -n "$BREWVER" ]] ; then
+  perlbrew install --as $BREWVER --notest $BREWOPTS -j $NUMTHREADS perl-$BREWVER
+  perlbrew use $BREWVER
+fi
+
+export PERL_MM_USE_DEFAULT=1 PERL_MM_NONINTERACTIVE=1 PERL_AUTOINSTALL_PREFER_CPAN=1
+
+# configure CPAN.pm - older versions get tickled by M::AI and
+# go into an endless loop when trying to autoconf themselves
+perl -e '
+  require CPAN;
+  require CPAN::FirstTime;
+  *CPAN::FirstTime::conf_sites = sub {};
+  CPAN::Config->load;
+  $CPAN::Config->{urllist} = [ "http://cpan.cpantesters.org/" ];
+  CPAN::Config->commit;
+' &> /dev/null
+
+# if this won't be a CLEANTEST (i.e. no deps) - run basic tests using SQLT
+# and set up whatever databases necessary
+if [[ "$CLEANTEST" != "true" ]]; then
+  # extra debian stuff
+  sudo apt-get -y install memcached firebird2.5-super
+  echo -e '\v'
+  echo
+
+  export DBICTEST_SQLT_DEPLOY=1
+
+  # memcached
+  export DBICTEST_MEMCACHED=127.0.0.1:11211
+
+  # mysql
+  mysql -e 'create database dbic_test;'
+  export DBICTEST_MYSQL_DSN='dbi:mysql:database=dbic_test;host=127.0.0.1'
+  export DBICTEST_MYSQL_USER=root
+
+  # pg
+  psql -c 'create database dbic_test;' -U postgres
+  export DBICTEST_PG_DSN='dbi:Pg:database=dbic_test;host=127.0.0.1'
+  export DBICTEST_PG_USER=postgres
+
+  # firebird
+  sudo perl -pi -e 's/\=no/=yes/' /etc/default/firebird2.5
+  sudo /etc/init.d/firebird2.5-super start
+  # FIXME: todo
+  #sudo gsec -add dbic_test -pw 123
+  #export DBICTEST_FIREBIRD_DSN=dbi:Firebird:dbname=/var/lib/firebird/2.5/data/dbic_test
+  #export DBICTEST_FIREBIRD_USER=dbic_test
+  #export DBICTEST_FIREBIRD_PASS=123
+  #export DBICTEST_FIREBIRD_INTERBASE_DSN=dbi:InterBase:dbname=/var/lib/firebird/2.5/data/dbic_test
+  #export DBICTEST_FIREBIRD_INTERBASE_USER=dbic_test
+  #export DBICTEST_FIREBIRD_INTERBASE_PASS=123
+
+  # oracle
+  # FIXME: todo
+  #DBICTEST_ORA_DSN=dbi:Oracle:host=localhost;sid=XE
+  #DBICTEST_ORA_USER=dbic_test
+  #DBICTEST_ORA_PASS=123
+  #DBICTEST_ORA_EXTRAUSER_DSN=dbi:Oracle:host=localhost;sid=XE
+  #DBICTEST_ORA_EXTRAUSER_USER=dbic_test_extra
+  #DBICTEST_ORA_EXTRAUSER_PASS=123
+  #ORACLE_HOME=/usr/lib/oracle/xe/app/oracle/product/10.2.0/client
+fi
+
+installdeps() {
+  if [[ -z "$@" ]] ; then return; fi
+
+  echo -n "Installing $@... "
+  # FIXME
+  # the --reinstall is here because for some reason e.g. `cpanm Carp` does
+  # not install a newer Carp since it already exists on the system
+  # investigation pending
+  if ! OUT=$( echo "$@" | xargs -n 1 -P $NUMTHREADS cpanm --reinstall --verbose --notest 2>&1 ) ; then
+    EX=$?
+    echo "FAILED !!!"
+    echo "$OUT"
+    exit $?
+  else
+    echo "done."
+  fi
+}
+
+# Install *hard dep* modules that typically appear in more than one dep
+#
+# *NEVER* add optional depenencies here - will make CLEANTEST=true smokes useless
+#
+# do it in several passes to minimize amount of cross-deps installing multiple
+# times, and to avoid module re-architecture breaking another install
+# (e.g. once Carp is upgraded there's no more Carp::Heavy)
+#
+installdeps $(cpanm --notest --quiet --showdeps Module::Build)
+installdeps Test::Exception Test::Fatal Module::Runtime Carp
+installdeps Sub::Name multidimensional namespace::clean Class::XSAccessor MRO::Compat
+installdeps DBI Moo Class::Accessor::Grouped
+
+if [[ "$CLEANTEST" = "true" ]]; then
+  # get the last inc/ off cpan - we will get rid of MI
+  # soon enough, but till then this will do
+  # the point is to have a *really* clean perl (the ones
+  # we build are guaranteed to be clean, without side
+  # effects from travis preinstalls)
+  #
+  # FIXME - not yet implemented - not sure how to reliably
+  # feed to wget this location: http://cpansearch.perl.org/src/GETTY/DBIx-Class-0.08204/inc/
+  echo TODOOOO - M::I is not supposed to be installed here - this test is useless now
+  installdeps Module::Install
+else
+  # we will be running all tests, preinstall MOAR stuff
+  installdeps Module::Install DateTime::Format::Strptime MooseX::Types JSON::Any Class::DBI
+fi
+
+# install the rest
+perl Makefile.PL
+installdeps $(make listalldeps)
+
+# FIXME - for some reason the above invocation does not upgrade outdated
+# core libs - just punt and install the remainder by hand (these run tests)
+# This snippet should not be here at all
+perl Makefile.PL &> /dev/null
+make installdeps
+
+# announce what are we running
+perl -V
+echo "Using $NUMTHREADS Travis-CI concurrent processes"
+
+# make sure we got everything we need
+perl Makefile.PL &> /dev/null
+if [[ -n "$(make listdeps)" ]] ; then
+  echo "Not all deps installed - something went wrong :("
+  make listdeps
+  exit 1
+fi
diff --git a/t/lib/DBICTest/RunMode.pm b/t/lib/DBICTest/RunMode.pm
index 868183f..9e99009 100644
--- a/t/lib/DBICTest/RunMode.pm
+++ b/t/lib/DBICTest/RunMode.pm
@@ -150,7 +150,9 @@ sub is_author {
 }
 
 sub is_smoker {
-  return ( $ENV{AUTOMATED_TESTING} && ! $ENV{PERL5_CPANM_IS_RUNNING} && ! $ENV{RELEASE_TESTING} )
+  return $ENV{TRAVIS} or (
+    $ENV{AUTOMATED_TESTING} && ! $ENV{PERL5_CPANM_IS_RUNNING} && ! $ENV{RELEASE_TESTING}
+  )
 }
 
 sub is_plain {