From: Peter Rabbitson Date: Fri, 17 Jun 2016 07:59:40 +0000 (+0200) Subject: (travis) Port all travis changes since last release X-Git-Tag: v0.082840~26 X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=commitdiff_plain;h=209e3f3725b8ec9ac0622bf41b9c9a017f6fcda0;p=dbsrgits%2FDBIx-Class.git (travis) Port all travis changes since last release --- diff --git a/.travis.yml b/.travis.yml index 54ee2ec..87d8fe7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,10 +14,6 @@ # # Smoke all branches except for blocked* and wip/* -# -# Additionally master does not smoke with bleadperl -# ( implemented in maint/travis-ci_scripts/10_before_install.bash ) -# branches: except: - /^wip\// @@ -35,7 +31,7 @@ notifications: email: recipients: - - ribasushi@cpan.org + - CPAN-CI@leporine.io on_success: change on_failure: always @@ -62,6 +58,7 @@ env: CLEANTEST=true perl: - "5.8" + - "5.10" - "5.22-extras" matrix: @@ -79,6 +76,12 @@ matrix: env: - CLEANTEST=false + - perl: "5.10" + sudo: required + dist: precise + env: + - CLEANTEST=false + - perl: "5.22-extras" sudo: required dist: precise @@ -132,12 +135,14 @@ matrix: - DBIC_TRACE_PROFILE=console - BREWVER=5.8.4 - # CLEANTEST: this particular perl is quite widespread + # Full test: this particular perl is quite widespread, single thread - perl: "5.8.8_thr" - sudo: false + sudo: required dist: precise + group: legacy env: - - CLEANTEST=true + - VCPU_USE=1 + - CLEANTEST=false - BREWVER=5.8.8 - BREWOPTS="-Duseithreads" @@ -182,8 +187,8 @@ matrix: dist: precise env: - CLEANTEST=false - - POISON_ENV=true - DBIC_TRACE=1 + - DBICTEST_VERSION_WARNS_INDISCRIMINATELY=1 - BREWVER=5.16.3 - BREWOPTS="-Duseithreads -Dusemorebits" @@ -199,9 +204,51 @@ matrix: - DBICTEST_VIA_REPLICATED=0 - DBICTEST_VERSION_WARNS_INDISCRIMINATELY=1 + # MAKE SURE TO KEEP THE FLAGS IDENTICAL TO CPERL BELOW + # allows for easier side-by-side comparison + # vcpu=1 for even more stable results + - perl: "5.22.2_thr_qm" + # explicit new infra spec preparing for a future forced upgrade + # also need to pull in a sufficiently new compiler for quadmath.h + sudo: required + dist: trusty + env: + - VCPU_USE=1 + - CLEANTEST=true + - POISON_ENV=true + - MVDT=false + - BREWVER=5.22.2 + - BREWOPTS="-Duseithreads -Dusequadmath" + ### # Start of the allow_failures block + # MAKE SURE TO KEEP THE FLAGS IDENTICAL TO STOCK 5.latest.comparable ABOVE + # allows for easier side-by-side comparison + # vcpu=1 for even more stable results + - perl: "cperl-5.22.3_thr_qm" + # explicit new infra spec preparing for a future forced upgrade + # also need to pull in a sufficiently new compiler for quadmath.h + sudo: required + dist: trusty + env: + - VCPU_USE=1 + - CLEANTEST=true + - POISON_ENV=true + - MVDT=false + - BREWVER=cperl-5.22.3 + - BREWOPTS="-Duseithreads -Dusequadmath" + + - perl: "cperl-master_thr" + sudo: false + dist: precise + env: + - CLEANTEST=true + - POISON_ENV=true + - MVDT=false + - BREWVER=cperl-master + - BREWOPTS="-Duseithreads" + # threaded oldest possible with blead CPAN - perl: "devcpan_5.8.1_thr_mb" sudo: false @@ -287,11 +334,12 @@ matrix: - POISON_ENV=true - BREWVER=blead - # bleadperl with blead CPAN + # bleadperl with blead CPAN, single thread - perl: "devcpan_bleadperl_thr_mb" sudo: required # explicitly do not specify dist - see what the default does env: + - VCPU_USE=1 - CLEANTEST=false - DEVREL_DEPS=true - BREWVER=blead @@ -311,6 +359,8 @@ matrix: allow_failures: # these run with various dev snapshots - allowed to fail + - perl: cperl-5.22.3_thr_qm + - perl: cperl-master_thr - perl: devcpan_5.8.1_thr_mb - perl: devcpan_5.8.1 - perl: devcpan_5.8.3_mb @@ -375,9 +425,9 @@ script: - set -e after_success: - # Check if we can assemble a dist properly if not in CLEANTEST + # Disabled # - - maint/getstatus maint/travis-ci_scripts/50_after_success.bash + #- maint/getstatus maint/travis-ci_scripts/50_after_success.bash after_failure: # Final sysinfo printout on fail diff --git a/maint/travis-ci_scripts/10_before_install.bash b/maint/travis-ci_scripts/10_before_install.bash index 80315a0..4fa1338 100755 --- a/maint/travis-ci_scripts/10_before_install.bash +++ b/maint/travis-ci_scripts/10_before_install.bash @@ -71,37 +71,63 @@ if [[ "$CLEANTEST" != "true" ]]; then "sudo bash -c 'dd if=/dev/zero of=/swap.img bs=256M count=5 && chmod 600 /swap.img && mkswap /swap.img && swapon /swap.img'" fi - export CACHE_DIR="/tmp/poormanscache" - - # - # FIXME these debconf lines should automate the firebird config but do not :((( - sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/enabled\tboolean\ttrue" | debconf-set-selections' - sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/sysdba_password/new_password\tpassword\t123" | debconf-set-selections' - - # these APT sources do not mean anything to us anyway - sudo rm -rf /etc/apt/sources.list.d/* - - # the actual package is built for lucid, installs fine on both precise and trusty - sudo bash -c 'echo "deb http://archive.canonical.com/ubuntu precise partner" >> /etc/apt/sources.list' # never installed, this looks like trusty if [[ ! -d /var/lib/mysql ]] ; then sudo dpkg --add-architecture i386 - extra_debs="$extra_debs postgresql mysql-server" + extra_debs+=( postgresql mysql-server ) fi + + # these APT sources do not mean anything to us anyway + sudo rm -rf /etc/apt/sources.list.d/* + + # + # FIXME these debconf lines should automate the firebird config but seem not to :((( + sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/enabled\tboolean\ttrue" | debconf-set-selections' + sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/sysdba_password/new_password\tpassword\t123" | debconf-set-selections' + run_or_err "Updating APT sources" "sudo apt-get update" + apt_install ${extra_debs[@]} libmysqlclient-dev memcached firebird2.5-super firebird2.5-dev expect - apt_install $extra_debs libmysqlclient-dev memcached firebird2.5-super firebird2.5-dev unixodbc-dev expect # need to stop them again, in case we installed them above (trusty) for d in mysql postgresql ; do run_or_err "Stopping $d" "sudo /etc/init.d/$d stop || /bin/true" done - run_or_err "Cloning poor man's cache from github" "git clone --depth=1 --single-branch --branch=oracle/10.2.0 https://github.com/poormanscache/poormanscache.git $CACHE_DIR && $CACHE_DIR/reassemble" - run_or_err "Installing OracleXE manually from deb" \ - "sudo dpkg -i $CACHE_DIR/apt_cache/bc-multiarch-travis_1.0_all.deb $CACHE_DIR/apt_cache/oracle-xe_10.2.0.1-1.1_i386.deb || sudo bash -c 'source maint/travis-ci_scripts/common.bash && apt_install -f'" + + export CACHE_DIR="/tmp/poormanscache" + mkdir "$CACHE_DIR" + + # FIXME - by default db2 eats too much memory, we won't be able to test on legacy infra + # someone needs to add a minimizing configuration akin to 9367d187 + if [[ "$(free -m | grep 'Mem:' | perl -p -e '$_ = (split /\s+/, $_)[1]')" -gt 4000 ]] ; then + run_or_err "Getting DB2 from poor man's cache github" ' + wget -qO- https://github.com/poormanscache/poormanscache/archive/DB2_ExC/9.7.5_deb_x86-64.tar.gz \ + | tar -C "$CACHE_DIR" -zx' + + # the actual package is built for lucid, installs fine on both precise and trusty + manual_debs+=( "db2exc_9.7.5-0lucid0_amd64.deb" ) + fi + + run_or_err "Getting Oracle from poor man's cache github" ' + wget -qO- https://github.com/poormanscache/poormanscache/archive/OracleXE/10.2.0_deb_mixed.tar.gz \ + | tar -C "$CACHE_DIR" -zx' + manual_debs+=( "bc-multiarch-travis_1.0_all.deb" "oracle-xe_10.2.0.1-1.1_i386.deb" ) + + + # reassemble chunked pieces ( working around github's filesize limit ) + for reass in $CACHE_DIR/*/reassemble ; do /bin/bash "$reass" ; done + + run_or_err "Installing RDBMS debs manually: $( echo ${manual_debs[@]/#/$CACHE_DIR/*/*/} )" \ + "sudo dpkg -i $( echo ${manual_debs[@]/#/$CACHE_DIR/*/*/} ) || sudo bash -c 'source maint/travis-ci_scripts/common.bash && apt_install -f'" + + + # needs to happen separately and *after* db2exc, as the former shits all over /usr/include (wtf?!) + # for more info look at /opt/ibm/db2/V9.7/instance/db2iutil :: create_links() + apt_install unixodbc-dev + ### config memcached run_or_err "Starting memcached" "sudo /etc/init.d/memcached start" @@ -248,5 +274,25 @@ FileUsage = 1 GRANT connect,resource TO $DBICTEST_ORA_EXTRAUSER_USER; '" - export ORACLE_HOME="$CACHE_DIR/ora_instaclient/x86-64/oracle_instaclient_10.2.0.5.0" + export ORACLE_HOME="$CACHE_DIR/poormanscache-OracleXE-10.2.0_deb_mixed/ora_instaclient/x86-64/oracle_instaclient_10.2.0.5.0" + +### config db2exc + # we may have skipped installation due to low memory + if dpkg -l db2exc &>/dev/null ; then + # WTF is this world-writable? + # Strip the write bit so it doesn't trip Ubuntu's symlink-in-/tmp attack mitigation + sudo chmod -R o-w ~dasusr1/das + + export DB2_HOME=/opt/ibm/db2/V9.7 + export DBICTEST_DB2_DSN=dbi:DB2:DATABASE=dbictest + export DBICTEST_DB2_USER=db2inst1 + export DBICTEST_DB2_PASS=abc123456 + + run_or_err "Set up DB2 users" \ + "echo -e '$DBICTEST_DB2_PASS\n$DBICTEST_DB2_PASS' | sudo passwd $DBICTEST_DB2_USER" + + run_or_err "Create DB2 database" \ + "sudo -u $DBICTEST_DB2_USER -i db2 'CREATE DATABASE dbictest' && sudo -u $DBICTEST_DB2_USER -i db2 'ACTIVATE DATABASE dbictest'" + fi + fi diff --git a/maint/travis-ci_scripts/20_install.bash b/maint/travis-ci_scripts/20_install.bash index 2d6fbf2..3a4ff7b 100755 --- a/maint/travis-ci_scripts/20_install.bash +++ b/maint/travis-ci_scripts/20_install.bash @@ -4,7 +4,7 @@ if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi # we need a mirror that both has the standard index and a backpan version rolled # into one, due to MDV testing -CPAN_MIRROR="http://cpan.metacpan.org/" +export CPAN_MIRROR="http://cpan.metacpan.org/" PERL_CPANM_OPT="$PERL_CPANM_OPT --mirror $CPAN_MIRROR" @@ -31,17 +31,32 @@ if [[ -n "$BREWVER" ]] ; then BREWSRC="$BREWVER" - if [[ "$BREWVER" == "schmorp_stableperl" ]] ; then + if is_cperl; then + if [[ "$BREWVER" == "cperl-master" ]] ; then + git clone --single-branch --depth=1 --branch=master https://github.com/perl11/cperl /tmp/cperl-master + BREWSRC="/tmp/cperl-master" + else + # FFS perlbrew ( see http://wollmers-perl.blogspot.de/2015/10/install-cperl-with-perlbrew.html ) + wget -qO- https://github.com/perl11/cperl/archive/$BREWVER.tar.gz > /tmp/cperl-$BREWVER.tar.gz + BREWSRC="/tmp/cperl-$BREWVER.tar.gz" + fi + elif [[ "$BREWVER" == "schmorp_stableperl" ]] ; then BREWSRC="http://stableperl.schmorp.de/dist/stableperl-5.22.0-1.001.tar.gz" fi run_or_err "Compiling/installing Perl $BREWVER (without testing, using ${perlbrew_jopt:-1} threads, may take up to 5 minutes)" \ "perlbrew install --as $BREWVER --notest --noman --verbose $BREWOPTS -j${perlbrew_jopt:-1} $BREWSRC" - # can not do 'perlbrew uss' in the run_or_err subshell above, or a $() - # furthermore `perlbrew use` returns 0 regardless of whether the perl is - # found (won't be there unless compilation suceeded, wich *ALSO* returns 0) - perlbrew use $BREWVER + # FIXME work around https://github.com/perl11/cperl/issues/144 + # (still affecting 5.22.3) + if is_cperl && ! [[ -f ~/perl5/perlbrew/perls/$BREWVER/bin/perl ]] ; then + ln -s ~/perl5/perlbrew/perls/$BREWVER/bin/cperl ~/perl5/perlbrew/perls/$BREWVER/bin/perl + fi + + # can not do 'perlbrew use' in the run_or_err subshell above, or a $() + # furthermore some versions of `perlbrew use` return 0 regardless of whether + # the perl is found (won't be there unless compilation suceeded, wich *ALSO* returns 0) + perlbrew use $BREWVER || /bin/true if [[ "$( perlbrew use | grep -oP '(?<=Currently using ).+' )" != "$BREWVER" ]] ; then echo_err "Unable to switch to $BREWVER - compilation failed...?" @@ -55,6 +70,13 @@ elif [[ "$CLEANTEST" == "true" ]] && [[ "$POISON_ENV" != "true" ]] ; then purge_sitelib fi +if [[ "$POISON_ENV" = "true" ]] ; then + # create a perlbrew-specific local lib + perlbrew lib create travis-local + perlbrew use "$( perlbrew use | grep -oP '(?<=Currently using ).+' )@travis-local" + echo_err "POISON_ENV active - adding a local lib: $(perlbrew use)" +fi + # configure CPAN.pm - older versions go into an endless loop # when trying to autoconf themselves CPAN_CFG_SCRIPT=" @@ -63,7 +85,7 @@ CPAN_CFG_SCRIPT=" *CPAN::FirstTime::conf_sites = sub {}; CPAN::Config->load; \$CPAN::Config->{urllist} = [qw{ $CPAN_MIRROR }]; - \$CPAN::Config->{halt_on_failure} = 1; + \$CPAN::Config->{halt_on_failure} = $( is_cperl && echo -n 0 || echo -n 1 ); CPAN::Config->commit; " run_or_err "Configuring CPAN.pm" "perl -e '$CPAN_CFG_SCRIPT'" @@ -86,6 +108,17 @@ fi # poison the environment if [[ "$POISON_ENV" = "true" ]] ; then + toggle_vars=( MVDT ) + + [[ "$CLEANTEST" == "true" ]] && toggle_vars+=( BREAK_CC ) + + for var in "${toggle_vars[@]}" ; do + if [[ -z "${!var}" ]] ; then + export $var=true + echo "POISON_ENV: setting $var to 'true'" + fi + done + # look through lib, find all mentioned DBIC* ENVvars and set them to true and see if anything explodes toggle_booleans=( $( grep -ohP '\bDBIC_[0-9_A-Z]+' -r lib/ --exclude-dir Optional | sort -u | grep -vP '^(DBIC_TRACE(_PROFILE)?|DBIC_.+_DEBUG)$' ) ) @@ -111,16 +144,20 @@ if [[ "$POISON_ENV" = "true" ]] ; then fi done - # emulate a local::lib-like env + +### emulate a local::lib-like env + # trick cpanm into executing true as shell - we just need the find+unpack run_or_err "Downloading latest stable DBIC from CPAN" \ "SHELL=/bin/true cpanm --look DBIx::Class" - export PERL5LIB="$( ls -d ~/.cpanm/latest-build/DBIx-Class-*/lib | tail -n1 ):$PERL5LIB" + # move it somewhere as following cpanm will clobber it + run_or_err "Moving latest stable DBIC from CPAN to /tmp" "mv ~/.cpanm/latest-build/DBIx-Class-*/lib /tmp/stable_dbic_lib" + + export PERL5LIB="/tmp/stable_dbic_lib:$PERL5LIB" # perldoc -l searches $(pwd)/lib in addition to PERL5LIB etc, hence the cd / echo_err "Latest stable DBIC (without deps) locatable via \$PERL5LIB at $(cd / && perldoc -l DBIx::Class)" - fi if [[ "$CLEANTEST" != "true" ]] ; then diff --git a/maint/travis-ci_scripts/30_before_script.bash b/maint/travis-ci_scripts/30_before_script.bash index fae39e6..3feaa2c 100755 --- a/maint/travis-ci_scripts/30_before_script.bash +++ b/maint/travis-ci_scripts/30_before_script.bash @@ -5,24 +5,59 @@ source maint/travis-ci_scripts/common.bash if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then exit 0 ; fi -# The prereq-install stage will not work with both POISON and DEVREL +# The DEVREL_DEPS prereq-install stage won't mix with MVDT # DEVREL wins -if [[ "$DEVREL_DEPS" = "true" ]] ; then - export POISON_ENV="" +if [[ "$DEVREL_DEPS" == "true" ]] ; then + export MVDT="" fi -# FIXME - this is a kludge in place of proper MDV testing. For the time +# Need a shitton of patches to run on cperl (luckily all provided) +if is_cperl ; then + + run_or_err "Downloading and installing cperl distroprefs" ' + wget -qO- https://github.com/rurban/distroprefs/archive/master.tar.gz |\ + tar -C $HOME/.cpan --strip-components 1 -zx distroprefs-master/prefs distroprefs-master/sources + ' + + # Argh -DFORTIFY_INC!!! + # FIXME - remove when M::I is gone + export PERL5LIB="$PERL5LIB:." + + # Also need to have YAML in place, otherwise the distroprefs are not readable + # work around https://github.com/perl11/cperl/issues/155#issuecomment-224862978 + perl -MYAML -e1 &>/dev/null || installdeps YAML + +fi + + +# announce what are we running +echo_err "$(ci_vm_state_text)" + + +# FIXME - this is a kludge in place of proper MVDT testing. For the time # being simply use the minimum versions of our DBI/DBDstack, to avoid # fuckups like 0.08260 (went unnoticed for 5 months) -if [[ "$POISON_ENV" = "true" ]] ; then +if [[ "$MVDT" == "true" ]] ; then # use url-spec for DBI due to https://github.com/miyagawa/cpanminus/issues/328 if [[ "$CLEANTEST" != "true" ]] || perl -M5.013003 -e1 &>/dev/null ; then # the fulltest may re-upgrade DBI, be conservative only on cleantests # earlier DBI will not compile without PERL_POLLUTE which was gone in 5.14 parallel_installdeps_notest T/TI/TIMB/DBI-1.614.tar.gz + + # FIXME work around DBD::DB2 being silly: https://rt.cpan.org/Ticket/Display.html?id=101659 + if [[ -n "$DBICTEST_DB2_DSN" ]] ; then + echo_err "Installing same DBI version into the main perl (above the current local::lib)" + $SHELL -lic "perlbrew use $( perlbrew use | grep -oP '(?<=Currently using )[^@]+' ) && parallel_installdeps_notest T/TI/TIMB/DBI-1.614.tar.gz" + fi else parallel_installdeps_notest T/TI/TIMB/DBI-1.57.tar.gz + + # FIXME work around DBD::DB2 being silly: https://rt.cpan.org/Ticket/Display.html?id=101659 + if [[ -n "$DBICTEST_DB2_DSN" ]] ; then + echo_err "Installing same DBI version into the main perl (above the current local::lib)" + $SHELL -lic "perlbrew use $( perlbrew use | grep -oP '(?<=Currently using )[^@]+' ) && parallel_installdeps_notest T/TI/TIMB/DBI-1.57.tar.gz" + fi fi # Test both minimum DBD::SQLite and minimum BigInt SQLite @@ -32,23 +67,38 @@ if [[ "$POISON_ENV" = "true" ]] ; then else parallel_installdeps_notest DBD::SQLite@1.29 fi +fi - # also try minimal tested installs *without* a compiler - if [[ "$CLEANTEST" = "true" ]]; then - - # Clone and P::S::XS are both bugs - # File::Spec can go away as soon as I dump Path::Class - # File::Path is there because of RT#107392 (sigh) - # List::Util can be excised after that as well (need to make my own max() routine for older perls) +# +# try minimal fully tested installs *without* a compiler (with some exceptions of course) +if [[ "$BREAK_CC" == "true" ]] ; then - installdeps Sub::Name Clone Package::Stash::XS \ - $( perl -MFile::Spec\ 3.26 -e1 &>/dev/null || echo "File::Path File::Spec" ) \ - $( perl -MList::Util\ 1.16 -e1 &>/dev/null || echo "List::Util" ) + [[ "$CLEANTEST" != "true" ]] && echo_err "Breaking the compiler without CLEANTEST makes no sense" && exit 1 - mkdir -p "$HOME/bin" # this is already in $PATH, just doesn't exist - run_or_err "Linking ~/bin/cc to /bin/false - thus essentially BREAKING the C compiler" \ - "ln -s /bin/false $HOME/bin/cc" - fi + # FIXME - working around RT#74707, https://metacpan.org/source/DOY/Package-Stash-0.37/Makefile.PL#L112-122 + # + # DEVREL_DEPS means our installer is cpanm, which will respect failures + # and the like, so stuff soft-failing (failed deps that are not in fact + # needed) will not fly. Add *EVEN MORE* stuff that needs a compiler + # + # FIXME - the PathTools 3.47 is to work around https://rt.cpan.org/Ticket/Display.html?id=107392 + # + installdeps Sub::Name Clone Package::Stash::XS \ + $( perl -MFile::Spec\ 3.26 -e1 &>/dev/null || echo "File::Path File::Spec" ) \ + $( perl -MList::Util\ 1.16 -e1 &>/dev/null || echo "List::Util" ) \ + $( [[ "$DEVREL_DEPS" == "true" ]] && ( perl -MFile::Spec\ 3.13 -e1 &>/dev/null || echo "S/SM/SMUELLER/PathTools-3.47.tar.gz" ) ) \ + $( perl -MDBI -e1 &>/dev/null || echo "DBI" ) \ + $( perl -MDBD::SQLite -e1 &>/dev/null || echo "DBD::SQLite" ) + + mkdir -p "$HOME/bin" # this is already in $PATH, just doesn't exist + run_or_err "Linking ~/bin/cc to /bin/false - thus essentially BREAKING the C compiler" \ + "ln -s /bin/false $HOME/bin/cc" + + # FIXME: working around RT#113682, and some other unfiled bugs + installdeps Module::Build Devel::GlobalDestruction Class::Accessor::Grouped + + run_or_err "Linking ~/bin/cc to /bin/true - BREAKING the C compiler even harder" \ + "ln -fs /bin/true $HOME/bin/cc" fi if [[ "$CLEANTEST" = "true" ]]; then @@ -57,7 +107,7 @@ if [[ "$CLEANTEST" = "true" ]]; then # the point is to have a *really* clean perl (the ones # we build are guaranteed to be clean, without side # effects from travis preinstalls) - + # # trick cpanm into executing true as shell - we just need the find+unpack [[ -d ~/.cpanm/latest-build/DBIx-Class-*/inc ]] || run_or_err "Downloading latest stable DBIC inc/ from CPAN" \ "SHELL=/bin/true cpanm --look DBIx::Class" @@ -73,8 +123,8 @@ if [[ "$CLEANTEST" = "true" ]]; then # handholding if [[ "$DEVREL_DEPS" = "true" ]] ; then - # FIXME - work around RT#110882, sigh... - perl -Mversion\ 0.87 -e 1 &>/dev/null || installdeps version@0.9912 + # nothing for now + /bin/true elif ! CPAN_is_sane ; then # no configure_requires - we will need the usual suspects anyway # without pre-installing these in one pass things won't yet work @@ -86,14 +136,11 @@ else # do the preinstall in several passes to minimize amount of cross-deps installing # multiple times, and to avoid module re-architecture breaking another install - # (e.g. once Carp is upgraded there's no more Carp::Heavy, - # while a File::Path upgrade may cause a parallel EUMM run to fail) + # (e.g. once Carp is upgraded there's no more Carp::Heavy) # - parallel_installdeps_notest File::Path parallel_installdeps_notest Carp parallel_installdeps_notest Module::Build - parallel_installdeps_notest File::Spec Module::Runtime - parallel_installdeps_notest Test::Exception Encode::Locale Test::Fatal + parallel_installdeps_notest Test::Exception Encode::Locale Test::Fatal Module::Runtime parallel_installdeps_notest Test::Warn B::Hooks::EndOfScope Test::Differences HTTP::Status parallel_installdeps_notest Test::Pod::Coverage Test::EOL Devel::GlobalDestruction Sub::Name MRO::Compat Class::XSAccessor URI::Escape HTML::Entities parallel_installdeps_notest YAML LWP Class::Trigger DateTime::Format::Builder Class::Accessor::Grouped Package::Variant @@ -114,13 +161,12 @@ else fi fi -# generate the makefile which will have different deps depending on -# the runmode and envvars set above -run_or_err "Configure on current branch" "perl Makefile.PL" # install (remaining) dependencies, sometimes with a gentle push if [[ "$CLEANTEST" = "true" ]]; then + run_or_err "Configure on current branch" "perl Makefile.PL" + # we are doing a devrel pass - try to upgrade *everything* (we will be using cpanm so safe-ish) if [[ "$DEVREL_DEPS" == "true" ]] ; then @@ -143,16 +189,34 @@ if [[ "$CLEANTEST" = "true" ]]; then installdeps $HARD_DEPS + run_or_err "Re-configure" "perl Makefile.PL" + else - parallel_installdeps_notest "$(make listdeps | sort -R)" + run_or_err "Configure on current branch with --with-optdeps" "perl Makefile.PL --with-optdeps" + + # FIXME - evil evil work around for https://github.com/Manwar/Test-Strict/issues/17 + if perl -M5.025 -e1 &>/dev/null; then + mkdir -p "$( perl -MConfig -e 'print $Config{sitelib}' )/Devel" + cat < "$( perl -MConfig -e 'print $Config{sitelib}' )/Devel/Cover.pm" +package Devel::Cover; +our \$VERSION = 0.43; +1; +MyDevelCover + fi + + # if we are smoking devrels - make sure we upgrade everything we know about + if [[ "$DEVREL_DEPS" == "true" ]] ; then + parallel_installdeps_notest "$(make listalldeps | sort -R)" + else + parallel_installdeps_notest "$(make listdeps | sort -R)" + fi + run_or_err "Re-configure with --with-optdeps" "perl Makefile.PL --with-optdeps" fi echo_err "$(tstamp) Dependency installation finished" -run_or_err "Re-configure" "perl Makefile.PL" - # make sure we got everything we need if [[ -n "$(make listdeps)" ]] ; then echo_err "$(tstamp) Not all deps installed - something went wrong :(" @@ -163,20 +227,12 @@ if [[ -n "$(make listdeps)" ]] ; then exit 1 fi -# check that our MDV somewhat works -if [[ "$POISON_ENV" = "true" ]] && ( perl -MDBD::SQLite\ 1.38 -e1 || perl -MDBI\ 1.615 -e1 ) &>/dev/null ; then +# check that our MVDT somewhat works +if [[ "$MVDT" == "true" ]] && ( perl -MDBD::SQLite\ 1.38 -e1 || perl -MDBI\ 1.615 -e1 ) &>/dev/null ; then echo_err "Something went wrong - higher versions of DBI and/or DBD::SQLite than we expected" exit 1 fi -if [[ "$CLEANTEST" = "true" ]] && perl -MModule::Build::Tiny -e1 &>/dev/null ; then - echo_err "Module::Build::Tiny pulled in during the basic depchain install - this must not happen" - exit 1 -fi - -# announce what are we running echo_err " ===================== DEPENDENCY CONFIGURATION COMPLETE ===================== -$(tstamp) Configuration phase seems to have taken $(date -ud "@$SECONDS" '+%H:%M:%S') (@$SECONDS) - -$(ci_vm_state_text)" +$(tstamp) Configuration phase seems to have taken $(date -ud "@$SECONDS" '+%H:%M:%S') (@$SECONDS)" diff --git a/maint/travis-ci_scripts/40_script.bash b/maint/travis-ci_scripts/40_script.bash index 9b38103..0a6ecd5 100755 --- a/maint/travis-ci_scripts/40_script.bash +++ b/maint/travis-ci_scripts/40_script.bash @@ -14,11 +14,14 @@ run_harness_tests() { ulim=$(( ( $(ps xH | wc -l) - 3 ) + 4 )) # (real count excluding header + ps + wc) + space for ( make + tee + harness + ) echo_err "$(tstamp) Setting process/thread limit to $ulim" ulimit -u $ulim - sleep 10 # needed to settle things down a bit + sleep 5 # needed to settle things down a bit fi make test 2> >(tee "$TEST_STDERR_LOG") } +# announce everything we have on this box +TRAVIS="" perl -Ilib t/00describe_environment.t >/dev/null + TEST_T0=$SECONDS if [[ "$CLEANTEST" = "true" ]] ; then echo_err "$(tstamp) Running tests with plain \`make test\`" @@ -27,6 +30,11 @@ if [[ "$CLEANTEST" = "true" ]] ; then else PROVECMD="prove -lrswTj$VCPU_USE xt t" + # List every single SKIP/TODO when they are visible + if [[ "$VCPU_USE" == 1 ]] ; then + PROVECMD="$PROVECMD --directives" + fi + echo_err "$(tstamp) running tests with \`$PROVECMD\`" $PROVECMD 2> >(tee "$TEST_STDERR_LOG") fi diff --git a/maint/travis-ci_scripts/50_after_success.bash b/maint/travis-ci_scripts/50_after_success.bash index fdb6a64..8b44371 100755 --- a/maint/travis-ci_scripts/50_after_success.bash +++ b/maint/travis-ci_scripts/50_after_success.bash @@ -9,7 +9,11 @@ if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] || [[ "$TRAVIS_PULL_REQUEST" != "false" ]] ; export VCPU_USE="$VCPU_AVAILABLE" export HARNESS_OPTIONS="j$VCPU_USE" + if [[ "$DEVREL_DEPS" == "true" ]] && perl -M5.008003 -e1 &>/dev/null ; then + + [[ "$BREAK_CC" == "true" ]] && run_or_err "Unbreaking previously broken ~/bin/cc" "rm $HOME/bin/cc" + # FIXME - Devel::Cover (brought by Test::Strict, but soon needed anyway) # does not test cleanly on 5.8.7 - just get it directly if perl -M5.008007 -e1 &>/dev/null && ! perl -M5.008008 -e1 &>/dev/null; then @@ -23,21 +27,20 @@ if [[ "$DEVREL_DEPS" == "true" ]] && perl -M5.008003 -e1 &>/dev/null ; then parallel_installdeps_notest YAML Lexical::SealRequireHints fi - parallel_installdeps_notest "Module::Install@1.15" - # FIXME Change when Moose goes away - installdeps Moose $(perl -Ilib -MDBIx::Class -e 'print join " ", keys %{DBIx::Class::Optional::Dependencies->req_list_for("dist_dir")}') + installdeps Moose $(perl -Ilib -MDBIx::Class::Optional::Dependencies=-list_missing,dist_dir) - run_or_err "Attempt to build a dist" "rm -rf inc/ && perl Makefile.PL --skip-author-deps && make dist" + run_or_err "Attempt to build a dist" "rm -rf inc/ && perl Makefile.PL && make dist" tarball_assembled=1 elif [[ "$CLEANTEST" != "true" ]] ; then - parallel_installdeps_notest "Module::Install@1.15" $(perl -Ilib -MDBIx::Class -e 'print join " ", keys %{DBIx::Class::Optional::Dependencies->req_list_for("dist_dir")}') + parallel_installdeps_notest $(perl -Ilib -MDBIx::Class::Optional::Dependencies=-list_missing,dist_dir) run_or_err "Attempt to build a dist from original checkout" "make dist" tarball_assembled=1 fi + if [[ -n "$tarball_assembled" ]] ; then echo "Contents of the resulting dist tarball:" @@ -88,9 +91,11 @@ if [[ -n "$tarball_assembled" ]] ; then export $e="" done + # FIXME - for some reason a plain `cpan .` does not work in this case + # no time to investigate run_or_err \ "Attempt to configure/test/build/install dist using latest CPAN@$(perl -MCPAN -e 'print CPAN->VERSION')" \ - "cpan ." + "perl -MCPAN -e 'install( q{.} )'" else run_or_err \ diff --git a/maint/travis-ci_scripts/common.bash b/maint/travis-ci_scripts/common.bash index de89f94..07c6e70 100755 --- a/maint/travis-ci_scripts/common.bash +++ b/maint/travis-ci_scripts/common.bash @@ -15,6 +15,14 @@ fi tstamp() { echo -n "[$(date '+%H:%M:%S')]" ; } +CPAN_is_sane() { perl -MCPAN\ 1.94_56 -e 1 &>/dev/null ; } + +CPAN_supports_BUILDPL() { perl -MCPAN\ 1.9205 -e1 &>/dev/null; } + +have_sudo() { sudo /bin/true &>/dev/null ; } + +is_cperl() { [[ "$BREWVER" =~ $( echo -n "^cperl-" ) ]] ; } + ci_vm_state_text() { echo " ========================== CI System information ============================ @@ -91,7 +99,7 @@ apt_install() { # flatten pkgs="$@" - run_or_err "Installing Debian APT packages: $pkgs" "sudo apt-get install --allow-unauthenticated --no-install-recommends -y $pkgs" + run_or_err "Installing APT packages: $pkgs" "sudo apt-get install --allow-unauthenticated --no-install-recommends -y $pkgs" } extract_prereqs() { @@ -135,8 +143,10 @@ extract_prereqs() { parallel_installdeps_notest() { if [[ -z "$@" ]] ; then return; fi + is_cperl && echo_err "cpanminus is not yet usable on cperl" && exit 1 + # one module spec per line - MODLIST="$(printf '%s\n' "$@")" + MODLIST="$(printf '%s\n' "$@" | sort -R)" # We want to trap the output of each process and serially append them to # each other as opposed to just dumping a jumbled up mass-log that would @@ -165,7 +175,7 @@ parallel_installdeps_notest() { " } -export -f parallel_installdeps_notest run_or_err echo_err tstamp +export -f parallel_installdeps_notest run_or_err echo_err tstamp is_cperl have_sudo CPAN_is_sane CPAN_supports_BUILDPL installdeps() { if [[ -z "$@" ]] ; then return; fi @@ -194,6 +204,8 @@ installdeps() { _dep_inst_with_test() { if [[ "$DEVREL_DEPS" == "true" ]] ; then + is_cperl && echo_err "cpanminus is not yet usable on cperl" && exit 1 + # --dev is already part of CPANM_OPT LASTCMD="$TIMEOUT_CMD cpanm $@" $LASTCMD 2>&1 || return 1 @@ -277,59 +289,5 @@ purge_sitelib() { ! $sitedirs->{$_} and ( -d _ ? rmdir : unlink ) } }, keys %$sitedirs ) ' - else - - cl_fn="/tmp/${TRAVIS_BUILD_ID}_Module_CoreList.pm"; - - [[ -s "$cl_fn" ]] || run_or_err \ - "Downloading latest Module::CoreList" \ - "curl -s --compress -o '$cl_fn' https://api.metacpan.org/source/Module::CoreList" - - perl -0777 -Ilib -MDBIx::Class::Optional::Dependencies -e ' - - # this is horrible, but really all we want is "has this ever been used" - # so a grep without a load is quite legit (and horrible) - my $mcl_source = <>; - - my %reqs_for_group = %{DBIx::Class::Optional::Dependencies->req_group_list}; - - my @all_possible_never_been_core_modpaths = map - { (my $mp = $_ . ".pm" ) =~ s|::|/|g; $mp } - grep - { $mcl_source !~ / ^ \s+ \x27 $_ \x27 \s* \=\> /mx } - ( - qw( - Module::Build::Tiny - ), - (map - { keys %{$reqs_for_group{$_}} } - grep - { !/^rdbms_|^dist_/ } - keys %reqs_for_group - ), - ) - ; - - # now that we have the list we can go ahead and destroy every single one - # of these modules without being concerned about breaking the base ability - # to install things - for my $mp ( sort { lc($a) cmp lc($b) } @all_possible_never_been_core_modpaths ) { - for my $incdir (@INC) { - -e "$incdir/$mp" - and - unlink "$incdir/$mp" - and - print "Nuking $incdir/$mp\n" - } - } - ' "$cl_fn" - fi } - - -CPAN_is_sane() { perl -MCPAN\ 1.94_56 -e 1 &>/dev/null ; } - -CPAN_supports_BUILDPL() { perl -MCPAN\ 1.9205 -e1 &>/dev/null; } - -have_sudo() { sudo /bin/true &>/dev/null ; } diff --git a/maint/travis-ci_scripts/lib/TAP/Harness/IgnoreNonessentialDzilAutogeneratedTests.pm b/maint/travis-ci_scripts/lib/TAP/Harness/IgnoreNonessentialDzilAutogeneratedTests.pm deleted file mode 100644 index 7f01716..0000000 --- a/maint/travis-ci_scripts/lib/TAP/Harness/IgnoreNonessentialDzilAutogeneratedTests.pm +++ /dev/null @@ -1,93 +0,0 @@ -package TAP::Harness::IgnoreNonessentialDzilAutogeneratedTests; - -use warnings; -use strict; - -use base 'TAP::Harness'; -use File::Spec (); -use IPC::Open3 'open3'; -use File::Temp (); -use List::Util 'first'; - -my $frivolous_test_map = { -# Test based on the extremely dep-heavy, *prone to failures* Test::CheckDeps -# - qr|^t/00-check-deps.t$| => [ - qr|^\Q# this test was generated with Dist::Zilla::Plugin::Test::CheckDeps|m, - - # older non-annotated versions - qr|use \s+ Test::CheckDeps .*? ^\Qcheck_dependencies('suggests')\E .*? \QBAIL_OUT("Missing dependencies") if !Test::More->builder->is_passing|smx, - ], - -# "does everything compile" tests are useless by definition - this is what the -# rest of the test suite is for -# - qr|^t/00-compile.t$| => [ - qr|^\Q# this test was generated with Dist::Zilla::Plugin::Test::Compile|m, - ], - -# The report prereq test managed to become fatal as well -# - qr|^t/00-report-prereqs.t$| => [ - qr|^\Q# This test was generated by Dist::Zilla::Plugin::Test::ReportPrereqs|m, - ], - -# Just future-proof the thing, catch anything autogened by dzil for a bit - qr|^t/00-| => [ - qr|^\Q# This test was generated by Dist::Zilla::|m, - ] -}; - -sub aggregate_tests { - my ($self, $aggregate, @all_tests) = @_; - - my ($run_tests, $skip_tests); - - TESTFILE: - for (@all_tests) { - my $fn = File::Spec::Unix->catpath( File::Spec->splitpath( $_ ) ); - - if (my $REs = $frivolous_test_map->{ - (first { $fn =~ $_ } keys %$frivolous_test_map ) || '' - }) { - my $slurptest = do { local (@ARGV, $/) = $fn; <> }; - $slurptest =~ $_ and push @$skip_tests, $fn and next TESTFILE for @$REs; - } - - push @$run_tests, $fn; - } - - if ($skip_tests) { - - for my $tfn (@$skip_tests) { - - (my $tfn_flattened = $tfn) =~ s|/|_|g; - - my $log_file = File::Temp->new( - DIR => '/tmp', - TEMPLATE => "AutoGenTest_${tfn_flattened}_XXXXX", - SUFFIX => '.txt', - ); - - # FIXME I have no idea why the fileno dance is necessary - will investigate later - # All I know is that if I pass in just $log_file - open3 ignores it >:( - my $pid = open3(undef, '>&'.fileno($log_file), undef, $^X, qw(-I blib -I arch/lib), $tfn ); - waitpid ($pid, 0); - my $ex = $?; - - if ($ex) { - # use qx as opposed to another open3 until I figure out the above - close $log_file or die "Unable to close $log_file: $!"; - chomp( my $url = `/usr/bin/nopaste -q -s Shadowcat -d $log_file < $log_file` ); - - $tfn .= "[would NOT have passed: $ex / $url]"; - } - } - - print STDERR "=== Skipping nonessential autogenerated tests: @$skip_tests\n"; - } - - return $self->SUPER::aggregate_tests($aggregate, @$run_tests); -} - -1; diff --git a/t/storage/quote_names.t b/t/storage/quote_names.t index e667ad3..ac65fa0 100644 --- a/t/storage/quote_names.t +++ b/t/storage/quote_names.t @@ -132,7 +132,7 @@ for my $db (sort { # if something was produced - it better be quoted if ( # the SQLT producer has no idea what quotes are :/ - $db ne 'SYBASE' + ! grep { $db eq $_ } qw( SYBASE DB2 ) and my $ddl = try { $schema->deployment_statements } ) {