[dbsrgits/DBIx-Class.git] / maint / travis-ci_scripts / 10_before_install.bash

#!/bin/bash

export SHORT_CIRCUIT_SMOKE

if have_sudo ; then

  # Stop pre-started RDBMS, move their data back to disk (save RAM)
  # sync for some settle time (not available on all platforms)
  for d in mysql postgresql ; do
    # maybe not even running
    run_or_err "Stopping $d" "sudo /etc/init.d/$d stop || /bin/true"

    # no longer available on newer build systems
    if [[ -d /var/ramfs/$d ]] ; then
      sudo rm -rf /var/lib/$d
      sudo mv /var/ramfs/$d /var/lib/
      sudo ln -s /var/lib/$d /var/ramfs/$d
    fi
  done
  /bin/sync
fi

# Sanity check VM before continuing
echo "
=============================================================================

= Startup Meminfo
$(free -m -t)

============================================================================="

CI_VM_MIN_FREE_MB=2000
if [[ "$(free -m | grep 'buffers/cache:' | perl -p -e '$_ = (split /\s+/, $_)[3]')" -lt "$CI_VM_MIN_FREE_MB" ]]; then
  SHORT_CIRCUIT_SMOKE=1
  echo_err "
=============================================================================

CI virtual machine stuck in a state with a lot of memory locked for no reason.
Under Travis this state usually results in a failed build.
Short-circuiting buildjob to avoid false negatives, please restart it manually.

============================================================================="

# pull requests are always scrutinized after the fact anyway - run a
# a simpler matrix
elif [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
  if [[ -n "$BREWVER" ]]; then
    # just don't brew anything
    SHORT_CIRCUIT_SMOKE=1
  else
    # running PRs with 1 thread is non-sensical
    VCPU_USE=""
  fi
fi

if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi

# Previously we were going off the OpenVZ vcpu count and dividing by 3
# With the new infrastructure, somply go with "something high"
export VCPU_AVAILABLE=10

if [[ -z "$VCPU_USE" ]] ; then
  export VCPU_USE="$VCPU_AVAILABLE"
fi


if [[ "$CLEANTEST" != "true" ]]; then

  if [[ -z "$(tail -n +2 /proc/swaps)" ]] ; then
    run_or_err "Configuring swap (for Oracle)" \
      "sudo bash -c 'dd if=/dev/zero of=/swap.img bs=256M count=5 && chmod 600 /swap.img && mkswap /swap.img && swapon /swap.img'"
  fi

  export CACHE_DIR="/tmp/poormanscache"

  #
  # FIXME these debconf lines should automate the firebird config but do not :(((
  sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/enabled\tboolean\ttrue" | debconf-set-selections'
  sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/sysdba_password/new_password\tpassword\t123" | debconf-set-selections'

  # these APT sources do not mean anything to us anyway
  sudo rm -rf /etc/apt/sources.list.d/*

  # never installed, this looks like trusty
  if [[ ! -d /var/lib/mysql ]] ; then
    sudo dpkg --add-architecture i386
    extra_debs="postgresql mysql-server"
  fi

  run_or_err "Updating APT sources" "sudo apt-get update"

  apt_install $extra_debs libmysqlclient-dev memcached firebird2.5-super firebird2.5-dev unixodbc-dev expect

  # need to stop them again
  if [[ -n "$extra_debs" ]] ; then
    for d in mysql postgresql ; do
      sudo /etc/init.d/$d stop || /bin/true
    done
  fi

  run_or_err "Cloning poor man's cache from github" "git clone --depth=1 --single-branch --branch=oracle/10.2.0 https://github.com/poormanscache/poormanscache.git $CACHE_DIR && $CACHE_DIR/reassemble"
  run_or_err "Installing OracleXE manually from deb" \
    "sudo dpkg -i $CACHE_DIR/apt_cache/bc-multiarch-travis_1.0_all.deb $CACHE_DIR/apt_cache/oracle-xe_10.2.0.1-1.1_i386.deb || sudo bash -c 'source maint/travis-ci_scripts/common.bash && apt_install -f'"

### config memcached
  run_or_err "Starting memcached" "sudo /etc/init.d/memcached start"
  export DBICTEST_MEMCACHED=127.0.0.1:11211

### config mysql
  run_or_err "Installing minimizing MySQL config" "\
     sudo bash -c 'rm /var/lib/mysql/ib*' \
  && sudo cp maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf /etc/mysql/conf.d/ \
  && sudo chmod 644 /etc/mysql/conf.d/*.cnf \
  "

  run_or_err "Starting MySQL" "sudo /etc/init.d/mysql start"
  run_or_err "Creating MySQL TestDB" "mysql -u root -e 'create database dbic_test;'"
  export DBICTEST_MYSQL_DSN='dbi:mysql:database=dbic_test;host=127.0.0.1'
  export DBICTEST_MYSQL_USER=root

### config pg
  run_or_err "Starting PostgreSQL" "sudo /etc/init.d/postgresql start"
  run_or_err "Creating PostgreSQL TestDB" "psql -c 'create database dbic_test;' -U postgres"
  export DBICTEST_PG_DSN='dbi:Pg:database=dbic_test;host=127.0.0.1'
  export DBICTEST_PG_USER=postgres

### conig firebird
  # poor man's deb config
  EXPECT_FB_SCRIPT='
    spawn dpkg-reconfigure --frontend=text firebird2.5-super
    expect "Enable Firebird server?"
    send "\177\177\177\177yes\r"
    expect "Password for SYSDBA"
    send "123\r"
    sleep 1
    expect eof
  '
  # creating testdb
  # FIXME - this step still fails from time to time >:(((
  # has to do with the FB reconfiguration I suppose
  # for now if it fails twice - simply skip FB testing
  for i in 1 2 ; do

    run_or_err "Re-configuring Firebird" "
      sync
      DEBIAN_FRONTEND=text sudo expect -c '$EXPECT_FB_SCRIPT'
      sleep 1
      sync
      # restart the server for good measure
      sudo /etc/init.d/firebird2.5-super stop || true
      sleep 1
      sync
      sudo /etc/init.d/firebird2.5-super start
      sleep 1
      sync
    "

    if run_or_err "Creating Firebird TestDB" \
      "echo \"CREATE DATABASE '/var/lib/firebird/2.5/data/dbic_test.fdb';\" | sudo isql-fb -u sysdba -p 123"
    then

      run_or_err "Fetching and building Firebird ODBC driver" '
        cd "$(mktemp -d)"
        wget -qO- http://sourceforge.net/projects/firebird/files/firebird-ODBC-driver/2.0.2-Release/OdbcFb-Source-2.0.2.153.gz/download | tar -zx
        cd Builds/Gcc.lin
        perl -p -i -e "s|/usr/lib64|/usr/lib/x86_64-linux-gnu|g" ../makefile.environ
        make -f makefile.linux
        sudo make -f makefile.linux install
      '

      sudo bash -c 'cat >> /etc/odbcinst.ini' <<< "
[Firebird]
Description     = InterBase/Firebird ODBC Driver
Driver          = /usr/lib/x86_64-linux-gnu/libOdbcFb.so
Setup           = /usr/lib/x86_64-linux-gnu/libOdbcFb.so
Threading       = 1
FileUsage       = 1
"

      export DBICTEST_FIREBIRD_DSN=dbi:Firebird:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb
      export DBICTEST_FIREBIRD_USER=SYSDBA
      export DBICTEST_FIREBIRD_PASS=123

      export DBICTEST_FIREBIRD_INTERBASE_DSN=dbi:InterBase:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb
      export DBICTEST_FIREBIRD_INTERBASE_USER=SYSDBA
      export DBICTEST_FIREBIRD_INTERBASE_PASS=123

      export DBICTEST_FIREBIRD_ODBC_DSN="dbi:ODBC:Driver=Firebird;Dbname=/var/lib/firebird/2.5/data/dbic_test.fdb"
      export DBICTEST_FIREBIRD_ODBC_USER=SYSDBA
      export DBICTEST_FIREBIRD_ODBC_PASS=123

      break
    fi

  done

### config oracle
  SRV_ORA_HOME=/usr/lib/oracle/xe/app/oracle/product/10.2.0/server

  # without this some of the more zealous tests can exhaust the amount
  # of listeners and oracle is too slow to spin extras up :(
  sudo bash -c "echo -e '\nprocesses=150' >> $SRV_ORA_HOME/config/scripts/init.ora"

  EXPECT_ORA_SCRIPT='
    spawn /etc/init.d/oracle-xe configure

    sleep 1
    set send_slow {1 .005}

    expect "Specify the HTTP port that will be used for Oracle Application Express"
    sleep 0.5
    send -s "8021\r"

    expect "Specify a port that will be used for the database listener"
    sleep 0.5
    send -s "1521\r"

    expect "Specify a password to be used for database accounts"
    sleep 0.5
    send -s "adminpass\r"

    expect "Confirm the password"
    sleep 0.5
    send -s "adminpass\r"

    expect "Do you want Oracle Database 10g Express Edition to be started on boot"
    sleep 0.5
    send -s "n\r"

    sleep 0.5
    expect "Configuring Database"

    sleep 1
    expect eof
    wait
  '

  # if we do not redirect to some random file, but instead try to capture
  # into a var the way run_or_err does - everything hangs
  # FIXME: I couldn't figure it out after 3 hours of headdesking,
  # would be nice to know the reason eventually
  run_or_err "Configuring OracleXE" "sudo $(which expect) -c '$EXPECT_ORA_SCRIPT' &>/tmp/ora_configure_10.2.log"

  export DBICTEST_ORA_DSN=dbi:Oracle://localhost:1521/XE
  export DBICTEST_ORA_USER=dbic_test
  export DBICTEST_ORA_PASS=abc123456
  export DBICTEST_ORA_EXTRAUSER_DSN="$DBICTEST_ORA_DSN"
  export DBICTEST_ORA_EXTRAUSER_USER=dbic_test_extra
  export DBICTEST_ORA_EXTRAUSER_PASS=abc123456

  run_or_err "Create Oracle users" "ORACLE_SID=XE ORACLE_HOME=$SRV_ORA_HOME $SRV_ORA_HOME/bin/sqlplus -L -S system/adminpass @/dev/stdin <<< '
    CREATE USER $DBICTEST_ORA_USER IDENTIFIED BY $DBICTEST_ORA_PASS;
    GRANT connect,resource TO $DBICTEST_ORA_USER;
    CREATE USER $DBICTEST_ORA_EXTRAUSER_USER IDENTIFIED BY $DBICTEST_ORA_EXTRAUSER_PASS;
    GRANT connect,resource TO $DBICTEST_ORA_EXTRAUSER_USER;
  '"

  export ORACLE_HOME="$CACHE_DIR/ora_instaclient/x86-64/oracle_instaclient_10.2.0.5.0"
fi
Commit	Line	Data
b58ecb01	1	#!/bin/bash
b58ecb01	2
01fd95b6	3	export SHORT_CIRCUIT_SMOKE
01fd95b6	4
36372426	5	if have_sudo ; then
	6
	7	# Stop pre-started RDBMS, move their data back to disk (save RAM)
	8	# sync for some settle time (not available on all platforms)
	9	for d in mysql postgresql ; do
	10	# maybe not even running
	11	run_or_err "Stopping $d" "sudo /etc/init.d/$d stop \|\| /bin/true"
	12
	13	# no longer available on newer build systems
	14	if [[ -d /var/ramfs/$d ]] ; then
	15	sudo rm -rf /var/lib/$d
	16	sudo mv /var/ramfs/$d /var/lib/
	17	sudo ln -s /var/lib/$d /var/ramfs/$d
	18	fi
	19	done
	20	/bin/sync
	21	fi
06278456	22
fcdd56ca	23	# Sanity check VM before continuing
	24	echo "
	25	=============================================================================
	26
	27	= Startup Meminfo
	28	$(free -m -t)
	29
	30	============================================================================="
	31
	32	CI_VM_MIN_FREE_MB=2000
	33	if [[ "$(free -m \| grep 'buffers/cache:' \| perl -p -e '$_ = (split /\s+/, $_)[3]')" -lt "$CI_VM_MIN_FREE_MB" ]]; then
	34	SHORT_CIRCUIT_SMOKE=1
	35	echo_err "
	36	=============================================================================
	37
	38	CI virtual machine stuck in a state with a lot of memory locked for no reason.
	39	Under Travis this state usually results in a failed build.
	40	Short-circuiting buildjob to avoid false negatives, please restart it manually.
	41
	42	============================================================================="
01fd95b6	43
	44	# pull requests are always scrutinized after the fact anyway - run a
	45	# a simpler matrix
	46	elif [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then
	47	if [[ -n "$BREWVER" ]]; then
	48	# just don't brew anything
	49	SHORT_CIRCUIT_SMOKE=1
	50	else
	51	# running PRs with 1 thread is non-sensical
	52	VCPU_USE=""
	53	fi
fcdd56ca	54	fi
fcdd56ca	55
b58ecb01	56	if [[ -n "$SHORT_CIRCUIT_SMOKE" ]] ; then return ; fi
b58ecb01	57
36372426	58	# Previously we were going off the OpenVZ vcpu count and dividing by 3
	59	# With the new infrastructure, somply go with "something high"
	60	export VCPU_AVAILABLE=10
5ac4a96d	61
	62	if [[ -z "$VCPU_USE" ]] ; then
	63	export VCPU_USE="$VCPU_AVAILABLE"
2a89732d	64	fi
003e97c5	65
003e97c5	66
36372426	67	if [[ "$CLEANTEST" != "true" ]]; then
b58ecb01	68
36372426	69	if [[ -z "$(tail -n +2 /proc/swaps)" ]] ; then
	70	run_or_err "Configuring swap (for Oracle)" \
	71	"sudo bash -c 'dd if=/dev/zero of=/swap.img bs=256M count=5 && chmod 600 /swap.img && mkswap /swap.img && swapon /swap.img'"
	72	fi
e861e4f0	73
36372426	74	export CACHE_DIR="/tmp/poormanscache"
1e735709	75
b58ecb01	76	#
	77	# FIXME these debconf lines should automate the firebird config but do not :(((
	78	sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/enabled\tboolean\ttrue" \| debconf-set-selections'
	79	sudo bash -c 'echo -e "firebird2.5-super\tshared/firebird/sysdba_password/new_password\tpassword\t123" \| debconf-set-selections'
	80
d50f2fe1	81	# these APT sources do not mean anything to us anyway
	82	sudo rm -rf /etc/apt/sources.list.d/*
	83
36372426	84	# never installed, this looks like trusty
	85	if [[ ! -d /var/lib/mysql ]] ; then
	86	sudo dpkg --add-architecture i386
	87	extra_debs="postgresql mysql-server"
	88	fi
	89
94464be8	90	run_or_err "Updating APT sources" "sudo apt-get update"
003e97c5	91
36372426	92	apt_install $extra_debs libmysqlclient-dev memcached firebird2.5-super firebird2.5-dev unixodbc-dev expect
	93
	94	# need to stop them again
	95	if [[ -n "$extra_debs" ]] ; then
	96	for d in mysql postgresql ; do
	97	sudo /etc/init.d/$d stop \|\| /bin/true
	98	done
	99	fi
	100
	101	run_or_err "Cloning poor man's cache from github" "git clone --depth=1 --single-branch --branch=oracle/10.2.0 https://github.com/poormanscache/poormanscache.git $CACHE_DIR && $CACHE_DIR/reassemble"
	102	run_or_err "Installing OracleXE manually from deb" \
	103	"sudo dpkg -i $CACHE_DIR/apt_cache/bc-multiarch-travis_1.0_all.deb $CACHE_DIR/apt_cache/oracle-xe_10.2.0.1-1.1_i386.deb \|\| sudo bash -c 'source maint/travis-ci_scripts/common.bash && apt_install -f'"
b58ecb01	104
b58ecb01	105	### config memcached
106c81a5	106	run_or_err "Starting memcached" "sudo /etc/init.d/memcached start"
b58ecb01	107	export DBICTEST_MEMCACHED=127.0.0.1:11211
	108
	109	### config mysql
3c26d329	110	run_or_err "Installing minimizing MySQL config" "\
	111	sudo bash -c 'rm /var/lib/mysql/ib*' \
	112	&& sudo cp maint/travis-ci_scripts/configs/minimal_mysql_travis.cnf /etc/mysql/conf.d/ \
	113	&& sudo chmod 644 /etc/mysql/conf.d/*.cnf \
	114	"
	115
fcdd56ca	116	run_or_err "Starting MySQL" "sudo /etc/init.d/mysql start"
36372426	117	run_or_err "Creating MySQL TestDB" "mysql -u root -e 'create database dbic_test;'"
b58ecb01	118	export DBICTEST_MYSQL_DSN='dbi:mysql:database=dbic_test;host=127.0.0.1'
	119	export DBICTEST_MYSQL_USER=root
	120
	121	### config pg
fcdd56ca	122	run_or_err "Starting PostgreSQL" "sudo /etc/init.d/postgresql start"
b58ecb01	123	run_or_err "Creating PostgreSQL TestDB" "psql -c 'create database dbic_test;' -U postgres"
	124	export DBICTEST_PG_DSN='dbi:Pg:database=dbic_test;host=127.0.0.1'
	125	export DBICTEST_PG_USER=postgres
	126
	127	### conig firebird
	128	# poor man's deb config
	129	EXPECT_FB_SCRIPT='
	130	spawn dpkg-reconfigure --frontend=text firebird2.5-super
	131	expect "Enable Firebird server?"
	132	send "\177\177\177\177yes\r"
	133	expect "Password for SYSDBA"
	134	send "123\r"
	135	sleep 1
003e97c5	136	expect eof
b58ecb01	137	'
b58ecb01	138	# creating testdb
	139	# FIXME - this step still fails from time to time >:(((
	140	# has to do with the FB reconfiguration I suppose
906c0355	141	# for now if it fails twice - simply skip FB testing
	142	for i in 1 2 ; do
	143
	144	run_or_err "Re-configuring Firebird" "
	145	sync
	146	DEBIAN_FRONTEND=text sudo expect -c '$EXPECT_FB_SCRIPT'
	147	sleep 1
	148	sync
	149	# restart the server for good measure
	150	sudo /etc/init.d/firebird2.5-super stop \|\| true
	151	sleep 1
	152	sync
	153	sudo /etc/init.d/firebird2.5-super start
	154	sleep 1
	155	sync
	156	"
	157
	158	if run_or_err "Creating Firebird TestDB" \
	159	"echo \"CREATE DATABASE '/var/lib/firebird/2.5/data/dbic_test.fdb';\" \| sudo isql-fb -u sysdba -p 123"
	160	then
da59679a	161
137dca08	162	run_or_err "Fetching and building Firebird ODBC driver" '
	163	cd "$(mktemp -d)"
	164	wget -qO- http://sourceforge.net/projects/firebird/files/firebird-ODBC-driver/2.0.2-Release/OdbcFb-Source-2.0.2.153.gz/download \| tar -zx
	165	cd Builds/Gcc.lin
	166	perl -p -i -e "s\|/usr/lib64\|/usr/lib/x86_64-linux-gnu\|g" ../makefile.environ
	167	make -f makefile.linux
	168	sudo make -f makefile.linux install
	169	'
	170
	171	sudo bash -c 'cat >> /etc/odbcinst.ini' <<< "
	172	[Firebird]
	173	Description = InterBase/Firebird ODBC Driver
	174	Driver = /usr/lib/x86_64-linux-gnu/libOdbcFb.so
	175	Setup = /usr/lib/x86_64-linux-gnu/libOdbcFb.so
	176	Threading = 1
	177	FileUsage = 1
	178	"
	179
906c0355	180	export DBICTEST_FIREBIRD_DSN=dbi:Firebird:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb
	181	export DBICTEST_FIREBIRD_USER=SYSDBA
	182	export DBICTEST_FIREBIRD_PASS=123
	183
	184	export DBICTEST_FIREBIRD_INTERBASE_DSN=dbi:InterBase:dbname=/var/lib/firebird/2.5/data/dbic_test.fdb
	185	export DBICTEST_FIREBIRD_INTERBASE_USER=SYSDBA
	186	export DBICTEST_FIREBIRD_INTERBASE_PASS=123
	187
137dca08	188	export DBICTEST_FIREBIRD_ODBC_DSN="dbi:ODBC:Driver=Firebird;Dbname=/var/lib/firebird/2.5/data/dbic_test.fdb"
	189	export DBICTEST_FIREBIRD_ODBC_USER=SYSDBA
	190	export DBICTEST_FIREBIRD_ODBC_PASS=123
	191
906c0355	192	break
	193	fi
	194
	195	done
b58ecb01	196
003e97c5	197	### config oracle
fa7907b0	198	SRV_ORA_HOME=/usr/lib/oracle/xe/app/oracle/product/10.2.0/server
	199
	200	# without this some of the more zealous tests can exhaust the amount
	201	# of listeners and oracle is too slow to spin extras up :(
	202	sudo bash -c "echo -e '\nprocesses=150' >> $SRV_ORA_HOME/config/scripts/init.ora"
	203
003e97c5	204	EXPECT_ORA_SCRIPT='
	205	spawn /etc/init.d/oracle-xe configure
	206
	207	sleep 1
	208	set send_slow {1 .005}
	209
	210	expect "Specify the HTTP port that will be used for Oracle Application Express"
	211	sleep 0.5
	212	send -s "8021\r"
	213
	214	expect "Specify a port that will be used for the database listener"
	215	sleep 0.5
	216	send -s "1521\r"
	217
	218	expect "Specify a password to be used for database accounts"
	219	sleep 0.5
	220	send -s "adminpass\r"
	221
	222	expect "Confirm the password"
	223	sleep 0.5
	224	send -s "adminpass\r"
	225
	226	expect "Do you want Oracle Database 10g Express Edition to be started on boot"
	227	sleep 0.5
	228	send -s "n\r"
	229
	230	sleep 0.5
	231	expect "Configuring Database"
	232
	233	sleep 1
	234	expect eof
	235	wait
	236	'
	237
	238	# if we do not redirect to some random file, but instead try to capture
	239	# into a var the way run_or_err does - everything hangs
	240	# FIXME: I couldn't figure it out after 3 hours of headdesking,
	241	# would be nice to know the reason eventually
	242	run_or_err "Configuring OracleXE" "sudo $(which expect) -c '$EXPECT_ORA_SCRIPT' &>/tmp/ora_configure_10.2.log"
003e97c5	243
	244	export DBICTEST_ORA_DSN=dbi:Oracle://localhost:1521/XE
	245	export DBICTEST_ORA_USER=dbic_test
	246	export DBICTEST_ORA_PASS=abc123456
fa7907b0	247	export DBICTEST_ORA_EXTRAUSER_DSN="$DBICTEST_ORA_DSN"
003e97c5	248	export DBICTEST_ORA_EXTRAUSER_USER=dbic_test_extra
	249	export DBICTEST_ORA_EXTRAUSER_PASS=abc123456
	250
	251	run_or_err "Create Oracle users" "ORACLE_SID=XE ORACLE_HOME=$SRV_ORA_HOME $SRV_ORA_HOME/bin/sqlplus -L -S system/adminpass @/dev/stdin <<< '
	252	CREATE USER $DBICTEST_ORA_USER IDENTIFIED BY $DBICTEST_ORA_PASS;
	253	GRANT connect,resource TO $DBICTEST_ORA_USER;
	254	CREATE USER $DBICTEST_ORA_EXTRAUSER_USER IDENTIFIED BY $DBICTEST_ORA_EXTRAUSER_PASS;
	255	GRANT connect,resource TO $DBICTEST_ORA_EXTRAUSER_USER;
	256	'"
	257
	258	export ORACLE_HOME="$CACHE_DIR/ora_instaclient/x86-64/oracle_instaclient_10.2.0.5.0"
b58ecb01	259	fi