[dbsrgits/DBIx-Class.git] / t / 93storage_replication.t

use strict;
use warnings;
use lib qw(t/lib);
use Test::More;
use DBICTest;

BEGIN {
    eval "use Moose; use Test::Moose";
    plan $@
        ? ( skip_all => 'needs Moose for testing' )
        : ( tests => 41 );
}

use_ok 'DBIx::Class::Storage::DBI::Replicated::Pool';
use_ok 'DBIx::Class::Storage::DBI::Replicated::Balancer';
use_ok 'DBIx::Class::Storage::DBI::Replicated::Replicant';
use_ok 'DBIx::Class::Storage::DBI::Replicated';

=head1 HOW TO USE

    This is a test of the replicated storage system.  This will work in one of
    two ways, either it was try to fake replication with a couple of SQLite DBs
    and creative use of copy, or if you define a couple of %ENV vars correctly
    will try to test those.  If you do that, it will assume the setup is properly
    replicating.  Your results may vary, but I have demonstrated this to work with
    mysql native replication.
    
=cut


## ----------------------------------------------------------------------------
## Build a class to hold all our required testing data and methods.
## ----------------------------------------------------------------------------

TESTSCHEMACLASSES: {

    ## --------------------------------------------------------------------- ##
    ## Create an object to contain your replicated stuff.
    ## --------------------------------------------------------------------- ##
    
    package DBIx::Class::DBI::Replicated::TestReplication;
   
    use DBICTest;
    use base qw/Class::Accessor::Fast/;
    
    __PACKAGE__->mk_accessors( qw/schema/ );

    ## Initialize the object
    
	sub new {
	    my $class = shift @_;
	    my $self = $class->SUPER::new(@_);
	
	    $self->schema( $self->init_schema );
	    return $self;
	}
    
    ## Get the Schema and set the replication storage type
    
    sub init_schema {
        my $class = shift @_;
        
        my $schema = DBICTest->init_schema(
            storage_type=>[
            	'::DBI::Replicated' => {
            		balancer_type=>'::Random',
            	}
            ],
            deploy_args=>{
                   add_drop_table => 1,
            },
        );

        return $schema;
    }
    
    sub generate_replicant_connect_info {}
    sub replicate {}
    sub cleanup {}

  
    ## --------------------------------------------------------------------- ##
    ## Subclass for when you are using SQLite for testing, this provides a fake
    ## replication support.
    ## --------------------------------------------------------------------- ##
        
    package DBIx::Class::DBI::Replicated::TestReplication::SQLite;

    use DBICTest;
    use File::Copy;    
    use base 'DBIx::Class::DBI::Replicated::TestReplication';
    
    __PACKAGE__->mk_accessors( qw/master_path slave_paths/ );
    
    ## Set the mastep path from DBICTest
    
	sub new {
	    my $class = shift @_;
	    my $self = $class->SUPER::new(@_);
	
	    $self->master_path( DBICTest->_sqlite_dbfilename );
	    $self->slave_paths([
            "t/var/DBIxClass_slave1.db",
            "t/var/DBIxClass_slave2.db",    
        ]);
        
	    return $self;
	}    
	
    ## Return an Array of ArrayRefs where each ArrayRef is suitable to use for
    ## $storage->connect_info to be used for connecting replicants.
    
    sub generate_replicant_connect_info {
        my $self = shift @_;
        my @dsn = map {
            "dbi:SQLite:${_}";
        } @{$self->slave_paths};
        
        return map { [$_,'','',{AutoCommit=>1}] } @dsn;
    }
    
    ## Do a 'good enough' replication by copying the master dbfile over each of
    ## the slave dbfiles.  If the master is SQLite we do this, otherwise we
    ## just do a one second pause to let the slaves catch up.
    
    sub replicate {
        my $self = shift @_;
        foreach my $slave (@{$self->slave_paths}) {
            copy($self->master_path, $slave);
        }
    }
    
    ## Cleanup after ourselves.  Unlink all gthe slave paths.
    
    sub cleanup {
        my $self = shift @_;
        foreach my $slave (@{$self->slave_paths}) {
            unlink $slave;
        }     
    }
    
    ## --------------------------------------------------------------------- ##
    ## Subclass for when you are setting the databases via custom export vars
    ## This is for when you have a replicating database setup that you are
    ## going to test against.  You'll need to define the correct $ENV and have
    ## two slave databases to test against, as well as a replication system
    ## that will replicate in less than 1 second.
    ## --------------------------------------------------------------------- ##
        
    package DBIx::Class::DBI::Replicated::TestReplication::Custom; 
    use base 'DBIx::Class::DBI::Replicated::TestReplication';
    
    ## Return an Array of ArrayRefs where each ArrayRef is suitable to use for
    ## $storage->connect_info to be used for connecting replicants.
    
    sub generate_replicant_connect_info { 
        return (
            [$ENV{"DBICTEST_SLAVE0_DSN"}, $ENV{"DBICTEST_SLAVE0_DBUSER"}, $ENV{"DBICTEST_SLAVE0_DBPASS"}, {AutoCommit => 1}],
            [$ENV{"DBICTEST_SLAVE1_DSN"}, $ENV{"DBICTEST_SLAVE1_DBUSER"}, $ENV{"DBICTEST_SLAVE1_DBPASS"}, {AutoCommit => 1}],           
        );
    }
    
    ## pause a bit to let the replication catch up 
    
    sub replicate {
    	sleep 1;
    } 
}

## ----------------------------------------------------------------------------
## Create an object and run some tests
## ----------------------------------------------------------------------------

## Thi first bunch of tests are basic, just make sure all the bits are behaving

my $replicated_class = DBICTest->has_custom_dsn ?
    'DBIx::Class::DBI::Replicated::TestReplication::Custom' :
    'DBIx::Class::DBI::Replicated::TestReplication::SQLite';

ok my $replicated = $replicated_class->new
    => 'Created a replication object';
    
isa_ok $replicated->schema
    => 'DBIx::Class::Schema';
    
isa_ok $replicated->schema->storage
    => 'DBIx::Class::Storage::DBI::Replicated';

ok $replicated->schema->storage->meta
    => 'has a meta object';
    
isa_ok $replicated->schema->storage->master
    => 'DBIx::Class::Storage::DBI';
    
isa_ok $replicated->schema->storage->pool
    => 'DBIx::Class::Storage::DBI::Replicated::Pool';
    
isa_ok $replicated->schema->storage->balancer
    => 'DBIx::Class::Storage::DBI::Replicated::Balancer'; 

ok my @replicant_connects = $replicated->generate_replicant_connect_info
    => 'got replication connect information';

ok my @replicated_storages = $replicated->schema->storage->connect_replicants(@replicant_connects)
    => 'Created some storages suitable for replicants';
    
isa_ok $replicated->schema->storage->balancer->current_replicant
    => 'DBIx::Class::Storage::DBI';
    
ok $replicated->schema->storage->pool->has_replicants
    => 'does have replicants';     

is $replicated->schema->storage->num_replicants => 2
    => 'has two replicants';
       
does_ok $replicated_storages[0]
    => 'DBIx::Class::Storage::DBI::Replicated::Replicant';

does_ok $replicated_storages[1]
    => 'DBIx::Class::Storage::DBI::Replicated::Replicant';
    
my @replicant_names = keys %{$replicated->schema->storage->replicants};

does_ok $replicated->schema->storage->replicants->{$replicant_names[0]}
    => 'DBIx::Class::Storage::DBI::Replicated::Replicant';

does_ok $replicated->schema->storage->replicants->{$replicant_names[1]}
    => 'DBIx::Class::Storage::DBI::Replicated::Replicant';  

## Add some info to the database

$replicated
    ->schema
    ->populate('Artist', [
        [ qw/artistid name/ ],
        [ 4, "Ozric Tentacles"],
    ]);
                
## Make sure all the slaves have the table definitions

$replicated->replicate;

## Make sure we can read the data.

ok my $artist1 = $replicated->schema->resultset('Artist')->find(4)
    => 'Created Result';

isa_ok $artist1
    => 'DBICTest::Artist';
    
is $artist1->name, 'Ozric Tentacles'
    => 'Found expected name for first result';

## Add some new rows that only the master will have  This is because
## we overload any type of write operation so that is must hit the master
## database.

$replicated
    ->schema
    ->populate('Artist', [
        [ qw/artistid name/ ],
        [ 5, "Doom's Children"],
        [ 6, "Dead On Arrival"],
        [ 7, "Watergate"],
    ]);

SKIP: {
    ## We can't do this test if we have a custom replicants, since we assume
    ## if there are custom one that you are trying to test a real replicating
    ## system.  See docs above for more.
    
    skip 'Cannot test inconsistent replication since you have a real replication system', 1
     if DBICTest->has_custom_dsn && $ENV{"DBICTEST_SLAVE0_DSN"};
    
	## Alright, the database 'cluster' is not in a consistent state.  When we do
	## a read now we expect bad news    
    is $replicated->schema->resultset('Artist')->find(5), undef
    => 'read after disconnect fails because it uses a replicant which we have neglected to "replicate" yet'; 
}

## Make sure all the slaves have the table definitions
$replicated->replicate;

## Should find some data now

ok my $artist2 = $replicated->schema->resultset('Artist')->find(5)
    => 'Sync succeed';
    
isa_ok $artist2
    => 'DBICTest::Artist';
    
is $artist2->name, "Doom's Children"
    => 'Found expected name for first result';

## What happens when we disconnect all the replicants?

is $replicated->schema->storage->pool->connected_replicants => 2
    => "both replicants are connected";
    
$replicated->schema->storage->replicants->{$replicant_names[0]}->disconnect;
$replicated->schema->storage->replicants->{$replicant_names[1]}->disconnect;

is $replicated->schema->storage->pool->connected_replicants => 0
    => "both replicants are now disconnected";

## All these should pass, since the database should automatically reconnect

ok my $artist3 = $replicated->schema->resultset('Artist')->find(6)
    => 'Still finding stuff.';
    
isa_ok $artist3
    => 'DBICTest::Artist';
    
is $artist3->name, "Dead On Arrival"
    => 'Found expected name for first result';

is $replicated->schema->storage->pool->connected_replicants => 1
    => "One replicant reconnected to handle the job";
    
## What happens when we try to select something that doesn't exist?

ok ! $replicated->schema->resultset('Artist')->find(666)
    => 'Correctly failed to find something.';
    
## test the reliable option

TESTRELIABLE: {
	
	$replicated->schema->storage->set_reliable_storage;
	
	ok $replicated->schema->resultset('Artist')->find(2)
	    => 'Read from master 1';
	
	ok $replicated->schema->resultset('Artist')->find(5)
	    => 'Read from master 2';
	    
    $replicated->schema->storage->set_balanced_storage;	    
	    
	ok $replicated->schema->resultset('Artist')->find(3)
        => 'Read from replicant';
}

## Make sure when reliable goes out of scope, we are using replicants again

ok $replicated->schema->resultset('Artist')->find(1)
    => 'back to replicant 1.';
    
ok $replicated->schema->resultset('Artist')->find(2)
    => 'back to replicant 2.';

## set all the replicants to inactive, and make sure the balancer falls back to
## the master.

$replicated->schema->storage->replicants->{$replicant_names[0]}->active(0);
$replicated->schema->storage->replicants->{$replicant_names[1]}->active(0);
    
ok $replicated->schema->resultset('Artist')->find(2)
    => 'Fallback to master';

$replicated->schema->storage->replicants->{$replicant_names[0]}->active(1);
$replicated->schema->storage->replicants->{$replicant_names[1]}->active(1);

ok $replicated->schema->resultset('Artist')->find(2)
    => 'Returned to replicates';
    
## Getting slave status tests

use Data::Dump qw/dump/;
my $lag1 = $replicated->schema->storage->replicants->{$replicant_names[0]}->lag_behind_master;
warn dump $lag1;
       
## Delete the old database files
$replicated->cleanup;
Commit	Line	Data
e4dc89b3	1	use strict;
	2	use warnings;
	3	use lib qw(t/lib);
e4dc89b3	4	use Test::More;
857d66ca	5	use DBICTest;
8f7986d6	6
86583fa7	7	BEGIN {
de5dc9ef	8	eval "use Moose; use Test::Moose";
86583fa7	9	plan $@
2bf79155	10	? ( skip_all => 'needs Moose for testing' )
de5dc9ef	11	: ( tests => 41 );
26ab719a	12	}
	13
	14	use_ok 'DBIx::Class::Storage::DBI::Replicated::Pool';
	15	use_ok 'DBIx::Class::Storage::DBI::Replicated::Balancer';
	16	use_ok 'DBIx::Class::Storage::DBI::Replicated::Replicant';
	17	use_ok 'DBIx::Class::Storage::DBI::Replicated';
0f83441a	18
89cf6a70	19	=head1 HOW TO USE
	20
	21	This is a test of the replicated storage system. This will work in one of
	22	two ways, either it was try to fake replication with a couple of SQLite DBs
	23	and creative use of copy, or if you define a couple of %ENV vars correctly
	24	will try to test those. If you do that, it will assume the setup is properly
	25	replicating. Your results may vary, but I have demonstrated this to work with
	26	mysql native replication.
	27
	28	=cut
	29
	30
0f83441a	31	## ----------------------------------------------------------------------------
	32	## Build a class to hold all our required testing data and methods.
	33	## ----------------------------------------------------------------------------
	34
857d66ca	35	TESTSCHEMACLASSES: {
2bf79155	36
857d66ca	37	## --------------------------------------------------------------------- ##
	38	## Create an object to contain your replicated stuff.
	39	## --------------------------------------------------------------------- ##
	40
2bf79155	41	package DBIx::Class::DBI::Replicated::TestReplication;
	42
	43	use DBICTest;
	44	use base qw/Class::Accessor::Fast/;
	45
857d66ca	46	__PACKAGE__->mk_accessors( qw/schema/ );
2bf79155	47
	48	## Initialize the object
	49
	50	sub new {
26ab719a	51	my $class = shift @_;
26ab719a	52	my $self = $class->SUPER::new(@_);
2bf79155	53
2bf79155	54	$self->schema( $self->init_schema );
2bf79155	55	return $self;
	56	}
	57
26ab719a	58	## Get the Schema and set the replication storage type
2bf79155	59
	60	sub init_schema {
	61	my $class = shift @_;
89cf6a70	62
cb6ec758	63	my $schema = DBICTest->init_schema(
106d5f3b	64	storage_type=>[
	65	'::DBI::Replicated' => {
	66	balancer_type=>'::Random',
89cf6a70	67	}
	68	],
	69	deploy_args=>{
	70	add_drop_table => 1,
	71	},
	72	);
cb6ec758	73
2bf79155	74	return $schema;
2bf79155	75	}
26ab719a	76
857d66ca	77	sub generate_replicant_connect_info {}
	78	sub replicate {}
	79	sub cleanup {}
	80
	81
	82	## --------------------------------------------------------------------- ##
	83	## Subclass for when you are using SQLite for testing, this provides a fake
	84	## replication support.
	85	## --------------------------------------------------------------------- ##
	86
	87	package DBIx::Class::DBI::Replicated::TestReplication::SQLite;
	88
	89	use DBICTest;
	90	use File::Copy;
	91	use base 'DBIx::Class::DBI::Replicated::TestReplication';
	92
	93	__PACKAGE__->mk_accessors( qw/master_path slave_paths/ );
	94
	95	## Set the mastep path from DBICTest
	96
	97	sub new {
	98	my $class = shift @_;
	99	my $self = $class->SUPER::new(@_);
	100
	101	$self->master_path( DBICTest->_sqlite_dbfilename );
	102	$self->slave_paths([
	103	"t/var/DBIxClass_slave1.db",
	104	"t/var/DBIxClass_slave2.db",
	105	]);
	106
	107	return $self;
	108	}
	109
26ab719a	110	## Return an Array of ArrayRefs where each ArrayRef is suitable to use for
	111	## $storage->connect_info to be used for connecting replicants.
	112
	113	sub generate_replicant_connect_info {
857d66ca	114	my $self = shift @_;
26ab719a	115	my @dsn = map {
	116	"dbi:SQLite:${_}";
	117	} @{$self->slave_paths};
	118
857d66ca	119	return map { [$_,'','',{AutoCommit=>1}] } @dsn;
26ab719a	120	}
	121
	122	## Do a 'good enough' replication by copying the master dbfile over each of
50336325	123	## the slave dbfiles. If the master is SQLite we do this, otherwise we
50336325	124	## just do a one second pause to let the slaves catch up.
26ab719a	125
	126	sub replicate {
	127	my $self = shift @_;
	128	foreach my $slave (@{$self->slave_paths}) {
	129	copy($self->master_path, $slave);
	130	}
	131	}
	132
	133	## Cleanup after ourselves. Unlink all gthe slave paths.
	134
	135	sub cleanup {
	136	my $self = shift @_;
	137	foreach my $slave (@{$self->slave_paths}) {
	138	unlink $slave;
	139	}
	140	}
857d66ca	141
	142	## --------------------------------------------------------------------- ##
	143	## Subclass for when you are setting the databases via custom export vars
	144	## This is for when you have a replicating database setup that you are
	145	## going to test against. You'll need to define the correct $ENV and have
	146	## two slave databases to test against, as well as a replication system
	147	## that will replicate in less than 1 second.
	148	## --------------------------------------------------------------------- ##
	149
	150	package DBIx::Class::DBI::Replicated::TestReplication::Custom;
	151	use base 'DBIx::Class::DBI::Replicated::TestReplication';
	152
	153	## Return an Array of ArrayRefs where each ArrayRef is suitable to use for
	154	## $storage->connect_info to be used for connecting replicants.
	155
	156	sub generate_replicant_connect_info {
	157	return (
	158	[$ENV{"DBICTEST_SLAVE0_DSN"}, $ENV{"DBICTEST_SLAVE0_DBUSER"}, $ENV{"DBICTEST_SLAVE0_DBPASS"}, {AutoCommit => 1}],
	159	[$ENV{"DBICTEST_SLAVE1_DSN"}, $ENV{"DBICTEST_SLAVE1_DBUSER"}, $ENV{"DBICTEST_SLAVE1_DBPASS"}, {AutoCommit => 1}],
	160	);
	161	}
	162
	163	## pause a bit to let the replication catch up
	164
	165	sub replicate {
	166	sleep 1;
	167	}
2bf79155	168	}
	169
	170	## ----------------------------------------------------------------------------
	171	## Create an object and run some tests
	172	## ----------------------------------------------------------------------------
	173
26ab719a	174	## Thi first bunch of tests are basic, just make sure all the bits are behaving
2bf79155	175
857d66ca	176	my $replicated_class = DBICTest->has_custom_dsn ?
	177	'DBIx::Class::DBI::Replicated::TestReplication::Custom' :
	178	'DBIx::Class::DBI::Replicated::TestReplication::SQLite';
	179
	180	ok my $replicated = $replicated_class->new
	181	=> 'Created a replication object';
2bf79155	182
26ab719a	183	isa_ok $replicated->schema
2bf79155	184	=> 'DBIx::Class::Schema';
2bf79155	185
26ab719a	186	isa_ok $replicated->schema->storage
	187	=> 'DBIx::Class::Storage::DBI::Replicated';
	188
	189	ok $replicated->schema->storage->meta
	190	=> 'has a meta object';
	191
	192	isa_ok $replicated->schema->storage->master
	193	=> 'DBIx::Class::Storage::DBI';
	194
	195	isa_ok $replicated->schema->storage->pool
	196	=> 'DBIx::Class::Storage::DBI::Replicated::Pool';
	197
	198	isa_ok $replicated->schema->storage->balancer
	199	=> 'DBIx::Class::Storage::DBI::Replicated::Balancer';
	200
	201	ok my @replicant_connects = $replicated->generate_replicant_connect_info
	202	=> 'got replication connect information';
	203
955a6df6	204	ok my @replicated_storages = $replicated->schema->storage->connect_replicants(@replicant_connects)
26ab719a	205	=> 'Created some storages suitable for replicants';
26ab719a	206
cb6ec758	207	isa_ok $replicated->schema->storage->balancer->current_replicant
26ab719a	208	=> 'DBIx::Class::Storage::DBI';
	209
	210	ok $replicated->schema->storage->pool->has_replicants
	211	=> 'does have replicants';
	212
	213	is $replicated->schema->storage->num_replicants => 2
	214	=> 'has two replicants';
	215
de5dc9ef	216	does_ok $replicated_storages[0]
26ab719a	217	=> 'DBIx::Class::Storage::DBI::Replicated::Replicant';
26ab719a	218
de5dc9ef	219	does_ok $replicated_storages[1]
26ab719a	220	=> 'DBIx::Class::Storage::DBI::Replicated::Replicant';
26ab719a	221
89cf6a70	222	my @replicant_names = keys %{$replicated->schema->storage->replicants};
de5dc9ef	223
de5dc9ef	224	does_ok $replicated->schema->storage->replicants->{$replicant_names[0]}
26ab719a	225	=> 'DBIx::Class::Storage::DBI::Replicated::Replicant';
26ab719a	226
de5dc9ef	227	does_ok $replicated->schema->storage->replicants->{$replicant_names[1]}
26ab719a	228	=> 'DBIx::Class::Storage::DBI::Replicated::Replicant';
	229
	230	## Add some info to the database
	231
	232	$replicated
	233	->schema
	234	->populate('Artist', [
	235	[ qw/artistid name/ ],
	236	[ 4, "Ozric Tentacles"],
	237	]);
	238
	239	## Make sure all the slaves have the table definitions
	240
	241	$replicated->replicate;
	242
	243	## Make sure we can read the data.
	244
	245	ok my $artist1 = $replicated->schema->resultset('Artist')->find(4)
	246	=> 'Created Result';
	247
	248	isa_ok $artist1
	249	=> 'DBICTest::Artist';
	250
	251	is $artist1->name, 'Ozric Tentacles'
	252	=> 'Found expected name for first result';
	253
	254	## Add some new rows that only the master will have This is because
	255	## we overload any type of write operation so that is must hit the master
	256	## database.
	257
	258	$replicated
	259	->schema
	260	->populate('Artist', [
	261	[ qw/artistid name/ ],
	262	[ 5, "Doom's Children"],
	263	[ 6, "Dead On Arrival"],
	264	[ 7, "Watergate"],
	265	]);
	266
89cf6a70	267	SKIP: {
	268	## We can't do this test if we have a custom replicants, since we assume
	269	## if there are custom one that you are trying to test a real replicating
	270	## system. See docs above for more.
	271
	272	skip 'Cannot test inconsistent replication since you have a real replication system', 1
	273	if DBICTest->has_custom_dsn && $ENV{"DBICTEST_SLAVE0_DSN"};
	274
	275	## Alright, the database 'cluster' is not in a consistent state. When we do
	276	## a read now we expect bad news
	277	is $replicated->schema->resultset('Artist')->find(5), undef
	278	=> 'read after disconnect fails because it uses a replicant which we have neglected to "replicate" yet';
	279	}
26ab719a	280
	281	## Make sure all the slaves have the table definitions
	282	$replicated->replicate;
	283
	284	## Should find some data now
	285
	286	ok my $artist2 = $replicated->schema->resultset('Artist')->find(5)
	287	=> 'Sync succeed';
	288
	289	isa_ok $artist2
	290	=> 'DBICTest::Artist';
	291
	292	is $artist2->name, "Doom's Children"
	293	=> 'Found expected name for first result';
	294
	295	## What happens when we disconnect all the replicants?
	296
50336325	297	is $replicated->schema->storage->pool->connected_replicants => 2
	298	=> "both replicants are connected";
	299
89cf6a70	300	$replicated->schema->storage->replicants->{$replicant_names[0]}->disconnect;
89cf6a70	301	$replicated->schema->storage->replicants->{$replicant_names[1]}->disconnect;
26ab719a	302
50336325	303	is $replicated->schema->storage->pool->connected_replicants => 0
	304	=> "both replicants are now disconnected";
	305
	306	## All these should pass, since the database should automatically reconnect
	307
26ab719a	308	ok my $artist3 = $replicated->schema->resultset('Artist')->find(6)
26ab719a	309	=> 'Still finding stuff.';
2bf79155	310
26ab719a	311	isa_ok $artist3
26ab719a	312	=> 'DBICTest::Artist';
2bf79155	313
26ab719a	314	is $artist3->name, "Dead On Arrival"
26ab719a	315	=> 'Found expected name for first result';
2bf79155	316
50336325	317	is $replicated->schema->storage->pool->connected_replicants => 1
50336325	318	=> "One replicant reconnected to handle the job";
6f6fb437	319
	320	## What happens when we try to select something that doesn't exist?
	321
	322	ok ! $replicated->schema->resultset('Artist')->find(666)
	323	=> 'Correctly failed to find something.';
cb6ec758	324
	325	## test the reliable option
	326
	327	TESTRELIABLE: {
	328
	329	$replicated->schema->storage->set_reliable_storage;
	330
	331	ok $replicated->schema->resultset('Artist')->find(2)
	332	=> 'Read from master 1';
	333
	334	ok $replicated->schema->resultset('Artist')->find(5)
	335	=> 'Read from master 2';
	336
	337	$replicated->schema->storage->set_balanced_storage;
	338
	339	ok $replicated->schema->resultset('Artist')->find(3)
9c748388	340	=> 'Read from replicant';
cb6ec758	341	}
cb6ec758	342
9c748388	343	## Make sure when reliable goes out of scope, we are using replicants again
cb6ec758	344
	345	ok $replicated->schema->resultset('Artist')->find(1)
	346	=> 'back to replicant 1.';
	347
	348	ok $replicated->schema->resultset('Artist')->find(2)
	349	=> 'back to replicant 2.';
2156bbdd	350
106d5f3b	351	## set all the replicants to inactive, and make sure the balancer falls back to
	352	## the master.
	353
89cf6a70	354	$replicated->schema->storage->replicants->{$replicant_names[0]}->active(0);
89cf6a70	355	$replicated->schema->storage->replicants->{$replicant_names[1]}->active(0);
106d5f3b	356
106d5f3b	357	ok $replicated->schema->resultset('Artist')->find(2)
de5dc9ef	358	=> 'Fallback to master';
	359
	360	$replicated->schema->storage->replicants->{$replicant_names[0]}->active(1);
	361	$replicated->schema->storage->replicants->{$replicant_names[1]}->active(1);
	362
	363	ok $replicated->schema->resultset('Artist')->find(2)
	364	=> 'Returned to replicates';
	365
	366	## Getting slave status tests
	367
	368	use Data::Dump qw/dump/;
	369	my $lag1 = $replicated->schema->storage->replicants->{$replicant_names[0]}->lag_behind_master;
	370	warn dump $lag1;
106d5f3b	371
0f83441a	372	## Delete the old database files
50336325	373	$replicated->cleanup;
0f83441a	374
	375
	376
	377
	378
	379