t/93storage_replication.t

   1 use strict;
   2 use warnings;
   3 use lib qw(t/lib);
   4 use Test::More;
   5 use Test::Exception;
   6 use DBICTest;
   7 use List::Util 'first';
   8 use Scalar::Util 'reftype';
   9
  10 BEGIN {
  11     eval "use DBIx::Class::Storage::DBI::Replicated; use Test::Moose";
  12     plan $@
  13         ? ( skip_all => "Deps not installed: $@" )
  14         : ( tests => 89 );
  15 }
  16
  17 use_ok 'DBIx::Class::Storage::DBI::Replicated::Pool';
  18 use_ok 'DBIx::Class::Storage::DBI::Replicated::Balancer';
  19 use_ok 'DBIx::Class::Storage::DBI::Replicated::Replicant';
  20 use_ok 'DBIx::Class::Storage::DBI::Replicated';
  21
  22 =head1 HOW TO USE
  23
  24     This is a test of the replicated storage system.  This will work in one of
  25     two ways, either it was try to fake replication with a couple of SQLite DBs
  26     and creative use of copy, or if you define a couple of %ENV vars correctly
  27     will try to test those.  If you do that, it will assume the setup is properly
  28     replicating.  Your results may vary, but I have demonstrated this to work with
  29     mysql native replication.
  30
  31 =cut
  32
  33
  34 ## ----------------------------------------------------------------------------
  35 ## Build a class to hold all our required testing data and methods.
  36 ## ----------------------------------------------------------------------------
  37
  38 TESTSCHEMACLASSES: {
  39
  40     ## --------------------------------------------------------------------- ##
  41     ## Create an object to contain your replicated stuff.
  42     ## --------------------------------------------------------------------- ##
  43
  44     package DBIx::Class::DBI::Replicated::TestReplication;
  45
  46     use DBICTest;
  47     use base qw/Class::Accessor::Fast/;
  48
  49     __PACKAGE__->mk_accessors( qw/schema/ );
  50
  51     ## Initialize the object
  52
  53         sub new {
  54             my ($class, $schema_method) = (shift, shift);
  55             my $self = $class->SUPER::new(@_);
  56
  57             $self->schema( $self->init_schema($schema_method) );
  58             return $self;
  59         }
  60
  61     ## Get the Schema and set the replication storage type
  62
  63     sub init_schema {
  64         # current SQLT SQLite producer does not handle DROP TABLE IF EXISTS, trap warnings here
  65         local $SIG{__WARN__} = sub { warn @_ unless $_[0] =~ /no such table.+DROP TABLE/ };
  66
  67         my ($class, $schema_method) = @_;
  68
  69         my $method = "get_schema_$schema_method";
  70         my $schema = $class->$method;
  71
  72         return $schema;
  73     }
  74
  75     sub get_schema_by_storage_type {
  76       DBICTest->init_schema(
  77         sqlite_use_file => 1,
  78         storage_type=>{
  79           '::DBI::Replicated' => {
  80             balancer_type=>'::Random',
  81             balancer_args=>{
  82               auto_validate_every=>100,
  83               master_read_weight => 1
  84             },
  85           }
  86         },
  87         deploy_args=>{
  88           add_drop_table => 1,
  89         },
  90       );
  91     }
  92
  93     sub get_schema_by_connect_info {
  94       DBICTest->init_schema(
  95         sqlite_use_file => 1,
  96         storage_type=> '::DBI::Replicated',
  97         balancer_type=>'::Random',
  98         balancer_args=> {
  99           auto_validate_every=>100,
 100           master_read_weight => 1
 101         },
 102         deploy_args=>{
 103           add_drop_table => 1,
 104         },
 105       );
 106     }
 107
 108     sub generate_replicant_connect_info {}
 109     sub replicate {}
 110     sub cleanup {}
 111
 112     ## --------------------------------------------------------------------- ##
 113     ## Add a connect_info option to test option merging.
 114     ## --------------------------------------------------------------------- ##
 115     {
 116     package DBIx::Class::Storage::DBI::Replicated;
 117
 118     use Moose;
 119
 120     __PACKAGE__->meta->make_mutable;
 121
 122     around connect_info => sub {
 123       my ($next, $self, $info) = @_;
 124       $info->[3]{master_option} = 1;
 125       $self->$next($info);
 126     };
 127
 128     __PACKAGE__->meta->make_immutable;
 129
 130     no Moose;
 131     }
 132
 133     ## --------------------------------------------------------------------- ##
 134     ## Subclass for when you are using SQLite for testing, this provides a fake
 135     ## replication support.
 136     ## --------------------------------------------------------------------- ##
 137
 138     package DBIx::Class::DBI::Replicated::TestReplication::SQLite;
 139
 140     use DBICTest;
 141     use File::Copy;
 142     use base 'DBIx::Class::DBI::Replicated::TestReplication';
 143
 144     __PACKAGE__->mk_accessors( qw/master_path slave_paths/ );
 145
 146     ## Set the mastep path from DBICTest
 147
 148         sub new {
 149             my $class = shift @_;
 150             my $self = $class->SUPER::new(@_);
 151
 152             $self->master_path( DBICTest->_sqlite_dbfilename );
 153             $self->slave_paths([
 154             "t/var/DBIxClass_slave1.db",
 155             "t/var/DBIxClass_slave2.db",
 156         ]);
 157
 158             return $self;
 159         }
 160
 161     ## Return an Array of ArrayRefs where each ArrayRef is suitable to use for
 162     ## $storage->connect_info to be used for connecting replicants.
 163
 164     sub generate_replicant_connect_info {
 165         my $self = shift @_;
 166         my @dsn = map {
 167             "dbi:SQLite:${_}";
 168         } @{$self->slave_paths};
 169
 170         my @connect_infos = map { [$_,'','',{AutoCommit=>1}] } @dsn;
 171
 172     # try a hashref too
 173         my $c = $connect_infos[0];
 174         $connect_infos[0] = {
 175           dsn => $c->[0],
 176           user => $c->[1],
 177           password => $c->[2],
 178           %{ $c->[3] }
 179         };
 180
 181         @connect_infos
 182     }
 183
 184     ## Do a 'good enough' replication by copying the master dbfile over each of
 185     ## the slave dbfiles.  If the master is SQLite we do this, otherwise we
 186     ## just do a one second pause to let the slaves catch up.
 187
 188     sub replicate {
 189         my $self = shift @_;
 190         foreach my $slave (@{$self->slave_paths}) {
 191             copy($self->master_path, $slave);
 192         }
 193     }
 194
 195     ## Cleanup after ourselves.  Unlink all gthe slave paths.
 196
 197     sub cleanup {
 198         my $self = shift @_;
 199         foreach my $slave (@{$self->slave_paths}) {
 200             unlink $slave;
 201         }
 202     }
 203
 204     ## --------------------------------------------------------------------- ##
 205     ## Subclass for when you are setting the databases via custom export vars
 206     ## This is for when you have a replicating database setup that you are
 207     ## going to test against.  You'll need to define the correct $ENV and have
 208     ## two slave databases to test against, as well as a replication system
 209     ## that will replicate in less than 1 second.
 210     ## --------------------------------------------------------------------- ##
 211
 212     package DBIx::Class::DBI::Replicated::TestReplication::Custom;
 213     use base 'DBIx::Class::DBI::Replicated::TestReplication';
 214
 215     ## Return an Array of ArrayRefs where each ArrayRef is suitable to use for
 216     ## $storage->connect_info to be used for connecting replicants.
 217
 218     sub generate_replicant_connect_info {
 219         return (
 220             [$ENV{"DBICTEST_SLAVE0_DSN"}, $ENV{"DBICTEST_SLAVE0_DBUSER"}, $ENV{"DBICTEST_SLAVE0_DBPASS"}, {AutoCommit => 1}],
 221             [$ENV{"DBICTEST_SLAVE1_DSN"}, $ENV{"DBICTEST_SLAVE1_DBUSER"}, $ENV{"DBICTEST_SLAVE1_DBPASS"}, {AutoCommit => 1}],
 222         );
 223     }
 224
 225     ## pause a bit to let the replication catch up
 226
 227     sub replicate {
 228         sleep 1;
 229     }
 230 }
 231
 232 ## ----------------------------------------------------------------------------
 233 ## Create an object and run some tests
 234 ## ----------------------------------------------------------------------------
 235
 236 ## Thi first bunch of tests are basic, just make sure all the bits are behaving
 237
 238 my $replicated_class = DBICTest->has_custom_dsn ?
 239     'DBIx::Class::DBI::Replicated::TestReplication::Custom' :
 240     'DBIx::Class::DBI::Replicated::TestReplication::SQLite';
 241
 242 my $replicated;
 243
 244 for my $method (qw/by_connect_info by_storage_type/) {
 245   ok $replicated = $replicated_class->new($method)
 246       => "Created a replication object $method";
 247
 248   isa_ok $replicated->schema
 249       => 'DBIx::Class::Schema';
 250
 251   isa_ok $replicated->schema->storage
 252       => 'DBIx::Class::Storage::DBI::Replicated';
 253
 254   isa_ok $replicated->schema->storage->balancer
 255       => 'DBIx::Class::Storage::DBI::Replicated::Balancer::Random'
 256       => 'configured balancer_type';
 257 }
 258
 259 ok $replicated->schema->storage->meta
 260     => 'has a meta object';
 261
 262 isa_ok $replicated->schema->storage->master
 263     => 'DBIx::Class::Storage::DBI';
 264
 265 isa_ok $replicated->schema->storage->pool
 266     => 'DBIx::Class::Storage::DBI::Replicated::Pool';
 267
 268 does_ok $replicated->schema->storage->balancer
 269     => 'DBIx::Class::Storage::DBI::Replicated::Balancer';
 270
 271 ok my @replicant_connects = $replicated->generate_replicant_connect_info
 272     => 'got replication connect information';
 273
 274 ok my @replicated_storages = $replicated->schema->storage->connect_replicants(@replicant_connects)
 275     => 'Created some storages suitable for replicants';
 276
 277 ok my @all_storages = $replicated->schema->storage->all_storages
 278     => '->all_storages';
 279
 280 is scalar @all_storages,
 281     3
 282     => 'correct number of ->all_storages';
 283
 284 is ((grep $_->isa('DBIx::Class::Storage::DBI'), @all_storages),
 285     3
 286     => '->all_storages are correct type');
 287
 288 my @all_storage_opts =
 289   grep { (reftype($_)||'') eq 'HASH' }
 290     map @{ $_->_connect_info }, @all_storages;
 291
 292 is ((grep $_->{master_option}, @all_storage_opts),
 293     3
 294     => 'connect_info was merged from master to replicants');
 295
 296 my @replicant_names = keys %{ $replicated->schema->storage->replicants };
 297
 298 ## Silence warning about not supporting the is_replicating method if using the
 299 ## sqlite dbs.
 300 $replicated->schema->storage->debugobj->silence(1)
 301   if first { m{^t/} } @replicant_names;
 302
 303 isa_ok $replicated->schema->storage->balancer->current_replicant
 304     => 'DBIx::Class::Storage::DBI';
 305
 306 $replicated->schema->storage->debugobj->silence(0);
 307
 308 ok $replicated->schema->storage->pool->has_replicants
 309     => 'does have replicants';
 310
 311 is $replicated->schema->storage->pool->num_replicants => 2
 312     => 'has two replicants';
 313
 314 does_ok $replicated_storages[0]
 315     => 'DBIx::Class::Storage::DBI::Replicated::Replicant';
 316
 317 does_ok $replicated_storages[1]
 318     => 'DBIx::Class::Storage::DBI::Replicated::Replicant';
 319
 320 does_ok $replicated->schema->storage->replicants->{$replicant_names[0]}
 321     => 'DBIx::Class::Storage::DBI::Replicated::Replicant';
 322
 323 does_ok $replicated->schema->storage->replicants->{$replicant_names[1]}
 324     => 'DBIx::Class::Storage::DBI::Replicated::Replicant';
 325
 326 ## Add some info to the database
 327
 328 $replicated
 329     ->schema
 330     ->populate('Artist', [
 331         [ qw/artistid name/ ],
 332         [ 4, "Ozric Tentacles"],
 333     ]);
 334
 335 ## Make sure all the slaves have the table definitions
 336
 337 $replicated->replicate;
 338 $replicated->schema->storage->replicants->{$replicant_names[0]}->active(1);
 339 $replicated->schema->storage->replicants->{$replicant_names[1]}->active(1);
 340
 341 ## Silence warning about not supporting the is_replicating method if using the
 342 ## sqlite dbs.
 343 $replicated->schema->storage->debugobj->silence(1)
 344   if first { m{^t/} } @replicant_names;
 345
 346 $replicated->schema->storage->pool->validate_replicants;
 347
 348 $replicated->schema->storage->debugobj->silence(0);
 349
 350 ## Make sure we can read the data.
 351
 352 ok my $artist1 = $replicated->schema->resultset('Artist')->find(4)
 353     => 'Created Result';
 354
 355 isa_ok $artist1
 356     => 'DBICTest::Artist';
 357
 358 is $artist1->name, 'Ozric Tentacles'
 359     => 'Found expected name for first result';
 360
 361 ## Check that master_read_weight is honored
 362 {
 363     no warnings 'once';
 364
 365     # turn off redefined warning
 366     local $SIG{__WARN__} = sub {};
 367
 368     local
 369     *DBIx::Class::Storage::DBI::Replicated::Balancer::Random::random_number =
 370         sub { 999 };
 371
 372     $replicated->schema->storage->balancer->increment_storage;
 373
 374     is $replicated->schema->storage->balancer->current_replicant,
 375        $replicated->schema->storage->master
 376        => 'master_read_weight is honored';
 377
 378     ## turn it off for the duration of the test
 379     $replicated->schema->storage->balancer->master_read_weight(0);
 380     $replicated->schema->storage->balancer->increment_storage;
 381 }
 382
 383 ## Add some new rows that only the master will have  This is because
 384 ## we overload any type of write operation so that is must hit the master
 385 ## database.
 386
 387 $replicated
 388     ->schema
 389     ->populate('Artist', [
 390         [ qw/artistid name/ ],
 391         [ 5, "Doom's Children"],
 392         [ 6, "Dead On Arrival"],
 393         [ 7, "Watergate"],
 394     ]);
 395
 396 ## Make sure all the slaves have the table definitions
 397 $replicated->replicate;
 398
 399 ## Should find some data now
 400
 401 ok my $artist2 = $replicated->schema->resultset('Artist')->find(5)
 402     => 'Sync succeed';
 403
 404 isa_ok $artist2
 405     => 'DBICTest::Artist';
 406
 407 is $artist2->name, "Doom's Children"
 408     => 'Found expected name for first result';
 409
 410 ## What happens when we disconnect all the replicants?
 411
 412 is $replicated->schema->storage->pool->connected_replicants => 2
 413     => "both replicants are connected";
 414
 415 $replicated->schema->storage->replicants->{$replicant_names[0]}->disconnect;
 416 $replicated->schema->storage->replicants->{$replicant_names[1]}->disconnect;
 417
 418 is $replicated->schema->storage->pool->connected_replicants => 0
 419     => "both replicants are now disconnected";
 420
 421 ## All these should pass, since the database should automatically reconnect
 422
 423 ok my $artist3 = $replicated->schema->resultset('Artist')->find(6)
 424     => 'Still finding stuff.';
 425
 426 isa_ok $artist3
 427     => 'DBICTest::Artist';
 428
 429 is $artist3->name, "Dead On Arrival"
 430     => 'Found expected name for first result';
 431
 432 is $replicated->schema->storage->pool->connected_replicants => 1
 433     => "At Least One replicant reconnected to handle the job";
 434
 435 ## What happens when we try to select something that doesn't exist?
 436
 437 ok ! $replicated->schema->resultset('Artist')->find(666)
 438     => 'Correctly failed to find something.';
 439
 440 ## test the reliable option
 441
 442 TESTRELIABLE: {
 443
 444         $replicated->schema->storage->set_reliable_storage;
 445
 446         ok $replicated->schema->resultset('Artist')->find(2)
 447             => 'Read from master 1';
 448
 449         ok $replicated->schema->resultset('Artist')->find(5)
 450             => 'Read from master 2';
 451
 452     $replicated->schema->storage->set_balanced_storage;
 453
 454         ok $replicated->schema->resultset('Artist')->find(3)
 455         => 'Read from replicant';
 456 }
 457
 458 ## Make sure when reliable goes out of scope, we are using replicants again
 459
 460 ok $replicated->schema->resultset('Artist')->find(1)
 461     => 'back to replicant 1.';
 462
 463 ok $replicated->schema->resultset('Artist')->find(2)
 464     => 'back to replicant 2.';
 465
 466 ## set all the replicants to inactive, and make sure the balancer falls back to
 467 ## the master.
 468
 469 $replicated->schema->storage->replicants->{$replicant_names[0]}->active(0);
 470 $replicated->schema->storage->replicants->{$replicant_names[1]}->active(0);
 471
 472 ## Silence warning about falling back to master.
 473 $replicated->schema->storage->debugobj->silence(1);
 474
 475 ok $replicated->schema->resultset('Artist')->find(2)
 476     => 'Fallback to master';
 477
 478 $replicated->schema->storage->debugobj->silence(0);
 479
 480 $replicated->schema->storage->replicants->{$replicant_names[0]}->active(1);
 481 $replicated->schema->storage->replicants->{$replicant_names[1]}->active(1);
 482
 483 ## Silence warning about not supporting the is_replicating method if using the
 484 ## sqlite dbs.
 485 $replicated->schema->storage->debugobj->silence(1)
 486   if first { m{^t/} } @replicant_names;
 487
 488 $replicated->schema->storage->pool->validate_replicants;
 489
 490 $replicated->schema->storage->debugobj->silence(0);
 491
 492 ok $replicated->schema->resultset('Artist')->find(2)
 493     => 'Returned to replicates';
 494
 495 ## Getting slave status tests
 496
 497 SKIP: {
 498     ## We skip this tests unless you have a custom replicants, since the default
 499     ## sqlite based replication tests don't support these functions.
 500
 501     skip 'Cannot Test Replicant Status on Non Replicating Database', 9
 502      unless DBICTest->has_custom_dsn && $ENV{"DBICTEST_SLAVE0_DSN"};
 503
 504     $replicated->replicate; ## Give the slaves a chance to catchup.
 505
 506         ok $replicated->schema->storage->replicants->{$replicant_names[0]}->is_replicating
 507             => 'Replicants are replicating';
 508
 509         is $replicated->schema->storage->replicants->{$replicant_names[0]}->lag_behind_master, 0
 510             => 'Replicant is zero seconds behind master';
 511
 512         ## Test the validate replicants
 513
 514         $replicated->schema->storage->pool->validate_replicants;
 515
 516         is $replicated->schema->storage->pool->active_replicants, 2
 517             => 'Still have 2 replicants after validation';
 518
 519         ## Force the replicants to fail the validate test by required their lag to
 520         ## be negative (ie ahead of the master!)
 521
 522     $replicated->schema->storage->pool->maximum_lag(-10);
 523     $replicated->schema->storage->pool->validate_replicants;
 524
 525     is $replicated->schema->storage->pool->active_replicants, 0
 526         => 'No way a replicant be be ahead of the master';
 527
 528     ## Let's be fair to the replicants again.  Let them lag up to 5
 529
 530     $replicated->schema->storage->pool->maximum_lag(5);
 531     $replicated->schema->storage->pool->validate_replicants;
 532
 533     is $replicated->schema->storage->pool->active_replicants, 2
 534         => 'Both replicants in good standing again';
 535
 536         ## Check auto validate
 537
 538         is $replicated->schema->storage->balancer->auto_validate_every, 100
 539             => "Got the expected value for auto validate";
 540
 541                 ## This will make sure we auto validatge everytime
 542                 $replicated->schema->storage->balancer->auto_validate_every(0);
 543
 544                 ## set all the replicants to inactive, and make sure the balancer falls back to
 545                 ## the master.
 546
 547                 $replicated->schema->storage->replicants->{$replicant_names[0]}->active(0);
 548                 $replicated->schema->storage->replicants->{$replicant_names[1]}->active(0);
 549
 550                 ## Ok, now when we go to run a query, autovalidate SHOULD reconnect
 551
 552         is $replicated->schema->storage->pool->active_replicants => 0
 553             => "both replicants turned off";
 554
 555         ok $replicated->schema->resultset('Artist')->find(5)
 556             => 'replicant reactivated';
 557
 558         is $replicated->schema->storage->pool->active_replicants => 2
 559             => "both replicants reactivated";
 560 }
 561
 562 ## Test the reliably callback
 563
 564 ok my $reliably = sub {
 565
 566     ok $replicated->schema->resultset('Artist')->find(5)
 567         => 'replicant reactivated';
 568
 569 } => 'created coderef properly';
 570
 571 $replicated->schema->storage->execute_reliably($reliably);
 572
 573 ## Try something with an error
 574
 575 ok my $unreliably = sub {
 576
 577     ok $replicated->schema->resultset('ArtistXX')->find(5)
 578         => 'replicant reactivated';
 579
 580 } => 'created coderef properly';
 581
 582 throws_ok {$replicated->schema->storage->execute_reliably($unreliably)}
 583     qr/Can't find source for ArtistXX/
 584     => 'Bad coderef throws proper error';
 585
 586 ## Make sure replication came back
 587
 588 ok $replicated->schema->resultset('Artist')->find(3)
 589     => 'replicant reactivated';
 590
 591 ## make sure transactions are set to execute_reliably
 592
 593 ok my $transaction = sub {
 594
 595         my $id = shift @_;
 596
 597         $replicated
 598             ->schema
 599             ->populate('Artist', [
 600                 [ qw/artistid name/ ],
 601                 [ $id, "Children of the Grave"],
 602             ]);
 603
 604     ok my $result = $replicated->schema->resultset('Artist')->find($id)
 605         => 'Found expected artist';
 606
 607     ok my $more = $replicated->schema->resultset('Artist')->find(1)
 608         => 'Found expected artist again';
 609
 610    return ($result, $more);
 611
 612 } => 'Created a coderef properly';
 613
 614 ## Test the transaction with multi return
 615 {
 616         ok my @return = $replicated->schema->txn_do($transaction, 666)
 617             => 'did transaction';
 618
 619             is $return[0]->id, 666
 620                 => 'first returned value is correct';
 621
 622             is $return[1]->id, 1
 623                 => 'second returned value is correct';
 624 }
 625
 626 ## Test that asking for single return works
 627 {
 628         ok my $return = $replicated->schema->txn_do($transaction, 777)
 629             => 'did transaction';
 630
 631             is $return->id, 777
 632                 => 'first returned value is correct';
 633 }
 634
 635 ## Test transaction returning a single value
 636
 637 {
 638         ok my $result = $replicated->schema->txn_do(sub {
 639                 ok my $more = $replicated->schema->resultset('Artist')->find(1)
 640                 => 'found inside a transaction';
 641                 return $more;
 642         }) => 'successfully processed transaction';
 643
 644         is $result->id, 1
 645            => 'Got expected single result from transaction';
 646 }
 647
 648 ## Make sure replication came back
 649
 650 ok $replicated->schema->resultset('Artist')->find(1)
 651     => 'replicant reactivated';
 652
 653 ## Test Discard changes
 654
 655 {
 656         ok my $artist = $replicated->schema->resultset('Artist')->find(2)
 657             => 'got an artist to test discard changes';
 658
 659         ok $artist->discard_changes
 660            => 'properly discard changes';
 661 }
 662
 663 ## Test some edge cases, like trying to do a transaction inside a transaction, etc
 664
 665 {
 666     ok my $result = $replicated->schema->txn_do(sub {
 667         return $replicated->schema->txn_do(sub {
 668                 ok my $more = $replicated->schema->resultset('Artist')->find(1)
 669                 => 'found inside a transaction inside a transaction';
 670                 return $more;
 671         });
 672     }) => 'successfully processed transaction';
 673
 674     is $result->id, 1
 675        => 'Got expected single result from transaction';
 676 }
 677
 678 {
 679     ok my $result = $replicated->schema->txn_do(sub {
 680         return $replicated->schema->storage->execute_reliably(sub {
 681                 return $replicated->schema->txn_do(sub {
 682                         return $replicated->schema->storage->execute_reliably(sub {
 683                                 ok my $more = $replicated->schema->resultset('Artist')->find(1)
 684                                 => 'found inside crazy deep transactions and execute_reliably';
 685                                 return $more;
 686                         });
 687                 });
 688         });
 689     }) => 'successfully processed transaction';
 690
 691     is $result->id, 1
 692        => 'Got expected single result from transaction';
 693 }
 694
 695 ## Test the force_pool resultset attribute.
 696
 697 {
 698         ok my $artist_rs = $replicated->schema->resultset('Artist')
 699         => 'got artist resultset';
 700
 701         ## Turn on Forced Pool Storage
 702         ok my $reliable_artist_rs = $artist_rs->search(undef, {force_pool=>'master'})
 703         => 'Created a resultset using force_pool storage';
 704
 705     ok my $artist = $reliable_artist_rs->find(2)
 706         => 'got an artist result via force_pool storage';
 707 }
 708
 709 ## Delete the old database files
 710 $replicated->cleanup;
 711
 712 # vim: sw=4 sts=4 :