X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?a=blobdiff_plain;f=lib%2FDBIx%2FClass%2FStorage%2FDBI%2FReplicated.pm;h=259cdc52bf777de0ad4c84145e316d5c81309e35;hb=a0ce462cf118f0da7c960a9e69ea7c6fb50f5f57;hp=bb6517a6b25d511849f718bedf2b523830526fdd;hpb=2ce6e9a68d68fa86e3b28df61496e85c996b2c1a;p=dbsrgits%2FDBIx-Class.git diff --git a/lib/DBIx/Class/Storage/DBI/Replicated.pm b/lib/DBIx/Class/Storage/DBI/Replicated.pm index bb6517a..259cdc5 100644 --- a/lib/DBIx/Class/Storage/DBI/Replicated.pm +++ b/lib/DBIx/Class/Storage/DBI/Replicated.pm @@ -1,15 +1,46 @@ package DBIx::Class::Storage::DBI::Replicated; +BEGIN { + use Carp::Clan qw/^DBIx::Class/; + + ## Modules required for Replication support not required for general DBIC + ## use, so we explicitly test for these. + + my %replication_required = ( + Moose => '0.77', + MooseX::AttributeHelpers => '0.12', + MooseX::Types => '0.10', + namespace::clean => '0.11', + Hash::Merge => '0.11' + ); + + my @didnt_load; + + for my $module (keys %replication_required) { + eval "use $module $replication_required{$module}"; + push @didnt_load, "$module $replication_required{$module}" + if $@; + } + + croak("@{[ join ', ', @didnt_load ]} are missing and are required for Replication") + if @didnt_load; +} + use Moose; -use Class::MOP; -use Moose::Util::TypeConstraints; use DBIx::Class::Storage::DBI; use DBIx::Class::Storage::DBI::Replicated::Pool; use DBIx::Class::Storage::DBI::Replicated::Balancer; +use DBIx::Class::Storage::DBI::Replicated::Types 'BalancerClassNamePart'; +use MooseX::Types::Moose qw/ClassName HashRef Object/; +use Scalar::Util 'reftype'; +use Carp::Clan qw/^DBIx::Class/; +use Hash::Merge 'merge'; + +use namespace::clean -except => 'meta'; =head1 NAME -DBIx::Class::Storage::DBI::Replicated - ALPHA Replicated database support +DBIx::Class::Storage::DBI::Replicated - BETA Replicated database support =head1 SYNOPSIS @@ -29,12 +60,28 @@ tasks. [$dsn3, $user, $pass, \%opts], ); + ## Now, just use the $schema as normal + $schema->resultset('Source')->search({name=>'etc'}); + + ## You can force a given query to use a particular storage using the search + ### attribute 'force_pool'. For example: + + my $RS = $schema->resultset('Source')->search(undef, {force_pool=>'master'}); + + ## Now $RS will force everything (both reads and writes) to use whatever was + ## setup as the master storage. 'master' is hardcoded to always point to the + ## Master, but you can also use any Replicant name. Please see: + ## L and the replicants attribute for + ## More. Also see transactions and L for alternative ways + ## to force read traffic to the master. + =head1 DESCRIPTION -Warning: This class is marked ALPHA. We are using this in development and have -some basic test coverage but the code hasn't yet been stressed by a variety -of databases. Individual DB's may have quirks we are not aware of. Please -use this in development and pass along your experiences/bug fixes. +Warning: This class is marked BETA. This has been running a production +website using MySQL native replication as its backend and we have some decent +test coverage but the code hasn't yet been stressed by a variety of databases. +Individual DB's may have quirks we are not aware of. Please use this in first +development and pass along your experiences/bug fixes. This class implements replicated data store for DBI. Currently you can define one master and numerous slave database connections. All write-type queries @@ -54,9 +101,21 @@ selected algorithm. The default algorithm is random weighted. The consistancy betweeen master and replicants is database specific. The Pool gives you a method to validate it's replicants, removing and replacing them -when they fail/pass predefined criteria. It is recommened that your application -define two schemas, one using the replicated storage and another that just -connects to the master. +when they fail/pass predefined criteria. Please make careful use of the ways +to force a query to run against Master when needed. + +=head1 REQUIREMENTS + +Replicated Storage has additional requirements not currently part of L + + Moose => 0.77 + MooseX::AttributeHelpers => 0.12 + MooseX::Types => 0.10 + namespace::clean => 0.11 + Hash::Merge => 0.11 + +You will need to install these modules manually via CPAN or make them part of the +Makefile for your distribution. =head1 ATTRIBUTES @@ -83,9 +142,8 @@ to: L. =cut has 'pool_type' => ( - is=>'ro', - isa=>'ClassName', - required=>1, + is=>'rw', + isa=>ClassName, default=>'DBIx::Class::Storage::DBI::Replicated::Pool', handles=>{ 'create_pool' => 'new', @@ -100,10 +158,9 @@ See L for available arguments. =cut has 'pool_args' => ( - is=>'ro', - isa=>'HashRef', + is=>'rw', + isa=>HashRef, lazy=>1, - required=>1, default=>sub { {} }, ); @@ -115,23 +172,9 @@ choose how to spread the query load across each replicant in the pool. =cut -subtype 'DBIx::Class::Storage::DBI::Replicated::BalancerClassNamePart', - as 'ClassName'; - -coerce 'DBIx::Class::Storage::DBI::Replicated::BalancerClassNamePart', - from 'Str', - via { - my $type = $_; - if($type=~m/^::/) { - $type = 'DBIx::Class::Storage::DBI::Replicated::Balancer'.$type; - } - Class::MOP::load_class($type); - $type; - }; - has 'balancer_type' => ( - is=>'ro', - isa=>'DBIx::Class::Storage::DBI::Replicated::BalancerClassNamePart', + is=>'rw', + isa=>BalancerClassNamePart, coerce=>1, required=>1, default=> 'DBIx::Class::Storage::DBI::Replicated::Balancer::First', @@ -148,8 +191,8 @@ See L for available arguments. =cut has 'balancer_args' => ( - is=>'ro', - isa=>'HashRef', + is=>'rw', + isa=>HashRef, lazy=>1, required=>1, default=>sub { {} }, @@ -181,7 +224,7 @@ is a class that takes a pool () =cut has 'balancer' => ( - is=>'ro', + is=>'rw', isa=>'DBIx::Class::Storage::DBI::Replicated::Balancer', lazy_build=>1, handles=>[qw/auto_validate_every/], @@ -216,7 +259,7 @@ Defines an object that implements the read side of L. has 'read_handler' => ( is=>'rw', - isa=>'Object', + isa=>Object, lazy_build=>1, handles=>[qw/ select @@ -233,8 +276,7 @@ Defines an object that implements the write side of L. has 'write_handler' => ( is=>'ro', - isa=>'Object', - lazy_build=>1, + isa=>Object, lazy_build=>1, handles=>[qw/ on_connect_do @@ -260,19 +302,83 @@ has 'write_handler' => ( txn_scope_guard sth deploy + with_deferred_fk_checks reload_row _prep_for_execute - configure_sqlt /], ); +has _master_connect_info_opts => + (is => 'rw', isa => HashRef, default => sub { {} }); + +=head2 around: connect_info + +Preserve master's C options (for merging with replicants.) +Also set any Replicated related options from connect_info, such as +C, C, C and C. + +=cut + +around connect_info => sub { + my ($next, $self, $info, @extra) = @_; + + my $wantarray = wantarray; + + my %opts; + for my $arg (@$info) { + next unless (reftype($arg)||'') eq 'HASH'; + %opts = %{ merge($arg, \%opts) }; + } + delete $opts{dsn}; + + if (@opts{qw/pool_type pool_args/}) { + $self->pool_type(delete $opts{pool_type}) + if $opts{pool_type}; + + $self->pool_args( + merge((delete $opts{pool_args} || {}), $self->pool_args) + ); + + $self->pool($self->_build_pool) + if $self->pool; + } + + if (@opts{qw/balancer_type balancer_args/}) { + $self->balancer_type(delete $opts{balancer_type}) + if $opts{balancer_type}; + + $self->balancer_args( + merge((delete $opts{balancer_args} || {}), $self->balancer_args) + ); + + $self->balancer($self->_build_balancer) + if $self->balancer; + } + + $self->_master_connect_info_opts(\%opts); + + my (@res, $res); + if ($wantarray) { + @res = $self->$next($info, @extra); + } else { + $res = $self->$next($info, @extra); + } + + # May have to reapply role if master will be reblessed to a more specific + # driver. + $self->master->_determine_driver; + DBIx::Class::Storage::DBI::Replicated::WithDSN->meta->apply($self->master); + + $wantarray ? @res : $res; +}; + =head1 METHODS This class defines the following methods. -=head2 new +=head2 BUILDARGS L when instantiating it's storage passed itself as the first argument. So we need to massage the arguments a bit so that all the @@ -280,10 +386,15 @@ bits get put into the correct places. =cut -around 'new' => sub { - my ($new, $self, $schema, $storage_type_args, @args) = @_; - return $self->$new(schema=>$schema, %$storage_type_args, @args); -}; +sub BUILDARGS { + my ($class, $schema, $storage_type_args, @args) = @_; + + return { + schema=>$schema, + %$storage_type_args, + @args + } +} =head2 _build_master @@ -293,7 +404,9 @@ Lazy builder for the L attribute. sub _build_master { my $self = shift @_; - DBIx::Class::Storage::DBI->new($self->schema); + my $master = DBIx::Class::Storage::DBI->new($self->schema); + DBIx::Class::Storage::DBI::Replicated::WithDSN->meta->apply($master); + $master } =head2 _build_pool @@ -348,13 +461,49 @@ sub _build_read_handler { =head2 around: connect_replicants All calls to connect_replicants needs to have an existing $schema tacked onto -top of the args, since L needs it. +top of the args, since L needs it, and any C +options merged with the master, with replicant opts having higher priority. =cut -around 'connect_replicants' => sub { - my ($method, $self, @args) = @_; - $self->$method($self->schema, @args); +around connect_replicants => sub { + my ($next, $self, @args) = @_; + + for my $r (@args) { + $r = [ $r ] unless reftype $r eq 'ARRAY'; + + croak "coderef replicant connect_info not supported" + if ref $r->[0] && reftype $r->[0] eq 'CODE'; + +# any connect_info options? + my $i = 0; + $i++ while $i < @$r && (reftype($r->[$i])||'') ne 'HASH'; + +# make one if none + $r->[$i] = {} unless $r->[$i]; + +# merge if two hashes + my @hashes = @$r[$i .. $#{$r}]; + + croak "invalid connect_info options" + if (grep { reftype($_) eq 'HASH' } @hashes) != @hashes; + + croak "too many hashrefs in connect_info" + if @hashes > 2; + + my %opts = %{ merge(reverse @hashes) }; + +# delete them + splice @$r, $i+1, ($#{$r} - $i), (); + +# merge with master + %opts = %{ merge(\%opts, $self->_master_connect_info_opts) }; + +# update + $r->[$i] = \%opts; + } + + $self->$next($self->schema, @args); }; =head2 all_storages @@ -369,7 +518,7 @@ sub all_storages { my $self = shift @_; return grep {defined $_ && blessed $_} ( $self->master, - $self->replicants, + values %{ $self->replicants }, ); } @@ -481,20 +630,6 @@ around 'txn_do' => sub { $self->execute_reliably(sub {$self->$txn_do($coderef, @args)}); }; -=head2 reload_row ($row) - -Overload to the reload_row method so that the reloading is always directed to -the master storage. - -=cut - -around 'reload_row' => sub { - my ($reload_row, $self, $row) = @_; - return $self->execute_reliably(sub { - return $self->$reload_row(shift); - }, $row); -}; - =head2 connected Check that the master and at least one of the replicants is connected. @@ -532,6 +667,7 @@ sub limit_dialect { foreach my $source ($self->all_storages) { $source->limit_dialect(@_); } + return $self->master->quote_char; } =head2 quote_char @@ -545,6 +681,7 @@ sub quote_char { foreach my $source ($self->all_storages) { $source->quote_char(@_); } + return $self->master->quote_char; } =head2 name_sep @@ -558,6 +695,7 @@ sub name_sep { foreach my $source ($self->all_storages) { $source->name_sep(@_); } + return $self->master->name_sep; } =head2 set_schema @@ -581,9 +719,12 @@ set a debug flag across all storages sub debug { my $self = shift @_; - foreach my $source ($self->all_storages) { - $source->debug(@_); + if(@_) { + foreach my $source ($self->all_storages) { + $source->debug(@_); + } } + return $self->master->debug; } =head2 debugobj @@ -594,9 +735,12 @@ set a debug object across all storages sub debugobj { my $self = shift @_; - foreach my $source ($self->all_storages) { - $source->debugobj(@_); + if(@_) { + foreach my $source ($self->all_storages) { + $source->debugobj(@_); + } } + return $self->master->debugobj; } =head2 debugfh @@ -607,9 +751,12 @@ set a debugfh object across all storages sub debugfh { my $self = shift @_; - foreach my $source ($self->all_storages) { - $source->debugfh(@_); + if(@_) { + foreach my $source ($self->all_storages) { + $source->debugfh(@_); + } } + return $self->master->debugfh; } =head2 debugcb @@ -620,9 +767,12 @@ set a debug callback across all storages sub debugcb { my $self = shift @_; - foreach my $source ($self->all_storages) { - $source->debugcb(@_); + if(@_) { + foreach my $source ($self->all_storages) { + $source->debugcb(@_); + } } + return $self->master->debugcb; } =head2 disconnect @@ -638,6 +788,58 @@ sub disconnect { } } +=head2 cursor_class + +set cursor class on all storages, or return master's + +=cut + +sub cursor_class { + my ($self, $cursor_class) = @_; + + if ($cursor_class) { + $_->cursor_class($cursor_class) for $self->all_storages; + } + $self->master->cursor_class; +} + +=head1 GOTCHAS + +Due to the fact that replicants can lag behind a master, you must take care to +make sure you use one of the methods to force read queries to a master should +you need realtime data integrity. For example, if you insert a row, and then +immediately re-read it from the database (say, by doing $row->discard_changes) +or you insert a row and then immediately build a query that expects that row +to be an item, you should force the master to handle reads. Otherwise, due to +the lag, there is no certainty your data will be in the expected state. + +For data integrity, all transactions automatically use the master storage for +all read and write queries. Using a transaction is the preferred and recommended +method to force the master to handle all read queries. + +Otherwise, you can force a single query to use the master with the 'force_pool' +attribute: + + my $row = $resultset->search(undef, {force_pool=>'master'})->find($pk); + +This attribute will safely be ignore by non replicated storages, so you can use +the same code for both types of systems. + +Lastly, you can use the L method, which works very much like +a transaction. + +For debugging, you can turn replication on/off with the methods L +and L, however this operates at a global level and is not +suitable if you have a shared Schema object being used by multiple processes, +such as on a web application server. You can get around this limitation by +using the Schema clone method. + + my $new_schema = $schema->clone; + $new_schema->set_reliable_storage; + + ## $new_schema will use only the Master storage for all reads/writes while + ## the $schema object will use replicated storage. + =head1 AUTHOR John Napiorkowski @@ -653,4 +855,6 @@ You may distribute this code under the same terms as Perl itself. =cut +__PACKAGE__->meta->make_immutable; + 1;