X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?p=dbsrgits%2FDBIx-Class.git;a=blobdiff_plain;f=lib%2FDBIx%2FClass%2FStorage%2FDBI%2FReplicated.pm;h=447747514dd38272da3c17fa1c9e635944407822;hp=d6eb2fdd42194723123e35392c97a0fd81ac2b7a;hb=64ae166780d0cb2b9577e506da9b9b240c146d20;hpb=ee356d004e088518205ac720b37c31a64c9cd861 diff --git a/lib/DBIx/Class/Storage/DBI/Replicated.pm b/lib/DBIx/Class/Storage/DBI/Replicated.pm index d6eb2fd..4477475 100644 --- a/lib/DBIx/Class/Storage/DBI/Replicated.pm +++ b/lib/DBIx/Class/Storage/DBI/Replicated.pm @@ -2,37 +2,22 @@ package DBIx::Class::Storage::DBI::Replicated; BEGIN { use Carp::Clan qw/^DBIx::Class/; - - ## Modules required for Replication support not required for general DBIC - ## use, so we explicitly test for these. - - my %replication_required = ( - Moose => '0.77', - MooseX::AttributeHelpers => '0.12', - MooseX::Types => '0.10', - namespace::clean => '0.11', - ); - - my @didnt_load; - - for my $module (keys %replication_required) { - eval "use $module $replication_required{$module}"; - push @didnt_load, "$module $replication_required{$module}" - if $@; - } - - croak("@{[ join ', ', @didnt_load ]} are missing and are required for Replication") - if @didnt_load; + use DBIx::Class; + croak('The following modules are required for Replication ' . DBIx::Class::Optional::Dependencies->req_missing_for ('replicated') ) + unless DBIx::Class::Optional::Dependencies->req_ok_for ('replicated'); } use Moose; use DBIx::Class::Storage::DBI; use DBIx::Class::Storage::DBI::Replicated::Pool; use DBIx::Class::Storage::DBI::Replicated::Balancer; -use DBIx::Class::Storage::DBI::Replicated::Types 'BalancerClassNamePart'; +use DBIx::Class::Storage::DBI::Replicated::Types qw/BalancerClassNamePart DBICSchema DBICStorageDBI/; use MooseX::Types::Moose qw/ClassName HashRef Object/; use Scalar::Util 'reftype'; -use Carp::Clan qw/^DBIx::Class/; +use Hash::Merge; +use List::Util qw/min max reduce/; +use Try::Tiny; +use namespace::clean; use namespace::clean -except => 'meta'; @@ -43,42 +28,56 @@ DBIx::Class::Storage::DBI::Replicated - BETA Replicated database support =head1 SYNOPSIS The Following example shows how to change an existing $schema to a replicated -storage type, add some replicated (readonly) databases, and perform reporting +storage type, add some replicated (read-only) databases, and perform reporting tasks. - ## Change storage_type in your schema class +You should set the 'storage_type attribute to a replicated type. You should +also define your arguments, such as which balancer you want and any arguments +that the Pool object should get. + + my $schema = Schema::Class->clone; $schema->storage_type( ['::DBI::Replicated', {balancer=>'::Random'}] ); - - ## Add some slaves. Basically this is an array of arrayrefs, where each - ## arrayref is database connect information - + $schema->connection(...); + +Next, you need to add in the Replicants. Basically this is an array of +arrayrefs, where each arrayref is database connect information. Think of these +arguments as what you'd pass to the 'normal' $schema->connect method. + $schema->storage->connect_replicants( [$dsn1, $user, $pass, \%opts], [$dsn2, $user, $pass, \%opts], [$dsn3, $user, $pass, \%opts], ); - - ## Now, just use the $schema as normal + +Now, just use the $schema as you normally would. Automatically all reads will +be delegated to the replicants, while writes to the master. + $schema->resultset('Source')->search({name=>'etc'}); - - ## You can force a given query to use a particular storage using the search - ### attribute 'force_pool'. For example: - + +You can force a given query to use a particular storage using the search +attribute 'force_pool'. For example: + my $RS = $schema->resultset('Source')->search(undef, {force_pool=>'master'}); - - ## Now $RS will force everything (both reads and writes) to use whatever was - ## setup as the master storage. 'master' is hardcoded to always point to the - ## Master, but you can also use any Replicant name. Please see: - ## L and the replicants attribute for - ## More. Also see transactions and L for alternative ways - ## to force read traffic to the master. - + +Now $RS will force everything (both reads and writes) to use whatever was setup +as the master storage. 'master' is hardcoded to always point to the Master, +but you can also use any Replicant name. Please see: +L and the replicants attribute for more. + +Also see transactions and L for alternative ways to +force read traffic to the master. In general, you should wrap your statements +in a transaction when you are reading and writing to the same tables at the +same time, since your replicants will often lag a bit behind the master. + +See L for more help and +walkthroughs. + =head1 DESCRIPTION Warning: This class is marked BETA. This has been running a production website using MySQL native replication as its backend and we have some decent test coverage but the code hasn't yet been stressed by a variety of databases. -Individual DB's may have quirks we are not aware of. Please use this in first +Individual DBs may have quirks we are not aware of. Please use this in first development and pass along your experiences/bug fixes. This class implements replicated data store for DBI. Currently you can define @@ -92,27 +91,20 @@ L. Additionally, some methods need to be distributed to all existing storages. This way our storage class is a drop in replacement for L. -Read traffic is spread across the replicants (slaves) occuring to a user +Read traffic is spread across the replicants (slaves) occurring to a user selected algorithm. The default algorithm is random weighted. =head1 NOTES -The consistancy betweeen master and replicants is database specific. The Pool -gives you a method to validate it's replicants, removing and replacing them +The consistency between master and replicants is database specific. The Pool +gives you a method to validate its replicants, removing and replacing them when they fail/pass predefined criteria. Please make careful use of the ways to force a query to run against Master when needed. =head1 REQUIREMENTS -Replicated Storage has additional requirements not currently part of L - - Moose => 0.77 - MooseX::AttributeHelpers => 0.12 - MooseX::Types => 0.10 - namespace::clean => 0.11 - -You will need to install these modules manually via CPAN or make them part of the -Makefile for your distribution. +Replicated Storage has additional requirements not currently part of +L. See L for more details. =head1 ATTRIBUTES @@ -126,14 +118,14 @@ The underlying L object this storage is attaching has 'schema' => ( is=>'rw', - isa=>'DBIx::Class::Schema', + isa=>DBICSchema, weak_ref=>1, required=>1, ); =head2 pool_type -Contains the classname which will instantiate the L object. Defaults +Contains the classname which will instantiate the L object. Defaults to: L. =cut @@ -150,7 +142,7 @@ has 'pool_type' => ( =head2 pool_args Contains a hashref of initialized information to pass to the Balancer object. -See L for available arguments. +See L for available arguments. =cut @@ -183,7 +175,7 @@ has 'balancer_type' => ( =head2 balancer_args Contains a hashref of initialized information to pass to the Balancer object. -See L for available arguments. +See L for available arguments. =cut @@ -197,7 +189,7 @@ has 'balancer_args' => ( =head2 pool -Is a or derived class. This is a +Is a L or derived class. This is a container class for one or more replicated databases. =cut @@ -207,7 +199,7 @@ has 'pool' => ( isa=>'DBIx::Class::Storage::DBI::Replicated::Pool', lazy_build=>1, handles=>[qw/ - connect_replicants + connect_replicants replicants has_replicants /], @@ -215,8 +207,8 @@ has 'pool' => ( =head2 balancer -Is a or derived class. This -is a class that takes a pool () +Is a L or derived class. This +is a class that takes a pool (L) =cut @@ -239,13 +231,13 @@ pool of databases that is allowed to handle write traffic. has 'master' => ( is=> 'ro', - isa=>'DBIx::Class::Storage::DBI', + isa=>DBICStorageDBI, lazy_build=>1, ); =head1 ATTRIBUTES IMPLEMENTING THE DBIx::Storage::DBI INTERFACE -The following methods are delegated all the methods required for the +The following methods are delegated all the methods required for the L interface. =head2 read_handler @@ -262,12 +254,17 @@ has 'read_handler' => ( select select_single columns_info_for - /], + _dbh_columns_info_for + _select + /], ); =head2 write_handler -Defines an object that implements the write side of L. +Defines an object that implements the write side of L, +as well as methods that don't write or read that can be called on only one +storage, methods that return a C<$dbh>, and any methods that don't make sense to +run on a replicant. =cut @@ -275,17 +272,21 @@ has 'write_handler' => ( is=>'ro', isa=>Object, lazy_build=>1, - handles=>[qw/ + handles=>[qw/ on_connect_do - on_disconnect_do + on_disconnect_do + on_connect_call + on_disconnect_call connect_info + _connect_info throw_exception sql_maker sqlt_type create_ddl_dir deployment_statements datetime_parser - datetime_parser_type + datetime_parser_type + build_datetime_parser last_insert_id insert insert_bulk @@ -300,20 +301,109 @@ has 'write_handler' => ( sth deploy with_deferred_fk_checks - + dbh_do reload_row + with_deferred_fk_checks _prep_for_execute - + + backup + is_datatype_numeric + _count_select + _subq_update_delete + svp_rollback + svp_begin + svp_release + relname_to_table_alias + _dbh_last_insert_id + _fix_bind_params + _default_dbi_connect_attributes + _dbi_connect_info + _dbic_connect_attributes + auto_savepoint + _sqlt_version_ok + _query_end + bind_attribute_by_data_type + transaction_depth + _dbh + _select_args + _dbh_execute_array + _sql_maker + _query_start + _sqlt_version_error + _per_row_update_delete + _dbh_begin_work + _dbh_execute_inserts_with_no_binds + _select_args_to_query + _svp_generate_name + _multipk_update_delete + source_bind_attributes + _normalize_connect_info + _parse_connect_do + _dbh_commit + _execute_array + savepoints + _sqlt_minimum_version + _sql_maker_opts + _conn_pid + _conn_tid + _dbh_autocommit + _native_data_type + _get_dbh + sql_maker_class + _dbh_rollback + _adjust_select_args_for_complex_prefetch + _resolve_ident_sources + _resolve_column_info + _prune_unused_joins + _strip_cond_qualifiers + _extract_order_columns + _resolve_aliastypes_from_select_args + _execute + _do_query + _dbh_sth + _dbh_execute + _prefetch_insert_auto_nextvals /], ); +my @unimplemented = qw( + _arm_global_destructor + _preserve_foreign_dbh + _verify_pid + _verify_tid + + get_use_dbms_capability + set_use_dbms_capability + get_dbms_capability + set_dbms_capability + _dbh_details + + sql_limit_dialect + + _inner_join_to_node + _group_over_selection +); + +# the capability framework +# not sure if CMOP->initialize does evil things to DBIC::S::DBI, fix if a problem +push @unimplemented, ( grep + { $_ =~ /^ _ (?: use | supports | determine_supports ) _ /x } + ( Class::MOP::Class->initialize('DBIx::Class::Storage::DBI')->get_all_method_names ) +); + +for my $method (@unimplemented) { + __PACKAGE__->meta->add_method($method, sub { + croak "$method must not be called on ".(blessed shift).' objects'; + }); +} + has _master_connect_info_opts => (is => 'rw', isa => HashRef, default => sub { {} }); =head2 around: connect_info -Preserve master's C options (for merging with replicants.) -Also set any Replicated related options from connect_info, such as +Preserves master's C options (for merging with replicants.) +Also sets any Replicated-related options from connect_info, such as C, C, C and C. =cut @@ -321,10 +411,14 @@ C, C, C and C. around connect_info => sub { my ($next, $self, $info, @extra) = @_; + my $wantarray = wantarray; + + my $merge = Hash::Merge->new('LEFT_PRECEDENT'); + my %opts; for my $arg (@$info) { next unless (reftype($arg)||'') eq 'HASH'; - %opts = (%opts, %$arg); + %opts = %{ $merge->merge($arg, \%opts) }; } delete $opts{dsn}; @@ -332,29 +426,47 @@ around connect_info => sub { $self->pool_type(delete $opts{pool_type}) if $opts{pool_type}; - $self->pool_args({ - %{ $self->pool_args }, - %{ delete $opts{pool_args} || {} } - }); + $self->pool_args( + $merge->merge((delete $opts{pool_args} || {}), $self->pool_args) + ); - $self->pool($self->_build_pool); + ## Since we possibly changed the pool_args, we need to clear the current + ## pool object so that next time it is used it will be rebuilt. + $self->clear_pool; } if (@opts{qw/balancer_type balancer_args/}) { $self->balancer_type(delete $opts{balancer_type}) if $opts{balancer_type}; - $self->balancer_args({ - %{ $self->balancer_args }, - %{ delete $opts{balancer_args} || {} } - }); + $self->balancer_args( + $merge->merge((delete $opts{balancer_args} || {}), $self->balancer_args) + ); - $self->balancer($self->_build_balancer); + $self->balancer($self->_build_balancer) + if $self->balancer; } $self->_master_connect_info_opts(\%opts); - $self->$next($info, @extra); + my (@res, $res); + if ($wantarray) { + @res = $self->$next($info, @extra); + } else { + $res = $self->$next($info, @extra); + } + + # Make sure master is blessed into the correct class and apply role to it. + my $master = $self->master; + $master->_determine_driver; + Moose::Meta::Class->initialize(ref $master); + + DBIx::Class::Storage::DBI::Replicated::WithDSN->meta->apply($master); + + # link pool back to master + $self->pool->master($master); + + $wantarray ? @res : $res; }; =head1 METHODS @@ -363,19 +475,19 @@ This class defines the following methods. =head2 BUILDARGS -L when instantiating it's storage passed itself as the +L when instantiating its storage passed itself as the first argument. So we need to massage the arguments a bit so that all the bits get put into the correct places. =cut sub BUILDARGS { - my ($class, $schema, $storage_type_args, @args) = @_; - + my ($class, $schema, $storage_type_args, @args) = @_; + return { - schema=>$schema, - %$storage_type_args, - @args + schema=>$schema, + %$storage_type_args, + @args } } @@ -388,7 +500,6 @@ Lazy builder for the L attribute. sub _build_master { my $self = shift @_; my $master = DBIx::Class::Storage::DBI->new($self->schema); - DBIx::Class::Storage::DBI::Replicated::WithDSN->meta->apply($master); $master } @@ -413,7 +524,7 @@ the balancer knows which pool it's balancing. sub _build_balancer { my $self = shift @_; $self->create_balancer( - pool=>$self->pool, + pool=>$self->pool, master=>$self->master, %{$self->balancer_args}, ); @@ -455,22 +566,40 @@ around connect_replicants => sub { for my $r (@args) { $r = [ $r ] unless reftype $r eq 'ARRAY'; - croak "coderef replicant connect_info not supported" + $self->throw_exception('coderef replicant connect_info not supported') if ref $r->[0] && reftype $r->[0] eq 'CODE'; # any connect_info options? my $i = 0; $i++ while $i < @$r && (reftype($r->[$i])||'') ne 'HASH'; -# make one if none +# make one if none $r->[$i] = {} unless $r->[$i]; # merge if two hashes - my %opts = map %$_, @$r[$i .. $#{$r}]; + my @hashes = @$r[$i .. $#{$r}]; + + $self->throw_exception('invalid connect_info options') + if (grep { reftype($_) eq 'HASH' } @hashes) != @hashes; + + $self->throw_exception('too many hashrefs in connect_info') + if @hashes > 2; + + my $merge = Hash::Merge->new('LEFT_PRECEDENT'); + my %opts = %{ $merge->merge(reverse @hashes) }; + +# delete them splice @$r, $i+1, ($#{$r} - $i), (); +# make sure master/replicants opts don't clash + my %master_opts = %{ $self->_master_connect_info_opts }; + if (exists $opts{dbh_maker}) { + delete @master_opts{qw/dsn user password/}; + } + delete $master_opts{dbh_maker}; + # merge with master - %opts = (%{ $self->_master_connect_info_opts }, %opts); + %opts = %{ $merge->merge(\%opts, \%master_opts) }; # update $r->[$i] = \%opts; @@ -498,7 +627,7 @@ sub all_storages { =head2 execute_reliably ($coderef, ?@args) Given a coderef, saves the current state of the L, forces it to -use reliable storage (ie sets it to the master), executes a coderef and then +use reliable storage (e.g. sets it to the master), executes a coderef and then restores the original state. Example: @@ -506,7 +635,7 @@ Example: my $reliably = sub { my $name = shift @_; $schema->resultset('User')->create({name=>$name}); - my $user_rs = $schema->resultset('User')->find({name=>$name}); + my $user_rs = $schema->resultset('User')->find({name=>$name}); return $user_rs; }; @@ -519,89 +648,71 @@ inserted something and need to get a resultset including it, etc. sub execute_reliably { my ($self, $coderef, @args) = @_; - + unless( ref $coderef eq 'CODE') { $self->throw_exception('Second argument must be a coderef'); } - + ##Get copy of master storage my $master = $self->master; - + ##Get whatever the current read hander is my $current = $self->read_handler; - + ##Set the read handler to master $self->read_handler($master); - + ## do whatever the caller needs my @result; my $want_array = wantarray; - - eval { + + try { if($want_array) { @result = $coderef->(@args); } elsif(defined $want_array) { ($result[0]) = ($coderef->(@args)); } else { $coderef->(@args); - } + } + } catch { + $self->throw_exception("coderef returned an error: $_"); + } finally { + ##Reset to the original state + $self->read_handler($current); }; - - ##Reset to the original state - $self->read_handler($current); - - ##Exception testing has to come last, otherwise you might leave the - ##read_handler set to master. - - if($@) { - $self->throw_exception("coderef returned an error: $@"); - } else { - return $want_array ? @result : $result[0]; - } + + return $want_array ? @result : $result[0]; } =head2 set_reliable_storage Sets the current $schema to be 'reliable', that is all queries, both read and write are sent to the master - + =cut sub set_reliable_storage { my $self = shift @_; my $schema = $self->schema; my $write_handler = $self->schema->storage->write_handler; - + $schema->storage->read_handler($write_handler); } =head2 set_balanced_storage Sets the current $schema to be use the for all reads, while all -writea are sent to the master only - +writes are sent to the master only + =cut sub set_balanced_storage { my $self = shift @_; my $schema = $self->schema; - my $write_handler = $self->schema->storage->balancer; - - $schema->storage->read_handler($write_handler); -} - -=head2 around: txn_do ($coderef) - -Overload to the txn_do method, which is delegated to whatever the -L is set to. We overload this in order to wrap in inside a -L method. - -=cut + my $balanced_handler = $self->schema->storage->balancer; -around 'txn_do' => sub { - my($txn_do, $self, $coderef, @args) = @_; - $self->execute_reliably(sub {$self->$txn_do($coderef, @args)}); -}; + $schema->storage->read_handler($balanced_handler); +} =head2 connected @@ -640,7 +751,7 @@ sub limit_dialect { foreach my $source ($self->all_storages) { $source->limit_dialect(@_); } - return $self->master->quote_char; + return $self->master->limit_dialect; } =head2 quote_char @@ -695,57 +806,42 @@ sub debug { if(@_) { foreach my $source ($self->all_storages) { $source->debug(@_); - } + } } return $self->master->debug; } =head2 debugobj -set a debug object across all storages +set a debug object =cut sub debugobj { my $self = shift @_; - if(@_) { - foreach my $source ($self->all_storages) { - $source->debugobj(@_); - } - } - return $self->master->debugobj; + return $self->master->debugobj(@_); } =head2 debugfh -set a debugfh object across all storages +set a debugfh object =cut sub debugfh { my $self = shift @_; - if(@_) { - foreach my $source ($self->all_storages) { - $source->debugfh(@_); - } - } - return $self->master->debugfh; + return $self->master->debugfh(@_); } =head2 debugcb -set a debug callback across all storages +set a debug callback =cut sub debugcb { my $self = shift @_; - if(@_) { - foreach my $source ($self->all_storages) { - $source->debugcb(@_); - } - } - return $self->master->debugcb; + return $self->master->debugcb(@_); } =head2 disconnect @@ -775,7 +871,195 @@ sub cursor_class { } $self->master->cursor_class; } - + +=head2 cursor + +set cursor class on all storages, or return master's, alias for L +above. + +=cut + +sub cursor { + my ($self, $cursor_class) = @_; + + if ($cursor_class) { + $_->cursor($cursor_class) for $self->all_storages; + } + $self->master->cursor; +} + +=head2 unsafe + +sets the L option on all storages or returns +master's current setting + +=cut + +sub unsafe { + my $self = shift; + + if (@_) { + $_->unsafe(@_) for $self->all_storages; + } + + return $self->master->unsafe; +} + +=head2 disable_sth_caching + +sets the L option on all storages +or returns master's current setting + +=cut + +sub disable_sth_caching { + my $self = shift; + + if (@_) { + $_->disable_sth_caching(@_) for $self->all_storages; + } + + return $self->master->disable_sth_caching; +} + +=head2 lag_behind_master + +returns the highest Replicant L +setting + +=cut + +sub lag_behind_master { + my $self = shift; + + return max map $_->lag_behind_master, $self->replicants; +} + +=head2 is_replicating + +returns true if all replicants return true for +L + +=cut + +sub is_replicating { + my $self = shift; + + return (grep $_->is_replicating, $self->replicants) == ($self->replicants); +} + +=head2 connect_call_datetime_setup + +calls L for all storages + +=cut + +sub connect_call_datetime_setup { + my $self = shift; + $_->connect_call_datetime_setup for $self->all_storages; +} + +sub _populate_dbh { + my $self = shift; + $_->_populate_dbh for $self->all_storages; +} + +sub _connect { + my $self = shift; + $_->_connect for $self->all_storages; +} + +sub _rebless { + my $self = shift; + $_->_rebless for $self->all_storages; +} + +sub _determine_driver { + my $self = shift; + $_->_determine_driver for $self->all_storages; +} + +sub _driver_determined { + my $self = shift; + + if (@_) { + $_->_driver_determined(@_) for $self->all_storages; + } + + return $self->master->_driver_determined; +} + +sub _init { + my $self = shift; + + $_->_init for $self->all_storages; +} + +sub _run_connection_actions { + my $self = shift; + + $_->_run_connection_actions for $self->all_storages; +} + +sub _do_connection_actions { + my $self = shift; + + if (@_) { + $_->_do_connection_actions(@_) for $self->all_storages; + } +} + +sub connect_call_do_sql { + my $self = shift; + $_->connect_call_do_sql(@_) for $self->all_storages; +} + +sub disconnect_call_do_sql { + my $self = shift; + $_->disconnect_call_do_sql(@_) for $self->all_storages; +} + +sub _seems_connected { + my $self = shift; + + return min map $_->_seems_connected, $self->all_storages; +} + +sub _ping { + my $self = shift; + + return min map $_->_ping, $self->all_storages; +} + +# not using the normalized_version, because we want to preserve +# version numbers much longer than the conventional xxx.yyyzzz +my $numify_ver = sub { + my $ver = shift; + my @numparts = split /\D+/, $ver; + my $format = '%d.' . (join '', ('%06d') x (@numparts - 1)); + + return sprintf $format, @numparts; +}; +sub _server_info { + my $self = shift; + + if (not $self->_dbh_details->{info}) { + $self->_dbh_details->{info} = ( + reduce { $a->[0] < $b->[0] ? $a : $b } + map [ $numify_ver->($_->{dbms_version}), $_ ], + map $_->_server_info, $self->all_storages + )->[1]; + } + + return $self->next::method; +} + +sub _get_server_version { + my $self = shift; + + return $self->_server_info->{dbms_version}; +} + =head1 GOTCHAS Due to the fact that replicants can lag behind a master, you must take care to @@ -809,7 +1093,7 @@ using the Schema clone method. my $new_schema = $schema->clone; $new_schema->set_reliable_storage; - + ## $new_schema will use only the Master storage for all reads/writes while ## the $schema object will use replicated storage.