X-Git-Url: http://git.shadowcat.co.uk/gitweb/gitweb.cgi?p=dbsrgits%2FDBIx-Class.git;a=blobdiff_plain;f=lib%2FDBIx%2FClass%2FStorage%2FDBI%2FReplicated.pm;h=447747514dd38272da3c17fa1c9e635944407822;hp=6302bf1ebac360671bb02075f28ab3ebcd8e66c3;hb=64ae166780d0cb2b9577e506da9b9b240c146d20;hpb=212cc5c25c31b2ec3ff4b4e20283321617db79e6 diff --git a/lib/DBIx/Class/Storage/DBI/Replicated.pm b/lib/DBIx/Class/Storage/DBI/Replicated.pm index 6302bf1..4477475 100644 --- a/lib/DBIx/Class/Storage/DBI/Replicated.pm +++ b/lib/DBIx/Class/Storage/DBI/Replicated.pm @@ -2,39 +2,22 @@ package DBIx::Class::Storage::DBI::Replicated; BEGIN { use Carp::Clan qw/^DBIx::Class/; - - ## Modules required for Replication support not required for general DBIC - ## use, so we explicitly test for these. - - my %replication_required = ( - 'Moose' => '0.77', - 'MooseX::AttributeHelpers' => '0.12', - 'MooseX::Types' => '0.10', - 'namespace::clean' => '0.11', - 'Hash::Merge' => '0.11' - ); - - my @didnt_load; - - for my $module (keys %replication_required) { - eval "use $module $replication_required{$module}"; - push @didnt_load, "$module $replication_required{$module}" - if $@; - } - - croak("@{[ join ', ', @didnt_load ]} are missing and are required for Replication") - if @didnt_load; + use DBIx::Class; + croak('The following modules are required for Replication ' . DBIx::Class::Optional::Dependencies->req_missing_for ('replicated') ) + unless DBIx::Class::Optional::Dependencies->req_ok_for ('replicated'); } use Moose; use DBIx::Class::Storage::DBI; use DBIx::Class::Storage::DBI::Replicated::Pool; use DBIx::Class::Storage::DBI::Replicated::Balancer; -use DBIx::Class::Storage::DBI::Replicated::Types 'BalancerClassNamePart'; +use DBIx::Class::Storage::DBI::Replicated::Types qw/BalancerClassNamePart DBICSchema DBICStorageDBI/; use MooseX::Types::Moose qw/ClassName HashRef Object/; use Scalar::Util 'reftype'; -use Carp::Clan qw/^DBIx::Class/; -use Hash::Merge 'merge'; +use Hash::Merge; +use List::Util qw/min max reduce/; +use Try::Tiny; +use namespace::clean; use namespace::clean -except => 'meta'; @@ -45,37 +28,39 @@ DBIx::Class::Storage::DBI::Replicated - BETA Replicated database support =head1 SYNOPSIS The Following example shows how to change an existing $schema to a replicated -storage type, add some replicated (readonly) databases, and perform reporting +storage type, add some replicated (read-only) databases, and perform reporting tasks. You should set the 'storage_type attribute to a replicated type. You should -also defined you arguments, such as which balancer you want and any arguments +also define your arguments, such as which balancer you want and any arguments that the Pool object should get. + my $schema = Schema::Class->clone; $schema->storage_type( ['::DBI::Replicated', {balancer=>'::Random'}] ); - -Next, you need to add in the Replicants. Basically this is an array of + $schema->connection(...); + +Next, you need to add in the Replicants. Basically this is an array of arrayrefs, where each arrayref is database connect information. Think of these arguments as what you'd pass to the 'normal' $schema->connect method. - + $schema->storage->connect_replicants( [$dsn1, $user, $pass, \%opts], [$dsn2, $user, $pass, \%opts], [$dsn3, $user, $pass, \%opts], ); - + Now, just use the $schema as you normally would. Automatically all reads will be delegated to the replicants, while writes to the master. $schema->resultset('Source')->search({name=>'etc'}); - + You can force a given query to use a particular storage using the search attribute 'force_pool'. For example: - + my $RS = $schema->resultset('Source')->search(undef, {force_pool=>'master'}); Now $RS will force everything (both reads and writes) to use whatever was setup -as the master storage. 'master' is hardcoded to always point to the Master, +as the master storage. 'master' is hardcoded to always point to the Master, but you can also use any Replicant name. Please see: L and the replicants attribute for more. @@ -86,13 +71,13 @@ same time, since your replicants will often lag a bit behind the master. See L for more help and walkthroughs. - + =head1 DESCRIPTION Warning: This class is marked BETA. This has been running a production website using MySQL native replication as its backend and we have some decent test coverage but the code hasn't yet been stressed by a variety of databases. -Individual DB's may have quirks we are not aware of. Please use this in first +Individual DBs may have quirks we are not aware of. Please use this in first development and pass along your experiences/bug fixes. This class implements replicated data store for DBI. Currently you can define @@ -106,28 +91,20 @@ L. Additionally, some methods need to be distributed to all existing storages. This way our storage class is a drop in replacement for L. -Read traffic is spread across the replicants (slaves) occuring to a user +Read traffic is spread across the replicants (slaves) occurring to a user selected algorithm. The default algorithm is random weighted. =head1 NOTES -The consistancy betweeen master and replicants is database specific. The Pool -gives you a method to validate it's replicants, removing and replacing them +The consistency between master and replicants is database specific. The Pool +gives you a method to validate its replicants, removing and replacing them when they fail/pass predefined criteria. Please make careful use of the ways to force a query to run against Master when needed. =head1 REQUIREMENTS -Replicated Storage has additional requirements not currently part of L - - Moose => 0.77 - MooseX::AttributeHelpers => 0.12 - MooseX::Types => 0.10 - namespace::clean => 0.11 - Hash::Merge => 0.11 - -You will need to install these modules manually via CPAN or make them part of the -Makefile for your distribution. +Replicated Storage has additional requirements not currently part of +L. See L for more details. =head1 ATTRIBUTES @@ -141,14 +118,14 @@ The underlying L object this storage is attaching has 'schema' => ( is=>'rw', - isa=>'DBIx::Class::Schema', + isa=>DBICSchema, weak_ref=>1, required=>1, ); =head2 pool_type -Contains the classname which will instantiate the L object. Defaults +Contains the classname which will instantiate the L object. Defaults to: L. =cut @@ -212,7 +189,7 @@ has 'balancer_args' => ( =head2 pool -Is a or derived class. This is a +Is a L or derived class. This is a container class for one or more replicated databases. =cut @@ -222,7 +199,7 @@ has 'pool' => ( isa=>'DBIx::Class::Storage::DBI::Replicated::Pool', lazy_build=>1, handles=>[qw/ - connect_replicants + connect_replicants replicants has_replicants /], @@ -230,8 +207,8 @@ has 'pool' => ( =head2 balancer -Is a or derived class. This -is a class that takes a pool () +Is a L or derived class. This +is a class that takes a pool (L) =cut @@ -254,13 +231,13 @@ pool of databases that is allowed to handle write traffic. has 'master' => ( is=> 'ro', - isa=>'DBIx::Class::Storage::DBI', + isa=>DBICStorageDBI, lazy_build=>1, ); =head1 ATTRIBUTES IMPLEMENTING THE DBIx::Storage::DBI INTERFACE -The following methods are delegated all the methods required for the +The following methods are delegated all the methods required for the L interface. =head2 read_handler @@ -277,12 +254,17 @@ has 'read_handler' => ( select select_single columns_info_for - /], + _dbh_columns_info_for + _select + /], ); =head2 write_handler -Defines an object that implements the write side of L. +Defines an object that implements the write side of L, +as well as methods that don't write or read that can be called on only one +storage, methods that return a C<$dbh>, and any methods that don't make sense to +run on a replicant. =cut @@ -290,18 +272,21 @@ has 'write_handler' => ( is=>'ro', isa=>Object, lazy_build=>1, - handles=>[qw/ + handles=>[qw/ on_connect_do - on_disconnect_do + on_disconnect_do + on_connect_call + on_disconnect_call connect_info + _connect_info throw_exception sql_maker sqlt_type create_ddl_dir deployment_statements datetime_parser - datetime_parser_type - build_datetime_parser + datetime_parser_type + build_datetime_parser last_insert_id insert insert_bulk @@ -316,29 +301,109 @@ has 'write_handler' => ( sth deploy with_deferred_fk_checks - dbh_do + dbh_do reload_row - with_deferred_fk_checks + with_deferred_fk_checks _prep_for_execute - backup - is_datatype_numeric - _count_select - _subq_count_select - _subq_update_delete - svp_rollback - svp_begin - svp_release + backup + is_datatype_numeric + _count_select + _subq_update_delete + svp_rollback + svp_begin + svp_release + relname_to_table_alias + _dbh_last_insert_id + _fix_bind_params + _default_dbi_connect_attributes + _dbi_connect_info + _dbic_connect_attributes + auto_savepoint + _sqlt_version_ok + _query_end + bind_attribute_by_data_type + transaction_depth + _dbh + _select_args + _dbh_execute_array + _sql_maker + _query_start + _sqlt_version_error + _per_row_update_delete + _dbh_begin_work + _dbh_execute_inserts_with_no_binds + _select_args_to_query + _svp_generate_name + _multipk_update_delete + source_bind_attributes + _normalize_connect_info + _parse_connect_do + _dbh_commit + _execute_array + savepoints + _sqlt_minimum_version + _sql_maker_opts + _conn_pid + _conn_tid + _dbh_autocommit + _native_data_type + _get_dbh + sql_maker_class + _dbh_rollback + _adjust_select_args_for_complex_prefetch + _resolve_ident_sources + _resolve_column_info + _prune_unused_joins + _strip_cond_qualifiers + _extract_order_columns + _resolve_aliastypes_from_select_args + _execute + _do_query + _dbh_sth + _dbh_execute + _prefetch_insert_auto_nextvals /], ); +my @unimplemented = qw( + _arm_global_destructor + _preserve_foreign_dbh + _verify_pid + _verify_tid + + get_use_dbms_capability + set_use_dbms_capability + get_dbms_capability + set_dbms_capability + _dbh_details + + sql_limit_dialect + + _inner_join_to_node + _group_over_selection +); + +# the capability framework +# not sure if CMOP->initialize does evil things to DBIC::S::DBI, fix if a problem +push @unimplemented, ( grep + { $_ =~ /^ _ (?: use | supports | determine_supports ) _ /x } + ( Class::MOP::Class->initialize('DBIx::Class::Storage::DBI')->get_all_method_names ) +); + +for my $method (@unimplemented) { + __PACKAGE__->meta->add_method($method, sub { + croak "$method must not be called on ".(blessed shift).' objects'; + }); +} + has _master_connect_info_opts => (is => 'rw', isa => HashRef, default => sub { {} }); =head2 around: connect_info -Preserve master's C options (for merging with replicants.) -Also set any Replicated related options from connect_info, such as +Preserves master's C options (for merging with replicants.) +Also sets any Replicated-related options from connect_info, such as C, C, C and C. =cut @@ -348,10 +413,12 @@ around connect_info => sub { my $wantarray = wantarray; + my $merge = Hash::Merge->new('LEFT_PRECEDENT'); + my %opts; for my $arg (@$info) { next unless (reftype($arg)||'') eq 'HASH'; - %opts = %{ merge($arg, \%opts) }; + %opts = %{ $merge->merge($arg, \%opts) }; } delete $opts{dsn}; @@ -360,11 +427,12 @@ around connect_info => sub { if $opts{pool_type}; $self->pool_args( - merge((delete $opts{pool_args} || {}), $self->pool_args) + $merge->merge((delete $opts{pool_args} || {}), $self->pool_args) ); - $self->pool($self->_build_pool) - if $self->pool; + ## Since we possibly changed the pool_args, we need to clear the current + ## pool object so that next time it is used it will be rebuilt. + $self->clear_pool; } if (@opts{qw/balancer_type balancer_args/}) { @@ -372,11 +440,11 @@ around connect_info => sub { if $opts{balancer_type}; $self->balancer_args( - merge((delete $opts{balancer_args} || {}), $self->balancer_args) + $merge->merge((delete $opts{balancer_args} || {}), $self->balancer_args) ); $self->balancer($self->_build_balancer) - if $self->balancer; + if $self->balancer; } $self->_master_connect_info_opts(\%opts); @@ -392,8 +460,12 @@ around connect_info => sub { my $master = $self->master; $master->_determine_driver; Moose::Meta::Class->initialize(ref $master); + DBIx::Class::Storage::DBI::Replicated::WithDSN->meta->apply($master); + # link pool back to master + $self->pool->master($master); + $wantarray ? @res : $res; }; @@ -403,19 +475,19 @@ This class defines the following methods. =head2 BUILDARGS -L when instantiating it's storage passed itself as the +L when instantiating its storage passed itself as the first argument. So we need to massage the arguments a bit so that all the bits get put into the correct places. =cut sub BUILDARGS { - my ($class, $schema, $storage_type_args, @args) = @_; - + my ($class, $schema, $storage_type_args, @args) = @_; + return { - schema=>$schema, - %$storage_type_args, - @args + schema=>$schema, + %$storage_type_args, + @args } } @@ -452,7 +524,7 @@ the balancer knows which pool it's balancing. sub _build_balancer { my $self = shift @_; $self->create_balancer( - pool=>$self->pool, + pool=>$self->pool, master=>$self->master, %{$self->balancer_args}, ); @@ -494,32 +566,40 @@ around connect_replicants => sub { for my $r (@args) { $r = [ $r ] unless reftype $r eq 'ARRAY'; - croak "coderef replicant connect_info not supported" + $self->throw_exception('coderef replicant connect_info not supported') if ref $r->[0] && reftype $r->[0] eq 'CODE'; # any connect_info options? my $i = 0; $i++ while $i < @$r && (reftype($r->[$i])||'') ne 'HASH'; -# make one if none +# make one if none $r->[$i] = {} unless $r->[$i]; # merge if two hashes my @hashes = @$r[$i .. $#{$r}]; - croak "invalid connect_info options" + $self->throw_exception('invalid connect_info options') if (grep { reftype($_) eq 'HASH' } @hashes) != @hashes; - croak "too many hashrefs in connect_info" + $self->throw_exception('too many hashrefs in connect_info') if @hashes > 2; - my %opts = %{ merge(reverse @hashes) }; + my $merge = Hash::Merge->new('LEFT_PRECEDENT'); + my %opts = %{ $merge->merge(reverse @hashes) }; # delete them splice @$r, $i+1, ($#{$r} - $i), (); +# make sure master/replicants opts don't clash + my %master_opts = %{ $self->_master_connect_info_opts }; + if (exists $opts{dbh_maker}) { + delete @master_opts{qw/dsn user password/}; + } + delete $master_opts{dbh_maker}; + # merge with master - %opts = %{ merge(\%opts, $self->_master_connect_info_opts) }; + %opts = %{ $merge->merge(\%opts, \%master_opts) }; # update $r->[$i] = \%opts; @@ -547,7 +627,7 @@ sub all_storages { =head2 execute_reliably ($coderef, ?@args) Given a coderef, saves the current state of the L, forces it to -use reliable storage (ie sets it to the master), executes a coderef and then +use reliable storage (e.g. sets it to the master), executes a coderef and then restores the original state. Example: @@ -555,7 +635,7 @@ Example: my $reliably = sub { my $name = shift @_; $schema->resultset('User')->create({name=>$name}); - my $user_rs = $schema->resultset('User')->find({name=>$name}); + my $user_rs = $schema->resultset('User')->find({name=>$name}); return $user_rs; }; @@ -568,74 +648,69 @@ inserted something and need to get a resultset including it, etc. sub execute_reliably { my ($self, $coderef, @args) = @_; - + unless( ref $coderef eq 'CODE') { $self->throw_exception('Second argument must be a coderef'); } - + ##Get copy of master storage my $master = $self->master; - + ##Get whatever the current read hander is my $current = $self->read_handler; - + ##Set the read handler to master $self->read_handler($master); - + ## do whatever the caller needs my @result; my $want_array = wantarray; - - eval { + + try { if($want_array) { @result = $coderef->(@args); } elsif(defined $want_array) { ($result[0]) = ($coderef->(@args)); } else { $coderef->(@args); - } + } + } catch { + $self->throw_exception("coderef returned an error: $_"); + } finally { + ##Reset to the original state + $self->read_handler($current); }; - - ##Reset to the original state - $self->read_handler($current); - - ##Exception testing has to come last, otherwise you might leave the - ##read_handler set to master. - - if($@) { - $self->throw_exception("coderef returned an error: $@"); - } else { - return $want_array ? @result : $result[0]; - } + + return $want_array ? @result : $result[0]; } =head2 set_reliable_storage Sets the current $schema to be 'reliable', that is all queries, both read and write are sent to the master - + =cut sub set_reliable_storage { my $self = shift @_; my $schema = $self->schema; my $write_handler = $self->schema->storage->write_handler; - + $schema->storage->read_handler($write_handler); } =head2 set_balanced_storage Sets the current $schema to be use the for all reads, while all -writea are sent to the master only - +writes are sent to the master only + =cut sub set_balanced_storage { my $self = shift @_; my $schema = $self->schema; my $balanced_handler = $self->schema->storage->balancer; - + $schema->storage->read_handler($balanced_handler); } @@ -676,7 +751,7 @@ sub limit_dialect { foreach my $source ($self->all_storages) { $source->limit_dialect(@_); } - return $self->master->quote_char; + return $self->master->limit_dialect; } =head2 quote_char @@ -731,57 +806,42 @@ sub debug { if(@_) { foreach my $source ($self->all_storages) { $source->debug(@_); - } + } } return $self->master->debug; } =head2 debugobj -set a debug object across all storages +set a debug object =cut sub debugobj { my $self = shift @_; - if(@_) { - foreach my $source ($self->all_storages) { - $source->debugobj(@_); - } - } - return $self->master->debugobj; + return $self->master->debugobj(@_); } =head2 debugfh -set a debugfh object across all storages +set a debugfh object =cut sub debugfh { my $self = shift @_; - if(@_) { - foreach my $source ($self->all_storages) { - $source->debugfh(@_); - } - } - return $self->master->debugfh; + return $self->master->debugfh(@_); } =head2 debugcb -set a debug callback across all storages +set a debug callback =cut sub debugcb { my $self = shift @_; - if(@_) { - foreach my $source ($self->all_storages) { - $source->debugcb(@_); - } - } - return $self->master->debugcb; + return $self->master->debugcb(@_); } =head2 disconnect @@ -811,7 +871,195 @@ sub cursor_class { } $self->master->cursor_class; } - + +=head2 cursor + +set cursor class on all storages, or return master's, alias for L +above. + +=cut + +sub cursor { + my ($self, $cursor_class) = @_; + + if ($cursor_class) { + $_->cursor($cursor_class) for $self->all_storages; + } + $self->master->cursor; +} + +=head2 unsafe + +sets the L option on all storages or returns +master's current setting + +=cut + +sub unsafe { + my $self = shift; + + if (@_) { + $_->unsafe(@_) for $self->all_storages; + } + + return $self->master->unsafe; +} + +=head2 disable_sth_caching + +sets the L option on all storages +or returns master's current setting + +=cut + +sub disable_sth_caching { + my $self = shift; + + if (@_) { + $_->disable_sth_caching(@_) for $self->all_storages; + } + + return $self->master->disable_sth_caching; +} + +=head2 lag_behind_master + +returns the highest Replicant L +setting + +=cut + +sub lag_behind_master { + my $self = shift; + + return max map $_->lag_behind_master, $self->replicants; +} + +=head2 is_replicating + +returns true if all replicants return true for +L + +=cut + +sub is_replicating { + my $self = shift; + + return (grep $_->is_replicating, $self->replicants) == ($self->replicants); +} + +=head2 connect_call_datetime_setup + +calls L for all storages + +=cut + +sub connect_call_datetime_setup { + my $self = shift; + $_->connect_call_datetime_setup for $self->all_storages; +} + +sub _populate_dbh { + my $self = shift; + $_->_populate_dbh for $self->all_storages; +} + +sub _connect { + my $self = shift; + $_->_connect for $self->all_storages; +} + +sub _rebless { + my $self = shift; + $_->_rebless for $self->all_storages; +} + +sub _determine_driver { + my $self = shift; + $_->_determine_driver for $self->all_storages; +} + +sub _driver_determined { + my $self = shift; + + if (@_) { + $_->_driver_determined(@_) for $self->all_storages; + } + + return $self->master->_driver_determined; +} + +sub _init { + my $self = shift; + + $_->_init for $self->all_storages; +} + +sub _run_connection_actions { + my $self = shift; + + $_->_run_connection_actions for $self->all_storages; +} + +sub _do_connection_actions { + my $self = shift; + + if (@_) { + $_->_do_connection_actions(@_) for $self->all_storages; + } +} + +sub connect_call_do_sql { + my $self = shift; + $_->connect_call_do_sql(@_) for $self->all_storages; +} + +sub disconnect_call_do_sql { + my $self = shift; + $_->disconnect_call_do_sql(@_) for $self->all_storages; +} + +sub _seems_connected { + my $self = shift; + + return min map $_->_seems_connected, $self->all_storages; +} + +sub _ping { + my $self = shift; + + return min map $_->_ping, $self->all_storages; +} + +# not using the normalized_version, because we want to preserve +# version numbers much longer than the conventional xxx.yyyzzz +my $numify_ver = sub { + my $ver = shift; + my @numparts = split /\D+/, $ver; + my $format = '%d.' . (join '', ('%06d') x (@numparts - 1)); + + return sprintf $format, @numparts; +}; +sub _server_info { + my $self = shift; + + if (not $self->_dbh_details->{info}) { + $self->_dbh_details->{info} = ( + reduce { $a->[0] < $b->[0] ? $a : $b } + map [ $numify_ver->($_->{dbms_version}), $_ ], + map $_->_server_info, $self->all_storages + )->[1]; + } + + return $self->next::method; +} + +sub _get_server_version { + my $self = shift; + + return $self->_server_info->{dbms_version}; +} + =head1 GOTCHAS Due to the fact that replicants can lag behind a master, you must take care to @@ -845,7 +1093,7 @@ using the Schema clone method. my $new_schema = $schema->clone; $new_schema->set_reliable_storage; - + ## $new_schema will use only the Master storage for all reads/writes while ## the $schema object will use replicated storage.