1 package DBIx::Class::DeploymentHandler::DeployMethod::SQL::Translator;
4 # ABSTRACT: Manage your SQL and Perl migrations in nicely laid out directories
7 use Carp qw( carp croak );
8 use DBIx::Class::DeploymentHandler::Logger;
9 use Log::Contextual qw(:log :dlog), -package_logger =>
10 DBIx::Class::DeploymentHandler::Logger->new({
11 env_prefix => 'DBICDH'
17 require SQL::Translator::Diff;
19 require DBIx::Class::Storage; # loaded for type constraint
20 use DBIx::Class::DeploymentHandler::Types;
22 use File::Path 'mkpath';
23 use File::Spec::Functions;
25 with 'DBIx::Class::DeploymentHandler::HandlesDeploy';
27 with 'DBIx::Class::DeploymentHandler::WithApplicatorDumple' => {
28 interface_role => 'DBIx::Class::DeploymentHandler::HandlesProvideSchema',
29 class_name => 'DBIx::Class::DeploymentHandler::ProvideSchema::SchemaLoader',
30 delegate_name => 'schema_provider',
31 attributes_to_assume => ['schema'],
32 attributes_to_copy => [qw( schema )],
41 has force_overwrite => (
48 isa => 'DBIx::Class::Schema',
54 isa => 'DBIx::Class::Storage',
61 my $s = $self->schema->storage;
62 $s->_determine_driver;
66 has sql_translator_args => (
69 default => sub { {} },
71 has script_directory => (
80 isa => 'DBIx::Class::DeploymentHandler::Databases',
82 default => sub { [qw( MySQL SQLite PostgreSQL )] },
91 has schema_version => (
97 # this will probably never get called as the DBICDH
98 # will be passing down a schema_version normally, which
99 # is built the same way, but we leave this in place
100 sub _build_schema_version {
102 $self->schema->schema_version
105 sub __ddl_consume_with_prefix {
106 my ($self, $type, $versions, $prefix) = @_;
107 my $base_dir = $self->script_directory;
109 my $main = catfile( $base_dir, $type );
111 catfile( $base_dir, '_common', $prefix, join q(-), @{$versions} );
114 catfile( $base_dir, '_common', $prefix, '_any' );
118 $dir = catfile($main, $prefix, join q(-), @{$versions})
120 if ($self->ignore_ddl) {
123 croak "$main does not exist; please write/generate some SQL"
126 my $dir_any = catfile($main, $prefix, '_any');
130 opendir my($dh), $dir;
132 map { $_ => "$dir/$_" }
133 grep { /\.(?:sql|pl|sql-\w+)$/ && -f "$dir/$_" }
137 die $_ unless $self->ignore_ddl;
139 for my $dirname (grep { -d $_ } $common, $common_any, $dir_any) {
140 opendir my($dh), $dirname;
141 for my $filename (grep { /\.(?:sql|pl)$/ && -f catfile($dirname,$_) } readdir $dh) {
142 unless ($files{$filename}) {
143 $files{$filename} = catfile($dirname,$filename);
149 return [@files{sort keys %files}]
152 sub _ddl_initialize_consume_filenames {
153 my ($self, $type, $version) = @_;
154 $self->__ddl_consume_with_prefix($type, [ $version ], 'initialize')
157 sub _ddl_schema_consume_filenames {
158 my ($self, $type, $version) = @_;
159 $self->__ddl_consume_with_prefix($type, [ $version ], 'deploy')
162 sub _ddl_protoschema_deploy_consume_filenames {
163 my ($self, $version) = @_;
164 my $base_dir = $self->script_directory;
166 my $dir = catfile( $base_dir, '_source', 'deploy', $version);
167 return [] unless -d $dir;
169 opendir my($dh), $dir;
170 my %files = map { $_ => "$dir/$_" } grep { /\.yml$/ && -f "$dir/$_" } readdir $dh;
173 return [@files{sort keys %files}]
176 sub _ddl_protoschema_upgrade_consume_filenames {
177 my ($self, $versions) = @_;
178 my $base_dir = $self->script_directory;
180 my $dir = catfile( $base_dir, '_preprocess_schema', 'upgrade', join q(-), @{$versions});
182 return [] unless -d $dir;
184 opendir my($dh), $dir;
185 my %files = map { $_ => "$dir/$_" } grep { /\.pl$/ && -f "$dir/$_" } readdir $dh;
188 return [@files{sort keys %files}]
191 sub _ddl_protoschema_downgrade_consume_filenames {
192 my ($self, $versions) = @_;
193 my $base_dir = $self->script_directory;
195 my $dir = catfile( $base_dir, '_preprocess_schema', 'downgrade', join q(-), @{$versions});
197 return [] unless -d $dir;
199 opendir my($dh), $dir;
200 my %files = map { $_ => "$dir/$_" } grep { /\.pl$/ && -f "$dir/$_" } readdir $dh;
203 return [@files{sort keys %files}]
206 sub _ddl_protoschema_produce_filename {
207 my ($self, $version) = @_;
208 my $dirname = catfile( $self->script_directory, '_source', 'deploy', $version );
209 mkpath($dirname) unless -d $dirname;
211 return catfile( $dirname, '001-auto.yml' );
214 sub _ddl_schema_produce_filename {
215 my ($self, $type, $version) = @_;
216 my $dirname = catfile( $self->script_directory, $type, 'deploy', $version );
217 mkpath($dirname) unless -d $dirname;
219 return catfile( $dirname, '001-auto.sql' );
222 sub _ddl_schema_upgrade_consume_filenames {
223 my ($self, $type, $versions) = @_;
224 $self->__ddl_consume_with_prefix($type, $versions, 'upgrade')
227 sub _ddl_schema_downgrade_consume_filenames {
228 my ($self, $type, $versions) = @_;
229 $self->__ddl_consume_with_prefix($type, $versions, 'downgrade')
232 sub _ddl_schema_upgrade_produce_filename {
233 my ($self, $type, $versions) = @_;
234 my $dir = $self->script_directory;
236 my $dirname = catfile( $dir, $type, 'upgrade', join q(-), @{$versions});
237 mkpath($dirname) unless -d $dirname;
239 return catfile( $dirname, '001-auto.sql' );
242 sub _ddl_schema_downgrade_produce_filename {
243 my ($self, $type, $versions, $dir) = @_;
244 my $dirname = catfile( $dir, $type, 'downgrade', join q(-), @{$versions} );
245 mkpath($dirname) unless -d $dirname;
247 return catfile( $dirname, '001-auto.sql');
251 my ($self, $sql) = @_;
252 my $storage = $self->storage;
254 $sql = [ _split_sql_chunk( @$sql ) ];
256 Dlog_trace { "Running SQL $_" } $sql;
257 foreach my $line (@{$sql}) {
258 $storage->_query_start($line);
259 # the whole reason we do this is so that we can see the line that was run
261 $storage->dbh_do (sub { $_[1]->do($line) });
264 die "$_ (running line '$line')"
266 $storage->_query_end($line);
268 return join "\n", @$sql
271 # split a chunk o' SQL into statements
272 sub _split_sql_chunk {
273 my @sql = map { split /;\n/, $_ } @_;
277 s/^(?:BEGIN|BEGIN TRANSACTION|COMMIT).*//mgi;
296 my ($self, $filename) = @_;
297 log_debug { "Running SQL from $filename" };
298 return $self->_run_sql_array($self->_read_sql_file($filename));
302 my ($self, $filename, $versions) = @_;
303 log_debug { "Running Perl from $filename" };
304 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
306 no warnings 'redefine';
307 my $fn = eval "$filedata";
309 Dlog_trace { "Running Perl $_" } $fn;
312 croak "$filename failed to compile: $@";
313 } elsif (ref $fn eq 'CODE') {
314 $fn->($self->migration_schema, $versions)
316 croak "$filename should define an anonymouse sub that takes a schema but it didn't!";
320 sub _run_sql_and_perl {
321 my ($self, $filenames, $sql_to_run, $versions) = @_;
322 my @files = @{$filenames};
323 my $guard = $self->schema->txn_scope_guard if $self->txn_wrap;
325 $self->_run_sql_array($sql_to_run) if $self->ignore_ddl;
327 my $sql = ($sql_to_run)?join ";\n", @$sql_to_run:'';
329 for my $filename (@files) {
330 if ($self->ignore_ddl && $filename =~ /^[^_]*-auto.*\.sql$/) {
332 } elsif ($filename =~ /\.sql$/) {
333 $sql .= $self->_run_sql($filename)
334 } elsif ( $filename =~ /\.pl$/ ) {
335 $self->_run_perl($filename, $versions)
337 croak "A file ($filename) got to deploy that wasn't sql or perl!";
341 $guard->commit if $self->txn_wrap;
348 my $version = (shift @_ || {})->{version} || $self->schema_version;
349 log_info { "deploying version $version" };
350 my $sqlt_type = $self->storage->sqlt_type;
352 if ($self->ignore_ddl) {
353 $sql = $self->_sql_from_yaml({},
354 '_ddl_protoschema_deploy_consume_filenames', $sqlt_type
357 return $self->_run_sql_and_perl($self->_ddl_schema_consume_filenames(
360 ), $sql, [$version]);
366 my $version = $args->{version} || $self->schema_version;
367 log_info { "initializing version $version" };
368 my $storage_type = $args->{storage_type} || $self->storage->sqlt_type;
370 my @files = @{$self->_ddl_initialize_consume_filenames(
375 for my $filename (@files) {
376 # We ignore sql for now (till I figure out what to do with it)
377 if ( $filename =~ /^(.+)\.pl$/ ) {
378 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
380 no warnings 'redefine';
381 my $fn = eval "$filedata";
385 croak "$filename failed to compile: $@";
386 } elsif (ref $fn eq 'CODE') {
389 croak "$filename should define an anonymous sub but it didn't!";
392 croak "A file ($filename) got to initialize_scripts that wasn't sql or perl!";
397 sub _sqldiff_from_yaml {
398 my ($self, $from_version, $to_version, $db, $direction) = @_;
399 my $dir = $self->script_directory;
402 ignore_constraint_names => 1,
403 ignore_index_names => 1,
404 %{$self->sql_translator_args}
409 my $prefilename = $self->_ddl_protoschema_produce_filename($from_version, $dir);
411 # should probably be a croak
412 carp("No previous schema file found ($prefilename)")
413 unless -e $prefilename;
415 my $t = SQL::Translator->new({
419 parser => 'SQL::Translator::Parser::YAML',
422 my $out = $t->translate( $prefilename )
425 $source_schema = $t->schema;
427 $source_schema->name( $prefilename )
428 unless $source_schema->name;
433 my $filename = $self->_ddl_protoschema_produce_filename($to_version, $dir);
435 # should probably be a croak
436 carp("No next schema file found ($filename)")
439 my $t = SQL::Translator->new({
443 parser => 'SQL::Translator::Parser::YAML',
446 my $out = $t->translate( $filename )
449 $dest_schema = $t->schema;
451 $dest_schema->name( $filename )
452 unless $dest_schema->name;
455 my $transform_files_method = "_ddl_protoschema_${direction}_consume_filenames";
456 my $transforms = $self->_coderefs_per_files(
457 $self->$transform_files_method([$from_version, $to_version])
459 $_->($source_schema, $dest_schema) for @$transforms;
461 return [SQL::Translator::Diff::schema_diff(
469 my ($self, $sqltargs, $from_file, $db) = @_;
470 my $schema = $self->schema;
471 my $version = $self->schema_version;
475 my $actual_file = $self->$from_file($version);
476 for my $yaml_filename (@{
477 DlogS_trace { "generating SQL from Serialized SQL Files: $_" }
478 (ref $actual_file?$actual_file:[$actual_file])
480 my $sqlt = SQL::Translator->new({
482 parser => 'SQL::Translator::Parser::YAML',
487 push @sql, $sqlt->translate($yaml_filename);
489 carp("Failed to translate to $db, skipping. (" . $sqlt->error . ")");
496 sub _prepare_install {
498 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
499 my $from_file = shift;
501 my $dir = $self->script_directory;
502 my $databases = $self->databases;
503 my $version = $self->schema_version;
505 foreach my $db (@$databases) {
506 my $sql = $self->_sql_from_yaml($sqltargs, $from_file, $db ) or next;
508 my $filename = $self->$to_file($db, $version, $dir);
510 if ($self->force_overwrite) {
511 carp "Overwriting existing DDL file - $filename";
514 die "Cannot overwrite '$filename', either enable force_overwrite or delete it"
517 open my $file, q(>), $filename;
518 print {$file} join ";\n", @$sql;
523 sub _resultsource_install_filename {
524 my ($self, $source_name) = @_;
526 my ($self, $type, $version) = @_;
527 my $dirname = catfile( $self->script_directory, $type, 'deploy', $version );
528 mkpath($dirname) unless -d $dirname;
530 return catfile( $dirname, "001-auto-$source_name.sql" );
534 sub _resultsource_protoschema_filename {
535 my ($self, $source_name) = @_;
537 my ($self, $version) = @_;
538 my $dirname = catfile( $self->script_directory, '_source', 'deploy', $version );
539 mkpath($dirname) unless -d $dirname;
541 return catfile( $dirname, "001-auto-$source_name.yml" );
545 sub install_resultsource {
546 my ($self, $args) = @_;
547 my $source = $args->{result_source}
548 or die 'result_source must be passed to install_resultsource';
549 my $version = $args->{version}
550 or die 'version must be passed to install_resultsource';
551 log_info { 'installing_resultsource ' . $source->source_name . ", version $version" };
552 my $rs_install_file =
553 $self->_resultsource_install_filename($source->source_name);
556 $self->$rs_install_file(
557 $self->storage->sqlt_type,
561 $self->_run_sql_and_perl($files, '', [$version]);
564 sub prepare_resultsource_install {
566 my $source = (shift @_)->{result_source};
567 log_info { 'preparing install for resultsource ' . $source->source_name };
569 my $install_filename = $self->_resultsource_install_filename($source->source_name);
570 my $proto_filename = $self->_resultsource_protoschema_filename($source->source_name);
571 $self->prepare_protoschema({
572 parser_args => { sources => [$source->source_name], }
574 $self->_prepare_install({}, $proto_filename, $install_filename);
578 log_info { 'preparing deploy' };
580 $self->prepare_protoschema({
581 # Exclude __VERSION so that it gets installed separately
582 parser_args => { sources => [grep { $_ ne '__VERSION' } $self->schema->sources], }
583 }, '_ddl_protoschema_produce_filename');
584 $self->_prepare_install({}, '_ddl_protoschema_produce_filename', '_ddl_schema_produce_filename');
587 sub prepare_upgrade {
588 my ($self, $args) = @_;
590 "preparing upgrade from $args->{from_version} to $args->{to_version}"
592 $self->_prepare_changegrade(
593 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'upgrade'
597 sub prepare_downgrade {
598 my ($self, $args) = @_;
600 "preparing downgrade from $args->{from_version} to $args->{to_version}"
602 $self->_prepare_changegrade(
603 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'downgrade'
607 sub _coderefs_per_files {
608 my ($self, $files) = @_;
609 no warnings 'redefine';
610 [map eval do { local( @ARGV, $/ ) = $_; <> }, @$files]
613 sub _prepare_changegrade {
614 my ($self, $from_version, $to_version, $version_set, $direction) = @_;
615 my $schema = $self->schema;
616 my $databases = $self->databases;
617 my $dir = $self->script_directory;
619 my $schema_version = $self->schema_version;
620 my $diff_file_method = "_ddl_schema_${direction}_produce_filename";
621 foreach my $db (@$databases) {
622 my $diff_file = $self->$diff_file_method($db, $version_set, $dir );
624 if ($self->force_overwrite) {
625 carp("Overwriting existing $direction-diff file - $diff_file");
628 die "Cannot overwrite '$diff_file', either enable force_overwrite or delete it"
632 open my $file, q(>), $diff_file;
633 print {$file} join ";\n", @{$self->_sqldiff_from_yaml($from_version, $to_version, $db, $direction)};
639 my ($self, $file) = @_;
642 local $/ = undef; #sluuuuuurp
644 open my $fh, '<', $file;
645 return [ _split_sql_chunk( <$fh> ) ];
648 sub downgrade_single_step {
650 my $version_set = (shift @_)->{version_set};
651 Dlog_info { "downgrade_single_step'ing $_" } $version_set;
653 my $sqlt_type = $self->storage->sqlt_type;
655 if ($self->ignore_ddl) {
656 $sql_to_run = $self->_sqldiff_from_yaml(
657 $version_set->[0], $version_set->[1], $sqlt_type, 'downgrade',
660 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_downgrade_consume_filenames(
663 ), $sql_to_run, $version_set);
668 sub upgrade_single_step {
670 my $version_set = (shift @_)->{version_set};
671 Dlog_info { "upgrade_single_step'ing $_" } $version_set;
673 my $sqlt_type = $self->storage->sqlt_type;
675 if ($self->ignore_ddl) {
676 $sql_to_run = $self->_sqldiff_from_yaml(
677 $version_set->[0], $version_set->[1], $sqlt_type, 'upgrade',
680 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_upgrade_consume_filenames(
683 ), $sql_to_run, $version_set);
687 sub prepare_protoschema {
689 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
692 = $self->$to_file($self->schema_version);
694 # we do this because the code that uses this sets parser args,
695 # so we just need to merge in the package
696 $sqltargs->{parser_args}{package} = $self->schema;
697 my $sqlt = SQL::Translator->new({
698 parser => 'SQL::Translator::Parser::DBIx::Class',
699 producer => 'SQL::Translator::Producer::YAML',
703 my $yml = $sqlt->translate;
705 croak("Failed to translate to YAML: " . $sqlt->error)
709 if ($self->force_overwrite) {
710 carp "Overwriting existing DDL-YML file - $filename";
713 die "Cannot overwrite '$filename', either enable force_overwrite or delete it"
717 open my $file, q(>), $filename;
722 __PACKAGE__->meta->make_immutable;
726 # vim: ts=2 sw=2 expandtab
732 This class is the meat of L<DBIx::Class::DeploymentHandler>. It takes care
733 of generating serialized schemata as well as sql files to move from one
734 version of a schema to the rest. One of the hallmark features of this class
735 is that it allows for multiple sql files for deploy and upgrade, allowing
736 developers to fine tune deployment. In addition it also allows for perl
737 files to be run at any stage of the process.
739 For basic usage see L<DBIx::Class::DeploymentHandler::HandlesDeploy>. What's
740 documented here is extra fun stuff or private methods.
742 =head1 DIRECTORY LAYOUT
744 Arguably this is the best feature of L<DBIx::Class::DeploymentHandler>.
745 It's spiritually based upon L<DBIx::Migration::Directories>, but has a
746 lot of extensions and modifications, so even if you are familiar with it,
747 please read this. I feel like the best way to describe the layout is with
748 the following example:
774 | | `- 002-remove-customers.pl
777 | | `- 002-generate-customers.pl
779 | `- 999-bump-action.pl
786 | |- 001-create_database.pl
787 | `- 002-create_users_and_permissions.pl
795 So basically, the code
799 on an C<SQLite> database that would simply run
800 C<$sql_migration_dir/SQLite/deploy/1/001-auto.sql>. Next,
802 $dm->upgrade_single_step([1,2])
804 would run C<$sql_migration_dir/SQLite/upgrade/1-2/001-auto.sql> followed by
805 C<$sql_migration_dir/_common/upgrade/1-2/002-generate-customers.pl>, and
806 finally punctuated by
807 C<$sql_migration_dir/_common/upgrade/_any/999-bump-action.pl>.
809 C<.pl> files don't have to be in the C<_common> directory, but most of the time
810 they should be, because perl scripts are generally database independent.
812 Note that unlike most steps in the process, C<initialize> will not run SQL, as
813 there may not even be an database at initialize time. It will run perl scripts
814 just like the other steps in the process, but nothing is passed to them.
815 Until people have used this more it will remain freeform, but a recommended use
816 of initialize is to have it prompt for username and password, and then call the
817 appropriate C<< CREATE DATABASE >> commands etc.
819 =head2 Directory Specification
821 The following subdirectories are recognized by this DeployMethod:
825 =item C<_source> This directory can contain the following directories:
829 =item C<deploy> This directory merely contains directories named after schema
830 versions, which in turn contain C<yaml> files that are serialized versions
831 of the schema at that version. These files are not for editing by hand.
835 =item C<_preprocess_schema> This directory can contain the following
840 =item C<downgrade> This directory merely contains directories named after
841 migrations, which are of the form C<$from_version-$to_version>. Inside of
842 these directories you may put Perl scripts which are to return a subref
843 that takes the arguments C<< $from_schema, $to_schema >>, which are
844 L<SQL::Translator::Schema> objects.
846 =item C<upgrade> This directory merely contains directories named after
847 migrations, which are of the form C<$from_version-$to_version>. Inside of
848 these directories you may put Perl scripts which are to return a subref
849 that takes the arguments C<< $from_schema, $to_schema >>, which are
850 L<SQL::Translator::Schema> objects.
854 =item C<$storage_type> This is a set of scripts that gets run depending on what
855 your storage type is. If you are not sure what your storage type is, take a
856 look at the producers listed for L<SQL::Translator>. Also note, C<_common>
857 is a special case. C<_common> will get merged into whatever other files you
858 already have. This directory can contain the following directories itself:
862 =item C<initialize> Gets run before the C<deploy> is C<deploy>ed. Has the
863 same structure as the C<deploy> subdirectory as well; that is, it has a
864 directory for each schema version. Unlike C<deploy>, C<upgrade>, and C<downgrade>
865 though, it can only run C<.pl> files, and the coderef in the perl files get
866 no arguments passed to them.
868 =item C<deploy> Gets run when the schema is C<deploy>ed. Structure is a
869 directory per schema version, and then files are merged with C<_common> and run
870 in filename order. C<.sql> files are merely run, as expected. C<.pl> files are
871 run according to L</PERL SCRIPTS>.
873 =item C<upgrade> Gets run when the schema is C<upgrade>d. Structure is a directory
874 per upgrade step, (for example, C<1-2> for upgrading from version 1 to version
875 2,) and then files are merged with C<_common> and run in filename order.
876 C<.sql> files are merely run, as expected. C<.pl> files are run according
879 =item C<downgrade> Gets run when the schema is C<downgrade>d. Structure is a directory
880 per downgrade step, (for example, C<2-1> for downgrading from version 2 to version
881 1,) and then files are merged with C<_common> and run in filename order.
882 C<.sql> files are merely run, as expected. C<.pl> files are run according
890 Note that there can be an C<_any> in the place of any of the versions (like
891 C<1-2> or C<1>), which means those scripts will be run B<every> time. So if
892 you have an C<_any> in C<_common/upgrade>, that script will get run for every
897 A perl script for this tool is very simple. It merely needs to contain an
898 anonymous sub that takes a L<DBIx::Class::Schema> and the version set as it's
901 A very basic perl script might look like:
911 # [1] for deploy, [1,2] for upgrade or downgrade, probably used with _any
912 my $versions = shift;
914 $schema->resultset('Users')->create({
922 This attribute will, when set to true (default is false), cause the DM to use
923 L<SQL::Translator> to use the C<_source>'s serialized SQL::Translator::Schema
924 instead of any pregenerated SQL. If you have a development server this is
925 probably the best plan of action as you will not be putting as many generated
926 files in your version control. Goes well with with C<databases> of C<[]>.
928 =attr force_overwrite
930 When this attribute is true generated files will be overwritten when the
931 methods which create such files are run again. The default is false, in which
932 case the program will die with a message saying which file needs to be deleted.
936 The L<DBIx::Class::Schema> (B<required>) that is used to talk to the database
937 and generate the DDL.
941 The L<DBIx::Class::Storage> that is I<actually> used to talk to the database
942 and generate the DDL. This is automatically created with L</_build_storage>.
944 =attr sql_translator_args
946 The arguments that get passed to L<SQL::Translator> when it's used.
948 =attr script_directory
950 The directory (default C<'sql'>) that scripts are stored in
954 The types of databases (default C<< [qw( MySQL SQLite PostgreSQL )] >>) to
959 Set to true (which is the default) to wrap all upgrades and deploys in a single
964 The version the schema on your harddrive is at. Defaults to
965 C<< $self->schema->schema_version >>.