1 package DBIx::Class::DeploymentHandler::DeployMethod::SQL::Translator;
4 # ABSTRACT: Manage your SQL and Perl migrations in nicely laid out directories
7 use Carp qw( carp croak );
8 use DBIx::Class::DeploymentHandler::Logger;
9 use Log::Contextual qw(:log :dlog), -package_logger =>
10 DBIx::Class::DeploymentHandler::Logger->new({
11 env_prefix => 'DBICDH'
17 require SQL::Translator::Diff;
19 require DBIx::Class::Storage; # loaded for type constraint
20 use DBIx::Class::DeploymentHandler::Types;
22 use File::Path 'mkpath';
23 use File::Spec::Functions;
25 with 'DBIx::Class::DeploymentHandler::HandlesDeploy';
33 has force_overwrite => (
40 isa => 'DBIx::Class::Schema',
46 isa => 'DBIx::Class::Storage',
53 my $s = $self->schema->storage;
54 $s->_determine_driver;
58 has sql_translator_args => (
61 default => sub { {} },
63 has script_directory => (
72 isa => 'DBIx::Class::DeploymentHandler::Databases',
74 default => sub { [qw( MySQL SQLite PostgreSQL )] },
83 has schema_version => (
89 # this will probably never get called as the DBICDH
90 # will be passing down a schema_version normally, which
91 # is built the same way, but we leave this in place
92 sub _build_schema_version {
94 $self->schema->schema_version
97 sub __ddl_consume_with_prefix {
98 my ($self, $type, $versions, $prefix) = @_;
99 my $base_dir = $self->script_directory;
101 my $main = catfile( $base_dir, $type );
103 catfile( $base_dir, '_common', $prefix, join q(-), @{$versions} );
106 catfile( $base_dir, '_common', $prefix, '_any' );
110 $dir = catfile($main, $prefix, join q(-), @{$versions})
112 if ($self->ignore_ddl) {
115 croak "$main does not exist; please write/generate some SQL"
118 my $dir_any = catfile($main, $prefix, '_any');
122 opendir my($dh), $dir;
124 map { $_ => "$dir/$_" }
125 grep { /\.(?:sql|pl|sql-\w+)$/ && -f "$dir/$_" }
129 die $_ unless $self->ignore_ddl;
131 for my $dirname (grep { -d $_ } $common, $common_any, $dir_any) {
132 opendir my($dh), $dirname;
133 for my $filename (grep { /\.(?:sql|pl)$/ && -f catfile($dirname,$_) } readdir $dh) {
134 unless ($files{$filename}) {
135 $files{$filename} = catfile($dirname,$filename);
141 return [@files{sort keys %files}]
144 sub _ddl_initialize_consume_filenames {
145 my ($self, $type, $version) = @_;
146 $self->__ddl_consume_with_prefix($type, [ $version ], 'initialize')
149 sub _ddl_schema_consume_filenames {
150 my ($self, $type, $version) = @_;
151 $self->__ddl_consume_with_prefix($type, [ $version ], 'deploy')
154 sub _ddl_protoschema_deploy_consume_filenames {
155 my ($self, $version) = @_;
156 my $base_dir = $self->script_directory;
158 my $dir = catfile( $base_dir, '_source', 'deploy', $version);
159 return [] unless -d $dir;
161 opendir my($dh), $dir;
162 my %files = map { $_ => "$dir/$_" } grep { /\.yml$/ && -f "$dir/$_" } readdir $dh;
165 return [@files{sort keys %files}]
168 sub _ddl_protoschema_upgrade_consume_filenames {
169 my ($self, $versions) = @_;
170 my $base_dir = $self->script_directory;
172 my $dir = catfile( $base_dir, '_preprocess_schema', 'upgrade', join q(-), @{$versions});
174 return [] unless -d $dir;
176 opendir my($dh), $dir;
177 my %files = map { $_ => "$dir/$_" } grep { /\.pl$/ && -f "$dir/$_" } readdir $dh;
180 return [@files{sort keys %files}]
183 sub _ddl_protoschema_downgrade_consume_filenames {
184 my ($self, $versions) = @_;
185 my $base_dir = $self->script_directory;
187 my $dir = catfile( $base_dir, '_preprocess_schema', 'downgrade', join q(-), @{$versions});
189 return [] unless -d $dir;
191 opendir my($dh), $dir;
192 my %files = map { $_ => "$dir/$_" } grep { /\.pl$/ && -f "$dir/$_" } readdir $dh;
195 return [@files{sort keys %files}]
198 sub _ddl_protoschema_produce_filename {
199 my ($self, $version) = @_;
200 my $dirname = catfile( $self->script_directory, '_source', 'deploy', $version );
201 mkpath($dirname) unless -d $dirname;
203 return catfile( $dirname, '001-auto.yml' );
206 sub _ddl_schema_produce_filename {
207 my ($self, $type, $version) = @_;
208 my $dirname = catfile( $self->script_directory, $type, 'deploy', $version );
209 mkpath($dirname) unless -d $dirname;
211 return catfile( $dirname, '001-auto.sql' );
214 sub _ddl_schema_upgrade_consume_filenames {
215 my ($self, $type, $versions) = @_;
216 $self->__ddl_consume_with_prefix($type, $versions, 'upgrade')
219 sub _ddl_schema_downgrade_consume_filenames {
220 my ($self, $type, $versions) = @_;
221 $self->__ddl_consume_with_prefix($type, $versions, 'downgrade')
224 sub _ddl_schema_upgrade_produce_filename {
225 my ($self, $type, $versions) = @_;
226 my $dir = $self->script_directory;
228 my $dirname = catfile( $dir, $type, 'upgrade', join q(-), @{$versions});
229 mkpath($dirname) unless -d $dirname;
231 return catfile( $dirname, '001-auto.sql' );
234 sub _ddl_schema_downgrade_produce_filename {
235 my ($self, $type, $versions, $dir) = @_;
236 my $dirname = catfile( $dir, $type, 'downgrade', join q(-), @{$versions} );
237 mkpath($dirname) unless -d $dirname;
239 return catfile( $dirname, '001-auto.sql');
243 my ($self, $sql) = @_;
244 my $storage = $self->storage;
246 $sql = [ _split_sql_chunk( @$sql ) ];
248 Dlog_trace { "Running SQL $_" } $sql;
249 foreach my $line (@{$sql}) {
250 $storage->_query_start($line);
251 # the whole reason we do this is so that we can see the line that was run
253 $storage->dbh_do (sub { $_[1]->do($line) });
256 die "$_ (running line '$line')"
258 $storage->_query_end($line);
260 return join "\n", @$sql
263 # split a chunk o' SQL into statements
264 sub _split_sql_chunk {
265 my @sql = map { split /;\n/, $_ } @_;
269 s/^(?:BEGIN|BEGIN TRANSACTION|COMMIT).*//mgi;
288 my ($self, $filename) = @_;
289 log_debug { "Running SQL from $filename" };
290 return $self->_run_sql_array($self->_read_sql_file($filename));
294 my ($self, $filename, $versions) = @_;
295 log_debug { "Running Perl from $filename" };
296 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
298 no warnings 'redefine';
299 my $fn = eval "$filedata";
301 Dlog_trace { "Running Perl $_" } $fn;
304 croak "$filename failed to compile: $@";
305 } elsif (ref $fn eq 'CODE') {
306 $fn->($self->schema, $versions)
308 croak "$filename should define an anonymouse sub that takes a schema but it didn't!";
312 sub _run_sql_and_perl {
313 my ($self, $filenames, $sql_to_run, $versions) = @_;
314 my @files = @{$filenames};
315 my $guard = $self->schema->txn_scope_guard if $self->txn_wrap;
317 $self->_run_sql_array($sql_to_run) if $self->ignore_ddl;
319 my $sql = ($sql_to_run)?join ";\n", @$sql_to_run:'';
321 for my $filename (@files) {
322 if ($self->ignore_ddl && $filename =~ /^[^_]*-auto.*\.sql$/) {
324 } elsif ($filename =~ /\.sql$/) {
325 $sql .= $self->_run_sql($filename)
326 } elsif ( $filename =~ /\.pl$/ ) {
327 $self->_run_perl($filename, $versions)
329 croak "A file ($filename) got to deploy that wasn't sql or perl!";
333 $guard->commit if $self->txn_wrap;
340 my $version = (shift @_ || {})->{version} || $self->schema_version;
341 log_info { "deploying version $version" };
342 my $sqlt_type = $self->storage->sqlt_type;
344 if ($self->ignore_ddl) {
345 $sql = $self->_sql_from_yaml({},
346 '_ddl_protoschema_deploy_consume_filenames', $sqlt_type
349 return $self->_run_sql_and_perl($self->_ddl_schema_consume_filenames(
352 ), $sql, [$version]);
358 my $version = $args->{version} || $self->schema_version;
359 log_info { "initializing version $version" };
360 my $storage_type = $args->{storage_type} || $self->storage->sqlt_type;
362 my @files = @{$self->_ddl_initialize_consume_filenames(
367 for my $filename (@files) {
368 # We ignore sql for now (till I figure out what to do with it)
369 if ( $filename =~ /^(.+)\.pl$/ ) {
370 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
372 no warnings 'redefine';
373 my $fn = eval "$filedata";
377 croak "$filename failed to compile: $@";
378 } elsif (ref $fn eq 'CODE') {
381 croak "$filename should define an anonymous sub but it didn't!";
384 croak "A file ($filename) got to initialize_scripts that wasn't sql or perl!";
389 sub _sqldiff_from_yaml {
390 my ($self, $from_version, $to_version, $db, $direction) = @_;
391 my $dir = $self->script_directory;
394 ignore_constraint_names => 1,
395 ignore_index_names => 1,
396 %{$self->sql_translator_args}
401 my $prefilename = $self->_ddl_protoschema_produce_filename($from_version, $dir);
403 # should probably be a croak
404 carp("No previous schema file found ($prefilename)")
405 unless -e $prefilename;
407 my $t = SQL::Translator->new({
411 parser => 'SQL::Translator::Parser::YAML',
414 my $out = $t->translate( $prefilename )
417 $source_schema = $t->schema;
419 $source_schema->name( $prefilename )
420 unless $source_schema->name;
425 my $filename = $self->_ddl_protoschema_produce_filename($to_version, $dir);
427 # should probably be a croak
428 carp("No next schema file found ($filename)")
431 my $t = SQL::Translator->new({
435 parser => 'SQL::Translator::Parser::YAML',
438 my $out = $t->translate( $filename )
441 $dest_schema = $t->schema;
443 $dest_schema->name( $filename )
444 unless $dest_schema->name;
447 my $transform_files_method = "_ddl_protoschema_${direction}_consume_filenames";
448 my $transforms = $self->_coderefs_per_files(
449 $self->$transform_files_method([$from_version, $to_version])
451 $_->($source_schema, $dest_schema) for @$transforms;
453 return [SQL::Translator::Diff::schema_diff(
461 my ($self, $sqltargs, $from_file, $db) = @_;
462 my $schema = $self->schema;
463 my $version = $self->schema_version;
467 my $actual_file = $self->$from_file($version);
468 for my $yaml_filename (@{
469 DlogS_trace { "generating SQL from Serialized SQL Files: $_" }
470 (ref $actual_file?$actual_file:[$actual_file])
472 my $sqlt = SQL::Translator->new({
474 parser => 'SQL::Translator::Parser::YAML',
479 push @sql, $sqlt->translate($yaml_filename);
481 carp("Failed to translate to $db, skipping. (" . $sqlt->error . ")");
488 sub _prepare_install {
490 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
491 my $from_file = shift;
493 my $dir = $self->script_directory;
494 my $databases = $self->databases;
495 my $version = $self->schema_version;
497 foreach my $db (@$databases) {
498 my $sql = $self->_sql_from_yaml($sqltargs, $from_file, $db ) or next;
500 my $filename = $self->$to_file($db, $version, $dir);
502 if ($self->force_overwrite) {
503 carp "Overwriting existing DDL file - $filename";
506 die "Cannot overwrite '$filename', either enable force_overwrite or delete it"
509 open my $file, q(>), $filename;
510 print {$file} join ";\n", @$sql;
515 sub _resultsource_install_filename {
516 my ($self, $source_name) = @_;
518 my ($self, $type, $version) = @_;
519 my $dirname = catfile( $self->script_directory, $type, 'deploy', $version );
520 mkpath($dirname) unless -d $dirname;
522 return catfile( $dirname, "001-auto-$source_name.sql" );
526 sub _resultsource_protoschema_filename {
527 my ($self, $source_name) = @_;
529 my ($self, $version) = @_;
530 my $dirname = catfile( $self->script_directory, '_source', 'deploy', $version );
531 mkpath($dirname) unless -d $dirname;
533 return catfile( $dirname, "001-auto-$source_name.yml" );
537 sub install_resultsource {
538 my ($self, $args) = @_;
539 my $source = $args->{result_source}
540 or die 'result_source must be passed to install_resultsource';
541 my $version = $args->{version}
542 or die 'version must be passed to install_resultsource';
543 log_info { 'installing_resultsource ' . $source->source_name . ", version $version" };
544 my $rs_install_file =
545 $self->_resultsource_install_filename($source->source_name);
548 $self->$rs_install_file(
549 $self->storage->sqlt_type,
553 $self->_run_sql_and_perl($files, '', [$version]);
556 sub prepare_resultsource_install {
558 my $source = (shift @_)->{result_source};
559 log_info { 'preparing install for resultsource ' . $source->source_name };
561 my $install_filename = $self->_resultsource_install_filename($source->source_name);
562 my $proto_filename = $self->_resultsource_protoschema_filename($source->source_name);
563 $self->prepare_protoschema({
564 parser_args => { sources => [$source->source_name], }
566 $self->_prepare_install({}, $proto_filename, $install_filename);
570 log_info { 'preparing deploy' };
572 $self->prepare_protoschema({
573 # Exclude __VERSION so that it gets installed separately
574 parser_args => { sources => [grep { $_ ne '__VERSION' } $self->schema->sources], }
575 }, '_ddl_protoschema_produce_filename');
576 $self->_prepare_install({}, '_ddl_protoschema_produce_filename', '_ddl_schema_produce_filename');
579 sub prepare_upgrade {
580 my ($self, $args) = @_;
582 "preparing upgrade from $args->{from_version} to $args->{to_version}"
584 $self->_prepare_changegrade(
585 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'upgrade'
589 sub prepare_downgrade {
590 my ($self, $args) = @_;
592 "preparing downgrade from $args->{from_version} to $args->{to_version}"
594 $self->_prepare_changegrade(
595 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'downgrade'
599 sub _coderefs_per_files {
600 my ($self, $files) = @_;
601 no warnings 'redefine';
602 [map eval do { local( @ARGV, $/ ) = $_; <> }, @$files]
605 sub _prepare_changegrade {
606 my ($self, $from_version, $to_version, $version_set, $direction) = @_;
607 my $schema = $self->schema;
608 my $databases = $self->databases;
609 my $dir = $self->script_directory;
611 my $schema_version = $self->schema_version;
612 my $diff_file_method = "_ddl_schema_${direction}_produce_filename";
613 foreach my $db (@$databases) {
614 my $diff_file = $self->$diff_file_method($db, $version_set, $dir );
616 if ($self->force_overwrite) {
617 carp("Overwriting existing $direction-diff file - $diff_file");
620 die "Cannot overwrite '$diff_file', either enable force_overwrite or delete it"
624 open my $file, q(>), $diff_file;
625 print {$file} join ";\n", @{$self->_sqldiff_from_yaml($from_version, $to_version, $db, $direction)};
631 my ($self, $file) = @_;
634 local $/ = undef; #sluuuuuurp
636 open my $fh, '<', $file;
637 return [ _split_sql_chunk( <$fh> ) ];
640 sub downgrade_single_step {
642 my $version_set = (shift @_)->{version_set};
643 Dlog_info { "downgrade_single_step'ing $_" } $version_set;
645 my $sqlt_type = $self->storage->sqlt_type;
647 if ($self->ignore_ddl) {
648 $sql_to_run = $self->_sqldiff_from_yaml(
649 $version_set->[0], $version_set->[1], $sqlt_type, 'downgrade',
652 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_downgrade_consume_filenames(
655 ), $sql_to_run, $version_set);
660 sub upgrade_single_step {
662 my $version_set = (shift @_)->{version_set};
663 Dlog_info { "upgrade_single_step'ing $_" } $version_set;
665 my $sqlt_type = $self->storage->sqlt_type;
667 if ($self->ignore_ddl) {
668 $sql_to_run = $self->_sqldiff_from_yaml(
669 $version_set->[0], $version_set->[1], $sqlt_type, 'upgrade',
672 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_upgrade_consume_filenames(
675 ), $sql_to_run, $version_set);
679 sub prepare_protoschema {
681 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
684 = $self->$to_file($self->schema_version);
686 # we do this because the code that uses this sets parser args,
687 # so we just need to merge in the package
688 $sqltargs->{parser_args}{package} = $self->schema;
689 my $sqlt = SQL::Translator->new({
690 parser => 'SQL::Translator::Parser::DBIx::Class',
691 producer => 'SQL::Translator::Producer::YAML',
695 my $yml = $sqlt->translate;
697 croak("Failed to translate to YAML: " . $sqlt->error)
701 if ($self->force_overwrite) {
702 carp "Overwriting existing DDL-YML file - $filename";
705 die "Cannot overwrite '$filename', either enable force_overwrite or delete it"
709 open my $file, q(>), $filename;
714 __PACKAGE__->meta->make_immutable;
718 # vim: ts=2 sw=2 expandtab
724 This class is the meat of L<DBIx::Class::DeploymentHandler>. It takes care
725 of generating serialized schemata as well as sql files to move from one
726 version of a schema to the rest. One of the hallmark features of this class
727 is that it allows for multiple sql files for deploy and upgrade, allowing
728 developers to fine tune deployment. In addition it also allows for perl
729 files to be run at any stage of the process.
731 For basic usage see L<DBIx::Class::DeploymentHandler::HandlesDeploy>. What's
732 documented here is extra fun stuff or private methods.
734 =head1 DIRECTORY LAYOUT
736 Arguably this is the best feature of L<DBIx::Class::DeploymentHandler>.
737 It's spiritually based upon L<DBIx::Migration::Directories>, but has a
738 lot of extensions and modifications, so even if you are familiar with it,
739 please read this. I feel like the best way to describe the layout is with
740 the following example:
766 | | `- 002-remove-customers.pl
769 | | `- 002-generate-customers.pl
771 | `- 999-bump-action.pl
778 | |- 001-create_database.pl
779 | `- 002-create_users_and_permissions.pl
787 So basically, the code
791 on an C<SQLite> database that would simply run
792 C<$sql_migration_dir/SQLite/deploy/1/001-auto.sql>. Next,
794 $dm->upgrade_single_step([1,2])
796 would run C<$sql_migration_dir/SQLite/upgrade/1-2/001-auto.sql> followed by
797 C<$sql_migration_dir/_common/upgrade/1-2/002-generate-customers.pl>, and
798 finally punctuated by
799 C<$sql_migration_dir/_common/upgrade/_any/999-bump-action.pl>.
801 C<.pl> files don't have to be in the C<_common> directory, but most of the time
802 they should be, because perl scripts are generally database independent.
804 Note that unlike most steps in the process, C<initialize> will not run SQL, as
805 there may not even be an database at initialize time. It will run perl scripts
806 just like the other steps in the process, but nothing is passed to them.
807 Until people have used this more it will remain freeform, but a recommended use
808 of initialize is to have it prompt for username and password, and then call the
809 appropriate C<< CREATE DATABASE >> commands etc.
811 =head2 Directory Specification
813 The following subdirectories are recognized by this DeployMethod:
817 =item C<_source> This directory can contain the following directories:
821 =item C<deploy> This directory merely contains directories named after schema
822 versions, which in turn contain C<yaml> files that are serialized versions
823 of the schema at that version. These files are not for editing by hand.
827 =item C<_preprocess_schema> This directory can contain the following
832 =item C<downgrade> This directory merely contains directories named after
833 migrations, which are of the form C<$from_version-$to_version>. Inside of
834 these directories you may put Perl scripts which are to return a subref
835 that takes the arguments C<< $from_schema, $to_schema >>, which are
836 L<SQL::Translator::Schema> objects.
838 =item C<upgrade> This directory merely contains directories named after
839 migrations, which are of the form C<$from_version-$to_version>. Inside of
840 these directories you may put Perl scripts which are to return a subref
841 that takes the arguments C<< $from_schema, $to_schema >>, which are
842 L<SQL::Translator::Schema> objects.
846 =item C<$storage_type> This is a set of scripts that gets run depending on what
847 your storage type is. If you are not sure what your storage type is, take a
848 look at the producers listed for L<SQL::Translator>. Also note, C<_common>
849 is a special case. C<_common> will get merged into whatever other files you
850 already have. This directory can contain the following directories itself:
854 =item C<initialize> Gets run before the C<deploy> is C<deploy>ed. Has the
855 same structure as the C<deploy> subdirectory as well; that is, it has a
856 directory for each schema version. Unlike C<deploy>, C<upgrade>, and C<downgrade>
857 though, it can only run C<.pl> files, and the coderef in the perl files get
858 no arguments passed to them.
860 =item C<deploy> Gets run when the schema is C<deploy>ed. Structure is a
861 directory per schema version, and then files are merged with C<_common> and run
862 in filename order. C<.sql> files are merely run, as expected. C<.pl> files are
863 run according to L</PERL SCRIPTS>.
865 =item C<upgrade> Gets run when the schema is C<upgrade>d. Structure is a directory
866 per upgrade step, (for example, C<1-2> for upgrading from version 1 to version
867 2,) and then files are merged with C<_common> and run in filename order.
868 C<.sql> files are merely run, as expected. C<.pl> files are run according
871 =item C<downgrade> Gets run when the schema is C<downgrade>d. Structure is a directory
872 per downgrade step, (for example, C<2-1> for downgrading from version 2 to version
873 1,) and then files are merged with C<_common> and run in filename order.
874 C<.sql> files are merely run, as expected. C<.pl> files are run according
882 Note that there can be an C<_any> in the place of any of the versions (like
883 C<1-2> or C<1>), which means those scripts will be run B<every> time. So if
884 you have an C<_any> in C<_common/upgrade>, that script will get run for every
889 A perl script for this tool is very simple. It merely needs to contain an
890 anonymous sub that takes a L<DBIx::Class::Schema> and the version set as it's
893 A very basic perl script might look like:
903 # [1] for deploy, [1,2] for upgrade or downgrade, probably used with _any
904 my $versions = shift;
906 $schema->resultset('Users')->create({
914 This attribute will, when set to true (default is false), cause the DM to use
915 L<SQL::Translator> to use the C<_source>'s serialized SQL::Translator::Schema
916 instead of any pregenerated SQL. If you have a development server this is
917 probably the best plan of action as you will not be putting as many generated
918 files in your version control. Goes well with with C<databases> of C<[]>.
920 =attr force_overwrite
922 When this attribute is true generated files will be overwritten when the
923 methods which create such files are run again. The default is false, in which
924 case the program will die with a message saying which file needs to be deleted.
928 The L<DBIx::Class::Schema> (B<required>) that is used to talk to the database
929 and generate the DDL.
933 The L<DBIx::Class::Storage> that is I<actually> used to talk to the database
934 and generate the DDL. This is automatically created with L</_build_storage>.
936 =attr sql_translator_args
938 The arguments that get passed to L<SQL::Translator> when it's used.
940 =attr script_directory
942 The directory (default C<'sql'>) that scripts are stored in
946 The types of databases (default C<< [qw( MySQL SQLite PostgreSQL )] >>) to
951 Set to true (which is the default) to wrap all upgrades and deploys in a single
956 The version the schema on your harddrive is at. Defaults to
957 C<< $self->schema->schema_version >>.