1 package DBIx::Class::DeploymentHandler::DeployMethod::SQL::Translator;
4 # ABSTRACT: Manage your SQL and Perl migrations in nicely laid out directories
7 use Carp qw( carp croak );
8 use DBIx::Class::DeploymentHandler::Logger;
9 use Log::Contextual qw(:log :dlog), -package_logger =>
10 DBIx::Class::DeploymentHandler::Logger->new({
11 env_prefix => 'DBICDH'
14 use Method::Signatures::Simple;
18 require SQL::Translator::Diff;
20 require DBIx::Class::Storage; # loaded for type constraint
21 use DBIx::Class::DeploymentHandler::Types;
23 use File::Path 'mkpath';
24 use File::Spec::Functions;
26 with 'DBIx::Class::DeploymentHandler::HandlesDeploy';
34 has force_overwrite => (
41 isa => 'DBIx::Class::Schema',
47 isa => 'DBIx::Class::Storage',
52 has progress_callback => (
54 isa => 'Maybe[CodeRef]',
57 method _build_storage {
58 my $s = $self->schema->storage;
59 $s->_determine_driver;
63 has sql_translator_args => (
66 default => sub { {} },
68 has script_directory => (
77 isa => 'DBIx::Class::DeploymentHandler::Databases',
79 default => sub { [qw( MySQL SQLite PostgreSQL )] },
88 has schema_version => (
94 # this will probably never get called as the DBICDH
95 # will be passing down a schema_version normally, which
96 # is built the same way, but we leave this in place
97 method _build_schema_version { $self->schema->schema_version }
99 method __ddl_consume_with_prefix($type, $versions, $prefix) {
100 my $base_dir = $self->script_directory;
102 my $main = catfile( $base_dir, $type );
104 catfile( $base_dir, '_common', $prefix, join q(-), @{$versions} );
107 catfile( $base_dir, '_common', $prefix, '_any' );
111 $dir = catfile($main, $prefix, join q(-), @{$versions})
113 if ($self->ignore_ddl) {
116 croak "$main does not exist; please write/generate some SQL"
119 my $dir_any = catfile($main, $prefix, '_any');
123 opendir my($dh), $dir;
125 map { $_ => "$dir/$_" }
126 grep { /\.(?:sql|pl|sql-\w+)$/ && -f "$dir/$_" }
130 die $_ unless $self->ignore_ddl;
132 for my $dirname (grep { -d $_ } $common, $common_any, $dir_any) {
133 opendir my($dh), $dirname;
134 for my $filename (grep { /\.(?:sql|pl)$/ && -f catfile($dirname,$_) } readdir $dh) {
135 unless ($files{$filename}) {
136 $files{$filename} = catfile($dirname,$filename);
142 return [@files{sort keys %files}]
145 method _ddl_initialize_consume_filenames($type, $version) {
146 $self->__ddl_consume_with_prefix($type, [ $version ], 'initialize')
149 method _ddl_schema_consume_filenames($type, $version) {
150 $self->__ddl_consume_with_prefix($type, [ $version ], 'deploy')
153 method _ddl_protoschema_deploy_consume_filenames($version) {
154 my $base_dir = $self->script_directory;
156 my $dir = catfile( $base_dir, '_source', 'deploy', $version);
157 return [] unless -d $dir;
159 opendir my($dh), $dir;
160 my %files = map { $_ => "$dir/$_" } grep { /\.yml$/ && -f "$dir/$_" } readdir $dh;
163 return [@files{sort keys %files}]
166 method _ddl_protoschema_upgrade_consume_filenames($versions) {
167 my $base_dir = $self->script_directory;
169 my $dir = catfile( $base_dir, '_preprocess_schema', 'upgrade', join q(-), @{$versions});
171 return [] unless -d $dir;
173 opendir my($dh), $dir;
174 my %files = map { $_ => "$dir/$_" } grep { /\.pl$/ && -f "$dir/$_" } readdir $dh;
177 return [@files{sort keys %files}]
180 method _ddl_protoschema_downgrade_consume_filenames($versions) {
181 my $base_dir = $self->script_directory;
183 my $dir = catfile( $base_dir, '_preprocess_schema', 'downgrade', join q(-), @{$versions});
185 return [] unless -d $dir;
187 opendir my($dh), $dir;
188 my %files = map { $_ => "$dir/$_" } grep { /\.pl$/ && -f "$dir/$_" } readdir $dh;
191 return [@files{sort keys %files}]
194 method _ddl_protoschema_produce_filename($version) {
195 my $dirname = catfile( $self->script_directory, '_source', 'deploy', $version );
196 mkpath($dirname) unless -d $dirname;
198 return catfile( $dirname, '001-auto.yml' );
201 method _ddl_schema_produce_filename($type, $version) {
202 my $dirname = catfile( $self->script_directory, $type, 'deploy', $version );
203 mkpath($dirname) unless -d $dirname;
205 return catfile( $dirname, '001-auto.sql' );
208 method _ddl_schema_upgrade_consume_filenames($type, $versions) {
209 $self->__ddl_consume_with_prefix($type, $versions, 'upgrade')
212 method _ddl_schema_downgrade_consume_filenames($type, $versions) {
213 $self->__ddl_consume_with_prefix($type, $versions, 'downgrade')
216 method _ddl_schema_upgrade_produce_filename($type, $versions) {
217 my $dir = $self->script_directory;
219 my $dirname = catfile( $dir, $type, 'upgrade', join q(-), @{$versions});
220 mkpath($dirname) unless -d $dirname;
222 return catfile( $dirname, '001-auto.sql' );
225 method _ddl_schema_downgrade_produce_filename($type, $versions, $dir) {
226 my $dirname = catfile( $dir, $type, 'downgrade', join q(-), @{$versions} );
227 mkpath($dirname) unless -d $dirname;
229 return catfile( $dirname, '001-auto.sql');
232 method _run_sql_array($sql) {
233 my $storage = $self->storage;
236 $_ && # remove blank lines
237 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
239 s/^\s+//; s/\s+$//; # trim whitespace
240 join '', grep { !/^--/ } split /\n/ # remove comments
243 Dlog_trace { "Running SQL $_" } $sql;
244 foreach my $line (@{$sql}) {
245 $storage->_query_start($line);
246 # the whole reason we do this is so that we can see the line that was run
248 $storage->dbh_do (sub { $_[1]->do($line) });
251 die "$_ (running line '$line')"
253 $storage->_query_end($line);
255 return join "\n", @$sql
258 method _run_sql($filename) {
259 log_debug { "Running SQL from $filename" };
260 return $self->_run_sql_array($self->_read_sql_file($filename));
263 method _run_perl($filename, $versions) {
264 log_debug { "Running Perl from $filename" };
265 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
267 no warnings 'redefine';
268 my $fn = eval "$filedata";
270 Dlog_trace { "Running Perl $_" } $fn;
273 croak "$filename failed to compile: $@";
274 } elsif (ref $fn eq 'CODE') {
275 $fn->($self->schema, $versions)
277 croak "$filename should define an anonymouse sub that takes a schema but it didn't!";
281 method _run_sql_and_perl($filenames, $sql_to_run, $versions) {
282 my @files = @{$filenames};
283 my $guard = $self->schema->txn_scope_guard if $self->txn_wrap;
285 $self->_run_sql_array($sql_to_run) if $self->ignore_ddl;
287 my $sql = ($sql_to_run)?join ";\n", @$sql_to_run:'';
290 for my $filename (@files) {
292 if ($self->ignore_ddl && $filename =~ /^[^_]*-auto.*\.sql$/) {
293 $self->progress_callback and $self->progress_callback->($i, scalar(@files), 'skip', $filename);
295 } elsif ($filename =~ /\.sql$/) {
296 $self->progress_callback and $self->progress_callback->($i, scalar(@files), 'sql', $filename);
297 $sql .= $self->_run_sql($filename)
298 } elsif ( $filename =~ /\.pl$/ ) {
299 $self->progress_callback and $self->progress_callback->($i, scalar(@files), 'perl', $filename);
300 $self->_run_perl($filename, $versions)
302 croak "A file ($filename) got to deploy that wasn't sql or perl!";
306 $guard->commit if $self->txn_wrap;
313 my $version = (shift @_ || {})->{version} || $self->schema_version;
314 log_info { "deploying version $version" };
315 my $sqlt_type = $self->storage->sqlt_type;
317 if ($self->ignore_ddl) {
318 $sql = $self->_sql_from_yaml({},
319 '_ddl_protoschema_deploy_consume_filenames', $sqlt_type
322 return $self->_run_sql_and_perl($self->_ddl_schema_consume_filenames(
325 ), $sql, [$version]);
331 my $version = $args->{version} || $self->schema_version;
332 log_info { "initializing version $version" };
333 my $storage_type = $args->{storage_type} || $self->storage->sqlt_type;
335 my @files = @{$self->_ddl_initialize_consume_filenames(
340 for my $filename (@files) {
341 # We ignore sql for now (till I figure out what to do with it)
342 if ( $filename =~ /^(.+)\.pl$/ ) {
343 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
345 no warnings 'redefine';
346 my $fn = eval "$filedata";
350 croak "$filename failed to compile: $@";
351 } elsif (ref $fn eq 'CODE') {
354 croak "$filename should define an anonymous sub but it didn't!";
357 croak "A file ($filename) got to initialize_scripts that wasn't sql or perl!";
362 method _sqldiff_from_yaml($from_version, $to_version, $db, $direction) {
363 my $dir = $self->script_directory;
366 ignore_constraint_names => 1,
367 ignore_index_names => 1,
368 %{$self->sql_translator_args}
373 my $prefilename = $self->_ddl_protoschema_produce_filename($from_version, $dir);
375 # should probably be a croak
376 carp("No previous schema file found ($prefilename)")
377 unless -e $prefilename;
379 my $t = SQL::Translator->new({
383 parser => 'SQL::Translator::Parser::YAML',
386 my $out = $t->translate( $prefilename )
389 $source_schema = $t->schema;
391 $source_schema->name( $prefilename )
392 unless $source_schema->name;
397 my $filename = $self->_ddl_protoschema_produce_filename($to_version, $dir);
399 # should probably be a croak
400 carp("No next schema file found ($filename)")
403 my $t = SQL::Translator->new({
407 parser => 'SQL::Translator::Parser::YAML',
410 my $out = $t->translate( $filename )
413 $dest_schema = $t->schema;
415 $dest_schema->name( $filename )
416 unless $dest_schema->name;
419 my $transform_files_method = "_ddl_protoschema_${direction}_consume_filenames";
420 my $transforms = $self->_coderefs_per_files(
421 $self->$transform_files_method([$from_version, $to_version])
423 $_->($source_schema, $dest_schema) for @$transforms;
425 return [SQL::Translator::Diff::schema_diff(
432 method _sql_from_yaml($sqltargs, $from_file, $db) {
433 my $schema = $self->schema;
434 my $version = $self->schema_version;
438 my $actual_file = $self->$from_file($version);
439 for my $yaml_filename (@{
440 DlogS_trace { "generating SQL from Serialized SQL Files: $_" }
441 (ref $actual_file?$actual_file:[$actual_file])
443 my $sqlt = SQL::Translator->new({
445 parser => 'SQL::Translator::Parser::YAML',
450 push @sql, $sqlt->translate($yaml_filename);
452 carp("Failed to translate to $db, skipping. (" . $sqlt->error . ")");
459 sub _prepare_install {
461 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
462 my $from_file = shift;
464 my $dir = $self->script_directory;
465 my $databases = $self->databases;
466 my $version = $self->schema_version;
468 foreach my $db (@$databases) {
469 my $sql = $self->_sql_from_yaml($sqltargs, $from_file, $db ) or next;
471 my $filename = $self->$to_file($db, $version, $dir);
473 if ($self->force_overwrite) {
474 carp "Overwriting existing DDL file - $filename";
477 die "Cannot overwrite '$filename', either enable force_overwrite or delete it"
480 open my $file, q(>), $filename;
481 print {$file} join ";\n", @$sql;
486 sub _resultsource_install_filename {
487 my ($self, $source_name) = @_;
489 my ($self, $type, $version) = @_;
490 my $dirname = catfile( $self->script_directory, $type, 'deploy', $version );
491 mkpath($dirname) unless -d $dirname;
493 return catfile( $dirname, "001-auto-$source_name.sql" );
497 sub _resultsource_protoschema_filename {
498 my ($self, $source_name) = @_;
500 my ($self, $version) = @_;
501 my $dirname = catfile( $self->script_directory, '_source', 'deploy', $version );
502 mkpath($dirname) unless -d $dirname;
504 return catfile( $dirname, "001-auto-$source_name.yml" );
508 sub install_resultsource {
509 my ($self, $args) = @_;
510 my $source = $args->{result_source}
511 or die 'result_source must be passed to install_resultsource';
512 my $version = $args->{version}
513 or die 'version must be passed to install_resultsource';
514 log_info { 'installing_resultsource ' . $source->source_name . ", version $version" };
515 my $rs_install_file =
516 $self->_resultsource_install_filename($source->source_name);
519 $self->$rs_install_file(
520 $self->storage->sqlt_type,
524 $self->_run_sql_and_perl($files, '', [$version]);
527 sub prepare_resultsource_install {
529 my $source = (shift @_)->{result_source};
530 log_info { 'preparing install for resultsource ' . $source->source_name };
532 my $install_filename = $self->_resultsource_install_filename($source->source_name);
533 my $proto_filename = $self->_resultsource_protoschema_filename($source->source_name);
534 $self->prepare_protoschema({
535 parser_args => { sources => [$source->source_name], }
537 $self->_prepare_install({}, $proto_filename, $install_filename);
541 log_info { 'preparing deploy' };
543 $self->prepare_protoschema({
544 # Exclude __VERSION so that it gets installed separately
545 parser_args => { sources => [grep { $_ ne '__VERSION' } $self->schema->sources], }
546 }, '_ddl_protoschema_produce_filename');
547 $self->_prepare_install({}, '_ddl_protoschema_produce_filename', '_ddl_schema_produce_filename');
550 sub prepare_upgrade {
551 my ($self, $args) = @_;
553 "preparing upgrade from $args->{from_version} to $args->{to_version}"
555 $self->_prepare_changegrade(
556 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'upgrade'
560 sub prepare_downgrade {
561 my ($self, $args) = @_;
563 "preparing downgrade from $args->{from_version} to $args->{to_version}"
565 $self->_prepare_changegrade(
566 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'downgrade'
570 method _coderefs_per_files($files) {
571 no warnings 'redefine';
572 [map eval do { local( @ARGV, $/ ) = $_; <> }, @$files]
575 method _prepare_changegrade($from_version, $to_version, $version_set, $direction) {
576 my $schema = $self->schema;
577 my $databases = $self->databases;
578 my $dir = $self->script_directory;
580 my $schema_version = $self->schema_version;
581 my $diff_file_method = "_ddl_schema_${direction}_produce_filename";
582 foreach my $db (@$databases) {
583 my $diff_file = $self->$diff_file_method($db, $version_set, $dir );
585 if ($self->force_overwrite) {
586 carp("Overwriting existing $direction-diff file - $diff_file");
589 die "Cannot overwrite '$diff_file', either enable force_overwrite or delete it"
593 open my $file, q(>), $diff_file;
594 print {$file} join ";\n", @{$self->_sqldiff_from_yaml($from_version, $to_version, $db, $direction)};
599 method _read_sql_file($file) {
602 open my $fh, '<', $file;
603 my @data = split /;\n/, join '', <$fh>;
607 $_ && # remove blank lines
608 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
610 s/^\s+//; s/\s+$//; # trim whitespace
611 join '', grep { !/^--/ } split /\n/ # remove comments
617 sub downgrade_single_step {
619 my $version_set = (shift @_)->{version_set};
620 Dlog_info { "downgrade_single_step'ing $_" } $version_set;
622 my $sqlt_type = $self->storage->sqlt_type;
624 if ($self->ignore_ddl) {
625 $sql_to_run = $self->_sqldiff_from_yaml(
626 $version_set->[0], $version_set->[1], $sqlt_type, 'downgrade',
629 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_downgrade_consume_filenames(
632 ), $sql_to_run, $version_set);
637 sub upgrade_single_step {
639 my $version_set = (shift @_)->{version_set};
640 Dlog_info { "upgrade_single_step'ing $_" } $version_set;
642 my $sqlt_type = $self->storage->sqlt_type;
644 if ($self->ignore_ddl) {
645 $sql_to_run = $self->_sqldiff_from_yaml(
646 $version_set->[0], $version_set->[1], $sqlt_type, 'upgrade',
649 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_upgrade_consume_filenames(
652 ), $sql_to_run, $version_set);
656 sub prepare_protoschema {
658 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
661 = $self->$to_file($self->schema_version);
663 # we do this because the code that uses this sets parser args,
664 # so we just need to merge in the package
665 $sqltargs->{parser_args}{package} = $self->schema;
666 my $sqlt = SQL::Translator->new({
667 parser => 'SQL::Translator::Parser::DBIx::Class',
668 producer => 'SQL::Translator::Producer::YAML',
672 my $yml = $sqlt->translate;
674 croak("Failed to translate to YAML: " . $sqlt->error)
678 if ($self->force_overwrite) {
679 carp "Overwriting existing DDL-YML file - $filename";
682 die "Cannot overwrite '$filename', either enable force_overwrite or delete it"
686 open my $file, q(>), $filename;
691 __PACKAGE__->meta->make_immutable;
695 # vim: ts=2 sw=2 expandtab
701 This class is the meat of L<DBIx::Class::DeploymentHandler>. It takes care
702 of generating serialized schemata as well as sql files to move from one
703 version of a schema to the rest. One of the hallmark features of this class
704 is that it allows for multiple sql files for deploy and upgrade, allowing
705 developers to fine tune deployment. In addition it also allows for perl
706 files to be run at any stage of the process.
708 For basic usage see L<DBIx::Class::DeploymentHandler::HandlesDeploy>. What's
709 documented here is extra fun stuff or private methods.
711 =head1 DIRECTORY LAYOUT
713 Arguably this is the best feature of L<DBIx::Class::DeploymentHandler>.
714 It's spiritually based upon L<DBIx::Migration::Directories>, but has a
715 lot of extensions and modifications, so even if you are familiar with it,
716 please read this. I feel like the best way to describe the layout is with
717 the following example:
743 | | `- 002-remove-customers.pl
746 | | `- 002-generate-customers.pl
748 | `- 999-bump-action.pl
755 | |- 001-create_database.pl
756 | `- 002-create_users_and_permissions.pl
764 So basically, the code
768 on an C<SQLite> database that would simply run
769 C<$sql_migration_dir/SQLite/deploy/1/001-auto.sql>. Next,
771 $dm->upgrade_single_step([1,2])
773 would run C<$sql_migration_dir/SQLite/upgrade/1-2/001-auto.sql> followed by
774 C<$sql_migration_dir/_common/upgrade/1-2/002-generate-customers.pl>, and
775 finally punctuated by
776 C<$sql_migration_dir/_common/upgrade/_any/999-bump-action.pl>.
778 C<.pl> files don't have to be in the C<_common> directory, but most of the time
779 they should be, because perl scripts are generally database independent.
781 Note that unlike most steps in the process, C<initialize> will not run SQL, as
782 there may not even be an database at initialize time. It will run perl scripts
783 just like the other steps in the process, but nothing is passed to them.
784 Until people have used this more it will remain freeform, but a recommended use
785 of initialize is to have it prompt for username and password, and then call the
786 appropriate C<< CREATE DATABASE >> commands etc.
788 =head2 Directory Specification
790 The following subdirectories are recognized by this DeployMethod:
794 =item C<_source> This directory can contain the following directories:
798 =item C<deploy> This directory merely contains directories named after schema
799 versions, which in turn contain C<yaml> files that are serialized versions
800 of the schema at that version. These files are not for editing by hand.
804 =item C<_preprocess_schema> This directory can contain the following
809 =item C<downgrade> This directory merely contains directories named after
810 migrations, which are of the form C<$from_version-$to_version>. Inside of
811 these directories you may put Perl scripts which are to return a subref
812 that takes the arguments C<< $from_schema, $to_schema >>, which are
813 L<SQL::Translator::Schema> objects.
815 =item C<upgrade> This directory merely contains directories named after
816 migrations, which are of the form C<$from_version-$to_version>. Inside of
817 these directories you may put Perl scripts which are to return a subref
818 that takes the arguments C<< $from_schema, $to_schema >>, which are
819 L<SQL::Translator::Schema> objects.
823 =item C<$storage_type> This is a set of scripts that gets run depending on what
824 your storage type is. If you are not sure what your storage type is, take a
825 look at the producers listed for L<SQL::Translator>. Also note, C<_common>
826 is a special case. C<_common> will get merged into whatever other files you
827 already have. This directory can contain the following directories itself:
831 =item C<initialize> Gets run before the C<deploy> is C<deploy>ed. Has the
832 same structure as the C<deploy> subdirectory as well; that is, it has a
833 directory for each schema version. Unlike C<deploy>, C<upgrade>, and C<downgrade>
834 though, it can only run C<.pl> files, and the coderef in the perl files get
835 no arguments passed to them.
837 =item C<deploy> Gets run when the schema is C<deploy>ed. Structure is a
838 directory per schema version, and then files are merged with C<_common> and run
839 in filename order. C<.sql> files are merely run, as expected. C<.pl> files are
840 run according to L</PERL SCRIPTS>.
842 =item C<upgrade> Gets run when the schema is C<upgrade>d. Structure is a directory
843 per upgrade step, (for example, C<1-2> for upgrading from version 1 to version
844 2,) and then files are merged with C<_common> and run in filename order.
845 C<.sql> files are merely run, as expected. C<.pl> files are run according
848 =item C<downgrade> Gets run when the schema is C<downgrade>d. Structure is a directory
849 per downgrade step, (for example, C<2-1> for downgrading from version 2 to version
850 1,) and then files are merged with C<_common> and run in filename order.
851 C<.sql> files are merely run, as expected. C<.pl> files are run according
859 Note that there can be an C<_any> in the place of any of the versions (like
860 C<1-2> or C<1>), which means those scripts will be run B<every> time. So if
861 you have an C<_any> in C<_common/upgrade>, that script will get run for every
866 A perl script for this tool is very simple. It merely needs to contain an
867 anonymous sub that takes a L<DBIx::Class::Schema> and the version set as it's
870 A very basic perl script might look like:
880 # [1] for deploy, [1,2] for upgrade or downgrade, probably used with _any
881 my $versions = shift;
883 $schema->resultset('Users')->create({
891 This attribute will, when set to true (default is false), cause the DM to use
892 L<SQL::Translator> to use the C<_source>'s serialized SQL::Translator::Schema
893 instead of any pregenerated SQL. If you have a development server this is
894 probably the best plan of action as you will not be putting as many generated
895 files in your version control. Goes well with with C<databases> of C<[]>.
897 =attr force_overwrite
899 When this attribute is true generated files will be overwritten when the
900 methods which create such files are run again. The default is false, in which
901 case the program will die with a message saying which file needs to be deleted.
905 The L<DBIx::Class::Schema> (B<required>) that is used to talk to the database
906 and generate the DDL.
910 The L<DBIx::Class::Storage> that is I<actually> used to talk to the database
911 and generate the DDL. This is automatically created with L</_build_storage>.
913 =attr sql_translator_args
915 The arguments that get passed to L<SQL::Translator> when it's used.
917 =attr script_directory
919 The directory (default C<'sql'>) that scripts are stored in
923 The types of databases (default C<< [qw( MySQL SQLite PostgreSQL )] >>) to
928 Set to true (which is the default) to wrap all upgrades and deploys in a single
933 The version the schema on your harddrive is at. Defaults to
934 C<< $self->schema->schema_version >>.