1 package DBIx::Class::DeploymentHandler::DeployMethod::SQL::Translator;
4 # ABSTRACT: Manage your SQL and Perl migrations in nicely laid out directories
7 use Carp qw( carp croak );
8 use DBIx::Class::DeploymentHandler::Logger;
9 use Log::Contextual qw(:log :dlog), -default_logger =>
10 DBIx::Class::DeploymentHandler::Logger->new({
11 env_prefix => 'DBICDH'
14 use Method::Signatures::Simple;
18 require SQL::Translator::Diff;
20 require DBIx::Class::Storage; # loaded for type constraint
21 use DBIx::Class::DeploymentHandler::Types;
23 use File::Path 'mkpath';
24 use File::Spec::Functions;
26 with 'DBIx::Class::DeploymentHandler::HandlesDeploy';
35 isa => 'DBIx::Class::Schema',
41 isa => 'DBIx::Class::Storage',
46 method _build_storage {
47 my $s = $self->schema->storage;
48 $s->_determine_driver;
52 has sql_translator_args => (
55 default => sub { {} },
57 has script_directory => (
66 isa => 'DBIx::Class::DeploymentHandler::Databases',
68 default => sub { [qw( MySQL SQLite PostgreSQL )] },
77 has schema_version => (
83 # this will probably never get called as the DBICDH
84 # will be passing down a schema_version normally, which
85 # is built the same way, but we leave this in place
86 method _build_schema_version { $self->schema->schema_version }
88 method __ddl_consume_with_prefix($type, $versions, $prefix) {
89 my $base_dir = $self->script_directory;
91 my $main = catfile( $base_dir, $type );
92 my $generic = catfile( $base_dir, '_generic' );
94 catfile( $base_dir, '_common', $prefix, join q(-), @{$versions} );
98 $dir = catfile($main, $prefix, join q(-), @{$versions})
99 } elsif (-d $generic) {
100 $dir = catfile($generic, $prefix, join q(-), @{$versions});
102 croak "neither $main or $generic exist; please write/generate some SQL";
107 opendir my($dh), $dir;
109 map { $_ => "$dir/$_" }
110 grep { /\.(?:sql|pl|sql-\w+)$/ && -f "$dir/$_" }
114 die $_ unless $self->ignore_ddl;
117 opendir my($dh), $common;
118 for my $filename (grep { /\.(?:sql|pl)$/ && -f catfile($common,$_) } readdir $dh) {
119 unless ($files{$filename}) {
120 $files{$filename} = catfile($common,$filename);
126 return [@files{sort keys %files}]
129 method _ddl_preinstall_consume_filenames($type, $version) {
130 $self->__ddl_consume_with_prefix($type, [ $version ], 'preinstall')
133 method _ddl_schema_consume_filenames($type, $version) {
134 $self->__ddl_consume_with_prefix($type, [ $version ], 'schema')
137 method _ddl_protoschema_up_consume_filenames($versions) {
138 my $base_dir = $self->script_directory;
140 my $dir = catfile( $base_dir, '_protoschema', 'up', join q(-), @{$versions});
142 return [] unless -d $dir;
144 opendir my($dh), $dir;
145 my %files = map { $_ => "$dir/$_" } grep { /\.pl$/ && -f "$dir/$_" } readdir $dh;
148 return [@files{sort keys %files}]
151 method _ddl_protoschema_down_consume_filenames($versions) {
152 my $base_dir = $self->script_directory;
154 my $dir = catfile( $base_dir, '_protoschema', 'down', join q(-), @{$versions});
156 return [] unless -d $dir;
158 opendir my($dh), $dir;
159 my %files = map { $_ => "$dir/$_" } grep { /\.pl$/ && -f "$dir/$_" } readdir $dh;
162 return [@files{sort keys %files}]
165 method _ddl_protoschema_produce_filename($version) {
166 my $dirname = catfile( $self->script_directory, '_protoschema', 'schema', $version );
167 mkpath($dirname) unless -d $dirname;
169 return catfile( $dirname, '001-auto.yml' );
172 method _ddl_schema_produce_filename($type, $version) {
173 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
174 mkpath($dirname) unless -d $dirname;
176 return catfile( $dirname, '001-auto.sql' );
179 method _ddl_schema_up_consume_filenames($type, $versions) {
180 $self->__ddl_consume_with_prefix($type, $versions, 'up')
183 method _ddl_schema_down_consume_filenames($type, $versions) {
184 $self->__ddl_consume_with_prefix($type, $versions, 'down')
187 method _ddl_schema_up_produce_filename($type, $versions) {
188 my $dir = $self->script_directory;
190 my $dirname = catfile( $dir, $type, 'up', join q(-), @{$versions});
191 mkpath($dirname) unless -d $dirname;
193 return catfile( $dirname, '001-auto.sql' );
196 method _ddl_schema_down_produce_filename($type, $versions, $dir) {
197 my $dirname = catfile( $dir, $type, 'down', join q(-), @{$versions} );
198 mkpath($dirname) unless -d $dirname;
200 return catfile( $dirname, '001-auto.sql');
203 method _run_sql_array($sql) {
204 my $storage = $self->storage;
207 $_ && # remove blank lines
208 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
210 s/^\s+//; s/\s+$//; # trim whitespace
211 join '', grep { !/^--/ } split /\n/ # remove comments
214 Dlog_trace { "Running SQL $_" } $sql;
215 foreach my $line (@{$sql}) {
216 $storage->_query_start($line);
217 # the whole reason we do this is so that we can see the line that was run
219 $storage->dbh_do (sub { $_[1]->do($line) });
222 die "$_ (running line '$line')"
224 $storage->_query_end($line);
226 return join "\n", @$sql
229 method _run_sql($filename) {
230 log_debug { "Running SQL from $filename" };
231 return $self->_run_sql_array($self->_read_sql_file($filename));
234 method _run_perl($filename) {
235 log_debug { "Running Perl from $filename" };
236 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
238 no warnings 'redefine';
239 my $fn = eval "$filedata";
241 Dlog_trace { "Running Perl $_" } $fn;
244 carp "$filename failed to compile: $@";
245 } elsif (ref $fn eq 'CODE') {
248 carp "$filename should define an anonymouse sub that takes a schema but it didn't!";
252 method _run_sql_and_perl($filenames, $sql_to_run) {
253 my @files = @{$filenames};
254 my $guard = $self->schema->txn_scope_guard if $self->txn_wrap;
256 $self->_run_sql_array($sql_to_run) if $self->ignore_ddl;
258 my $sql = ($sql_to_run)?join ";\n", @$sql_to_run:'';
260 for my $filename (@files) {
261 if ($self->ignore_ddl && $filename =~ /^[^_]*-auto.*\.sql$/) {
263 } elsif ($filename =~ /\.sql$/) {
264 $sql .= $self->_run_sql($filename)
265 } elsif ( $filename =~ /\.pl$/ ) {
266 $self->_run_perl($filename)
268 croak "A file ($filename) got to deploy that wasn't sql or perl!";
272 $guard->commit if $self->txn_wrap;
279 my $version = (shift @_ || {})->{version} || $self->schema_version;
280 log_info { "deploying version $version" };
281 my $sqlt_type = $self->storage->sqlt_type;
283 if ($self->ignore_ddl) {
284 $sql = $self->_sql_from_yaml({},
285 '_ddl_protoschema_produce_filename', $sqlt_type
288 return $self->_run_sql_and_perl($self->_ddl_schema_consume_filenames(
297 my $version = $args->{version} || $self->schema_version;
298 log_info { "preinstalling version $version" };
299 my $storage_type = $args->{storage_type} || $self->storage->sqlt_type;
301 my @files = @{$self->_ddl_preinstall_consume_filenames(
306 for my $filename (@files) {
307 # We ignore sql for now (till I figure out what to do with it)
308 if ( $filename =~ /^(.+)\.pl$/ ) {
309 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
311 no warnings 'redefine';
312 my $fn = eval "$filedata";
316 carp "$filename failed to compile: $@";
317 } elsif (ref $fn eq 'CODE') {
320 carp "$filename should define an anonymous sub but it didn't!";
323 croak "A file ($filename) got to preinstall_scripts that wasn't sql or perl!";
328 method _sqldiff_from_yaml($from_version, $to_version, $db, $direction) {
329 my $dir = $self->script_directory;
332 ignore_constraint_names => 1,
333 ignore_index_names => 1,
334 %{$self->sql_translator_args}
339 my $prefilename = $self->_ddl_protoschema_produce_filename($from_version, $dir);
341 # should probably be a croak
342 carp("No previous schema file found ($prefilename)")
343 unless -e $prefilename;
345 my $t = SQL::Translator->new({
349 parser => 'SQL::Translator::Parser::YAML',
352 my $out = $t->translate( $prefilename )
355 $source_schema = $t->schema;
357 $source_schema->name( $prefilename )
358 unless $source_schema->name;
363 my $filename = $self->_ddl_protoschema_produce_filename($to_version, $dir);
365 # should probably be a croak
366 carp("No next schema file found ($filename)")
369 my $t = SQL::Translator->new({
373 parser => 'SQL::Translator::Parser::YAML',
376 my $out = $t->translate( $filename )
379 $dest_schema = $t->schema;
381 $dest_schema->name( $filename )
382 unless $dest_schema->name;
385 my $transform_files_method = "_ddl_protoschema_${direction}_consume_filenames";
386 my $transforms = $self->_coderefs_per_files(
387 $self->$transform_files_method([$from_version, $to_version])
389 $_->($source_schema, $dest_schema) for @$transforms;
391 return [SQL::Translator::Diff::schema_diff(
398 method _sql_from_yaml($sqltargs, $from_file, $db) {
399 my $schema = $self->schema;
400 my $version = $self->schema_version;
402 my $sqlt = SQL::Translator->new({
404 parser => 'SQL::Translator::Parser::YAML',
409 my $yaml_filename = $self->$from_file($version);
411 my @sql = $sqlt->translate($yaml_filename);
413 carp("Failed to translate to $db, skipping. (" . $sqlt->error . ")");
419 sub _prepare_install {
421 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
422 my $from_file = shift;
424 my $dir = $self->script_directory;
425 my $databases = $self->databases;
426 my $version = $self->schema_version;
428 foreach my $db (@$databases) {
429 my $sql = $self->_sql_from_yaml($sqltargs, $from_file, $db ) or next;
431 my $filename = $self->$to_file($db, $version, $dir);
433 carp "Overwriting existing DDL file - $filename";
436 open my $file, q(>), $filename;
437 print {$file} join ";\n", @$sql;
442 sub _resultsource_install_filename {
443 my ($self, $source_name) = @_;
445 my ($self, $type, $version) = @_;
446 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
447 mkpath($dirname) unless -d $dirname;
449 return catfile( $dirname, "001-auto-$source_name.sql" );
453 sub _resultsource_protoschema_filename {
454 my ($self, $source_name) = @_;
456 my ($self, $version) = @_;
457 my $dirname = catfile( $self->script_directory, '_protoschema', $version );
458 mkpath($dirname) unless -d $dirname;
460 return catfile( $dirname, "001-auto-$source_name.yml" );
464 sub install_resultsource {
465 my ($self, $args) = @_;
466 my $source = $args->{result_source};
467 my $version = $args->{version};
468 log_info { 'installing_resultsource ' . $source->source_name . ", version $version" };
469 my $rs_install_file =
470 $self->_resultsource_install_filename($source->source_name);
473 $self->$rs_install_file(
474 $self->storage->sqlt_type,
478 $self->_run_sql_and_perl($files);
481 sub prepare_resultsource_install {
483 my $source = (shift @_)->{result_source};
484 log_info { 'preparing install for resultsource ' . $source->source_name };
486 my $install_filename = $self->_resultsource_install_filename($source->source_name);
487 my $proto_filename = $self->_resultsource_protoschema_filename($source->source_name);
488 $self->prepare_protoschema({
489 parser_args => { sources => [$source->source_name], }
491 $self->_prepare_install({}, $proto_filename, $install_filename);
495 log_info { 'preparing deploy' };
497 $self->prepare_protoschema({}, '_ddl_protoschema_produce_filename');
498 $self->_prepare_install({}, '_ddl_protoschema_produce_filename', '_ddl_schema_produce_filename');
501 sub prepare_upgrade {
502 my ($self, $args) = @_;
504 "preparing upgrade from $args->{from_version} to $args->{to_version}"
506 $self->_prepare_changegrade(
507 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'up'
511 sub prepare_downgrade {
512 my ($self, $args) = @_;
514 "preparing downgrade from $args->{from_version} to $args->{to_version}"
516 $self->_prepare_changegrade(
517 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'down'
521 method _coderefs_per_files($files) {
522 no warnings 'redefine';
523 [map eval do { local( @ARGV, $/ ) = $_; <> }, @$files]
526 method _prepare_changegrade($from_version, $to_version, $version_set, $direction) {
527 my $schema = $self->schema;
528 my $databases = $self->databases;
529 my $dir = $self->script_directory;
531 my $schema_version = $self->schema_version;
532 my $diff_file_method = "_ddl_schema_${direction}_produce_filename";
533 foreach my $db (@$databases) {
534 my $diff_file = $self->$diff_file_method($db, $version_set, $dir );
536 carp("Overwriting existing $direction-diff file - $diff_file");
540 open my $file, q(>), $diff_file;
541 print {$file} join ";\n", @{$self->_sqldiff_from_yaml($from_version, $to_version, $db, $direction)};
546 method _read_sql_file($file) {
549 open my $fh, '<', $file;
550 my @data = split /;\n/, join '', <$fh>;
554 $_ && # remove blank lines
555 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
557 s/^\s+//; s/\s+$//; # trim whitespace
558 join '', grep { !/^--/ } split /\n/ # remove comments
564 sub downgrade_single_step {
566 my $version_set = (shift @_)->{version_set};
567 Dlog_info { "downgrade_single_step'ing $_" } $version_set;
569 my $sqlt_type = $self->storage->sqlt_type;
571 if ($self->ignore_ddl) {
572 $sql_to_run = $self->_sqldiff_from_yaml(
573 $version_set->[0], $version_set->[1], $sqlt_type, 'down',
576 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_down_consume_filenames(
584 sub upgrade_single_step {
586 my $version_set = (shift @_)->{version_set};
587 Dlog_info { "upgrade_single_step'ing $_" } $version_set;
589 my $sqlt_type = $self->storage->sqlt_type;
591 if ($self->ignore_ddl) {
592 $sql_to_run = $self->_sqldiff_from_yaml(
593 $version_set->[0], $version_set->[1], $sqlt_type, 'up',
596 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_up_consume_filenames(
603 sub prepare_protoschema {
605 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
608 = $self->$to_file($self->schema_version);
610 # we do this because the code that uses this sets parser args,
611 # so we just need to merge in the package
612 $sqltargs->{parser_args}{package} = $self->schema;
613 my $sqlt = SQL::Translator->new({
614 parser => 'SQL::Translator::Parser::DBIx::Class',
615 producer => 'SQL::Translator::Producer::YAML',
619 my $yml = $sqlt->translate;
621 croak("Failed to translate to YAML: " . $sqlt->error)
625 carp "Overwriting existing DDL-YML file - $filename";
629 open my $file, q(>), $filename;
634 __PACKAGE__->meta->make_immutable;
638 # vim: ts=2 sw=2 expandtab
644 This class is the meat of L<DBIx::Class::DeploymentHandler>. It takes care
645 of generating serialized schemata as well as sql files to move from one
646 version of a schema to the rest. One of the hallmark features of this class
647 is that it allows for multiple sql files for deploy and upgrade, allowing
648 developers to fine tune deployment. In addition it also allows for perl
649 files to be run at any stage of the process.
651 For basic usage see L<DBIx::Class::DeploymentHandler::HandlesDeploy>. What's
652 documented here is extra fun stuff or private methods.
654 =head1 DIRECTORY LAYOUT
656 Arguably this is the best feature of L<DBIx::Class::DeploymentHandler>. It's
657 heavily based upon L<DBIx::Migration::Directories>, but has some extensions and
658 modifications, so even if you are familiar with it, please read this. I feel
659 like the best way to describe the layout is with the following example:
685 | | `- 002-remove-customers.pl
688 | `- 002-generate-customers.pl
699 | `- 002-create-stored-procedures.sql
706 | |- 001-create_database.pl
707 | `- 002-create_users_and_permissions.pl
715 So basically, the code
719 on an C<SQLite> database that would simply run
720 C<$sql_migration_dir/SQLite/schema/1/001-auto.sql>. Next,
722 $dm->upgrade_single_step([1,2])
724 would run C<$sql_migration_dir/SQLite/up/1-2/001-auto.sql> followed by
725 C<$sql_migration_dir/_common/up/1-2/002-generate-customers.pl>.
727 C<.pl> files don't have to be in the C<_common> directory, but most of the time
728 they should be, because perl scripts are generally be database independent.
730 C<_generic> exists for when you for some reason are sure that your SQL is
731 generic enough to run on all databases. Good luck with that one.
733 Note that unlike most steps in the process, C<preinstall> will not run SQL, as
734 there may not even be an database at preinstall time. It will run perl scripts
735 just like the other steps in the process, but nothing is passed to them.
736 Until people have used this more it will remain freeform, but a recommended use
737 of preinstall is to have it prompt for username and password, and then call the
738 appropriate C<< CREATE DATABASE >> commands etc.
740 =head2 Directory Specification
742 The following subdirectories are recognized by this DeployMethod:
746 =item C<_protoschema> This directory can have the following subdirs:
750 =item C<down> This directory merely contains directories named after
751 migrations, which are of the form C<$from_version-$to_version>. Inside of
752 these directories you may put Perl scripts which are to return a subref
753 that takes the arguments C<< $from_schema, $to_schema >>, which are
754 L<SQL::Translator::Schema> objects.
756 =item C<up> This directory merely contains directories named after
757 migrations, which are of the form C<$from_version-$to_version>. Inside of
758 these directories you may put Perl scripts which are to return a subref
759 that takes the arguments C<< $from_schema, $to_schema >>, which are
760 L<SQL::Translator::Schema> objects.
762 =item C<schema> This directory merely contains directories named after schema
763 versions, which in turn contain C<yaml> files that are serialized versions
764 of the schema at that version. These files are not for editing by hand.
772 A perl script for this tool is very simple. It merely needs to contain an
773 anonymous sub that takes a L<DBIx::Class::Schema> as it's only argument.
774 A very basic perl script might look like:
784 $schema->resultset('Users')->create({
792 The L<DBIx::Class::Schema> (B<required>) that is used to talk to the database
793 and generate the DDL.
797 The L<DBIx::Class::Storage> that is I<actually> used to talk to the database
798 and generate the DDL. This is automatically created with L</_build_storage>.
800 =attr sql_translator_args
802 The arguments that get passed to L<SQL::Translator> when it's used.
804 =attr script_directory
806 The directory (default C<'sql'>) that scripts are stored in
810 The types of databases (default C<< [qw( MySQL SQLite PostgreSQL )] >>) to
815 Set to true (which is the default) to wrap all upgrades and deploys in a single
820 The version the schema on your harddrive is at. Defaults to
821 C<< $self->schema->schema_version >>.
825 =head2 __ddl_consume_with_prefix
827 $dm->__ddl_consume_with_prefix( 'SQLite', [qw( 1.00 1.01 )], 'up' )
829 This is the meat of the multi-file upgrade/deploy stuff. It returns a list of
830 files in the order that they should be run for a generic "type" of upgrade.
831 You should not be calling this in user code.
833 =head2 _ddl_schema_consume_filenames
835 $dm->__ddl_schema_consume_filenames( 'SQLite', [qw( 1.00 )] )
837 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
840 =head2 _ddl_schema_produce_filename
842 $dm->__ddl_schema_produce_filename( 'SQLite', [qw( 1.00 )] )
844 Returns a single file in which an initial schema will be stored.
846 =head2 _ddl_schema_up_consume_filenames
848 $dm->_ddl_schema_up_consume_filenames( 'SQLite', [qw( 1.00 )] )
850 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
853 =head2 _ddl_schema_down_consume_filenames
855 $dm->_ddl_schema_down_consume_filenames( 'SQLite', [qw( 1.00 )] )
857 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for a
860 =head2 _ddl_schema_up_produce_filenames
862 $dm->_ddl_schema_up_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
864 Returns a single file in which the sql to upgrade from one schema to another
867 =head2 _ddl_schema_down_produce_filename
869 $dm->_ddl_schema_down_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
871 Returns a single file in which the sql to downgrade from one schema to another
874 =head2 _resultsource_install_filename
876 my $filename_fn = $dm->_resultsource_install_filename('User');
877 $dm->$filename_fn('SQLite', '1.00')
879 Returns a function which in turn returns a single filename used to install a
880 single resultsource. Weird interface is convenient for me. Deal with it.
882 =head2 _run_sql_and_perl
884 $dm->_run_sql_and_perl([qw( list of filenames )])
886 Simply put, this runs the list of files passed to it. If the file ends in
887 C<.sql> it runs it as sql and if it ends in C<.pl> it runs it as a perl file.
889 Depending on L</txn_wrap> all of the files run will be wrapped in a single
892 =head2 _prepare_install
894 $dm->_prepare_install({ add_drop_table => 0 }, sub { 'file_to_create' })
896 Generates the sql file for installing the database. First arg is simply
897 L<SQL::Translator> args and the second is a coderef that returns the filename
900 =head2 _prepare_changegrade
902 $dm->_prepare_changegrade('1.00', '1.01', [qw( 1.00 1.01)], 'up')
904 Generates the sql file for migrating from one schema version to another. First
905 arg is the version to start from, second is the version to go to, third is the
906 L<version set|DBIx::Class::DeploymentHandler/VERSION SET>, and last is the
907 direction of the changegrade, be it 'up' or 'down'.
909 =head2 _read_sql_file
911 $dm->_read_sql_file('foo.sql')
913 Reads a sql file and returns lines in an C<ArrayRef>. Strips out comments,
914 transactions, and blank lines.