1 package DBIx::Class::DeploymentHandler::DeployMethod::SQL::Translator;
4 # ABSTRACT: Manage your SQL and Perl migrations in nicely laid out directories
7 use Carp qw( carp croak );
8 use DBIx::Class::DeploymentHandler::Logger;
9 use Log::Contextual qw(:log :dlog), -default_logger =>
10 DBIx::Class::DeploymentHandler::Logger->new({
11 env_prefix => 'DBICDH'
14 use Method::Signatures::Simple;
18 require SQL::Translator::Diff;
20 require DBIx::Class::Storage; # loaded for type constraint
21 use DBIx::Class::DeploymentHandler::Types;
23 use File::Path 'mkpath';
24 use File::Spec::Functions;
26 with 'DBIx::Class::DeploymentHandler::HandlesDeploy';
35 isa => 'DBIx::Class::Schema',
41 isa => 'DBIx::Class::Storage',
46 method _build_storage {
47 my $s = $self->schema->storage;
48 $s->_determine_driver;
52 has sql_translator_args => (
55 default => sub { {} },
57 has script_directory => (
66 isa => 'DBIx::Class::DeploymentHandler::Databases',
68 default => sub { [qw( MySQL SQLite PostgreSQL )] },
77 has schema_version => (
83 # this will probably never get called as the DBICDH
84 # will be passing down a schema_version normally, which
85 # is built the same way, but we leave this in place
86 method _build_schema_version { $self->schema->schema_version }
88 method __ddl_consume_with_prefix($type, $versions, $prefix) {
89 my $base_dir = $self->script_directory;
91 my $main = catfile( $base_dir, $type );
92 my $generic = catfile( $base_dir, '_generic' );
94 catfile( $base_dir, '_common', $prefix, join q(-), @{$versions} );
98 $dir = catfile($main, $prefix, join q(-), @{$versions})
99 } elsif (-d $generic) {
100 $dir = catfile($generic, $prefix, join q(-), @{$versions});
102 croak "neither $main or $generic exist; please write/generate some SQL";
105 opendir my($dh), $dir;
106 my %files = map { $_ => "$dir/$_" } grep { /\.(?:sql|pl|sql-\w+)$/ && -f "$dir/$_" } readdir $dh;
110 opendir my($dh), $common;
111 for my $filename (grep { /\.(?:sql|pl)$/ && -f catfile($common,$_) } readdir $dh) {
112 unless ($files{$filename}) {
113 $files{$filename} = catfile($common,$filename);
119 return [@files{sort keys %files}]
122 method _ddl_preinstall_consume_filenames($type, $version) {
123 $self->__ddl_consume_with_prefix($type, [ $version ], 'preinstall')
126 method _ddl_schema_consume_filenames($type, $version) {
127 $self->__ddl_consume_with_prefix($type, [ $version ], 'schema')
130 method _ddl_protoschema_produce_filename($version) {
131 my $dirname = catfile( $self->script_directory, '_protoschema', $version );
132 mkpath($dirname) unless -d $dirname;
134 return catfile( $dirname, '001-auto.yml' );
137 method _ddl_schema_produce_filename($type, $version) {
138 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
139 mkpath($dirname) unless -d $dirname;
141 return catfile( $dirname, '001-auto.sql' );
144 method _ddl_schema_up_consume_filenames($type, $versions) {
145 $self->__ddl_consume_with_prefix($type, $versions, 'up')
148 method _ddl_schema_down_consume_filenames($type, $versions) {
149 $self->__ddl_consume_with_prefix($type, $versions, 'down')
152 method _ddl_schema_up_produce_filename($type, $versions) {
153 my $dir = $self->script_directory;
155 my $dirname = catfile( $dir, $type, 'up', join q(-), @{$versions});
156 mkpath($dirname) unless -d $dirname;
158 return catfile( $dirname, '001-auto.sql' );
161 method _ddl_schema_down_produce_filename($type, $versions, $dir) {
162 my $dirname = catfile( $dir, $type, 'down', join q(-), @{$versions} );
163 mkpath($dirname) unless -d $dirname;
165 return catfile( $dirname, '001-auto.sql');
168 method _run_sql_array($sql) {
169 my $storage = $self->storage;
172 $_ && # remove blank lines
173 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
175 s/^\s+//; s/\s+$//; # trim whitespace
176 join '', grep { !/^--/ } split /\n/ # remove comments
179 Dlog_trace { "Running SQL $_" } $sql;
180 foreach my $line (@{$sql}) {
181 $storage->_query_start($line);
182 # the whole reason we do this is so that we can see the line that was run
184 $storage->dbh_do (sub { $_[1]->do($line) });
187 die "$_ (running line '$line')"
189 $storage->_query_end($line);
191 return join "\n", @$sql
194 method _run_sql($filename) {
195 log_debug { "Running SQL from $filename" };
196 return $self->_run_sql_array($self->_read_sql_file($filename));
199 method _run_perl($filename) {
200 log_debug { "Running Perl from $filename" };
201 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
203 no warnings 'redefine';
204 my $fn = eval "$filedata";
206 Dlog_trace { "Running Perl $_" } $fn;
209 carp "$filename failed to compile: $@";
210 } elsif (ref $fn eq 'CODE') {
213 carp "$filename should define an anonymouse sub that takes a schema but it didn't!";
217 method _run_sql_and_perl($filenames) {
218 my @files = @{$filenames};
219 my $guard = $self->schema->txn_scope_guard if $self->txn_wrap;
222 for my $filename (@files) {
223 if ($filename =~ /\.sql$/) {
224 $sql .= $self->_run_sql($filename)
225 } elsif ( $filename =~ /\.pl$/ ) {
226 $self->_run_perl($filename)
228 croak "A file ($filename) got to deploy that wasn't sql or perl!";
232 $guard->commit if $self->txn_wrap;
237 method _deploy($version) {
238 return $self->_run_sql_and_perl($self->_ddl_schema_consume_filenames(
239 $self->storage->sqlt_type,
246 my $version = (shift @_ || {})->{version} || $self->schema_version;
247 log_info { "deploying version $version" };
248 $self->_deploy($version);
254 my $version = $args->{version} || $self->schema_version;
255 log_info { "preinstalling version $version" };
256 my $storage_type = $args->{storage_type} || $self->storage->sqlt_type;
258 my @files = @{$self->_ddl_preinstall_consume_filenames(
263 for my $filename (@files) {
264 # We ignore sql for now (till I figure out what to do with it)
265 if ( $filename =~ /^(.+)\.pl$/ ) {
266 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
268 no warnings 'redefine';
269 my $fn = eval "$filedata";
273 carp "$filename failed to compile: $@";
274 } elsif (ref $fn eq 'CODE') {
277 carp "$filename should define an anonymous sub but it didn't!";
280 croak "A file ($filename) got to preinstall_scripts that wasn't sql or perl!";
285 method _sqldiff_from_yaml($from_version, $to_version, $db) {
286 my $dir = $self->script_directory;
289 ignore_constraint_names => 1,
290 ignore_index_names => 1,
291 %{$self->sql_translator_args}
296 my $prefilename = $self->_ddl_protoschema_produce_filename($from_version, $dir);
298 # should probably be a croak
299 carp("No previous schema file found ($prefilename)")
300 unless -e $prefilename;
302 my $t = SQL::Translator->new({
306 parser => 'SQL::Translator::Parser::YAML',
309 my $out = $t->translate( $prefilename )
312 $source_schema = $t->schema;
314 $source_schema->name( $prefilename )
315 unless $source_schema->name;
320 my $filename = $self->_ddl_protoschema_produce_filename($to_version, $dir);
322 # should probably be a croak
323 carp("No next schema file found ($filename)")
326 my $t = SQL::Translator->new({
330 parser => 'SQL::Translator::Parser::YAML',
333 my $out = $t->translate( $filename )
336 $dest_schema = $t->schema;
338 $dest_schema->name( $filename )
339 unless $dest_schema->name;
341 return [SQL::Translator::Diff::schema_diff(
348 method _sql_from_yaml($sqltargs, $from_file, $db) {
349 my $schema = $self->schema;
350 my $version = $self->schema_version;
352 my $sqlt = SQL::Translator->new({
354 parser => 'SQL::Translator::Parser::YAML',
359 my $yaml_filename = $self->$from_file($version);
361 my @sql = $sqlt->translate($yaml_filename);
363 carp("Failed to translate to $db, skipping. (" . $sqlt->error . ")");
369 sub _prepare_install {
371 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
372 my $from_file = shift;
374 my $dir = $self->script_directory;
375 my $databases = $self->databases;
376 my $version = $self->schema_version;
378 foreach my $db (@$databases) {
379 my $sql = $self->_sql_from_yaml($sqltargs, $from_file, $db ) or next;
381 my $filename = $self->$to_file($db, $version, $dir);
383 carp "Overwriting existing DDL file - $filename";
386 open my $file, q(>), $filename;
387 print {$file} join ";\n", @$sql;
392 sub _resultsource_install_filename {
393 my ($self, $source_name) = @_;
395 my ($self, $type, $version) = @_;
396 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
397 mkpath($dirname) unless -d $dirname;
399 return catfile( $dirname, "001-auto-$source_name.sql" );
403 sub _resultsource_protoschema_filename {
404 my ($self, $source_name) = @_;
406 my ($self, $version) = @_;
407 my $dirname = catfile( $self->script_directory, '_protoschema', $version );
408 mkpath($dirname) unless -d $dirname;
410 return catfile( $dirname, "001-auto-$source_name.yml" );
414 sub install_resultsource {
415 my ($self, $args) = @_;
416 my $source = $args->{result_source};
417 my $version = $args->{version};
418 log_info { 'installing_resultsource ' . $source->source_name . ", version $version" };
419 my $rs_install_file =
420 $self->_resultsource_install_filename($source->source_name);
423 $self->$rs_install_file(
424 $self->storage->sqlt_type,
428 $self->_run_sql_and_perl($files);
431 sub prepare_resultsource_install {
433 my $source = (shift @_)->{result_source};
434 log_info { 'preparing install for resultsource ' . $source->source_name };
436 my $install_filename = $self->_resultsource_install_filename($source->source_name);
437 my $proto_filename = $self->_resultsource_protoschema_filename($source->source_name);
438 $self->prepare_protoschema({
439 parser_args => { sources => [$source->source_name], }
441 $self->_prepare_install({}, $proto_filename, $install_filename);
445 log_info { 'preparing deploy' };
447 $self->prepare_protoschema({}, '_ddl_protoschema_produce_filename');
448 $self->_prepare_install({}, '_ddl_protoschema_produce_filename', '_ddl_schema_produce_filename');
451 sub prepare_upgrade {
452 my ($self, $args) = @_;
454 "preparing upgrade from $args->{from_version} to $args->{to_version}"
456 $self->_prepare_changegrade(
457 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'up'
461 sub prepare_downgrade {
462 my ($self, $args) = @_;
464 "preparing downgrade from $args->{from_version} to $args->{to_version}"
466 $self->_prepare_changegrade(
467 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'down'
471 method _prepare_changegrade($from_version, $to_version, $version_set, $direction) {
472 my $schema = $self->schema;
473 my $databases = $self->databases;
474 my $dir = $self->script_directory;
476 return if $self->ignore_ddl;
478 my $schema_version = $self->schema_version;
479 my $diff_file_method = "_ddl_schema_${direction}_produce_filename";
480 foreach my $db (@$databases) {
481 my $diff_file = $self->$diff_file_method($db, $version_set, $dir );
483 carp("Overwriting existing $direction-diff file - $diff_file");
487 open my $file, q(>), $diff_file;
488 print {$file} join ";\n", @{$self->_sqldiff_from_yaml($from_version, $to_version, $db)};
493 method _read_sql_file($file) {
496 open my $fh, '<', $file;
497 my @data = split /;\n/, join '', <$fh>;
501 $_ && # remove blank lines
502 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
504 s/^\s+//; s/\s+$//; # trim whitespace
505 join '', grep { !/^--/ } split /\n/ # remove comments
511 sub downgrade_single_step {
513 my $version_set = (shift @_)->{version_set};
514 Dlog_info { "downgrade_single_step'ing $_" } $version_set;
516 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_down_consume_filenames(
517 $self->storage->sqlt_type,
524 sub upgrade_single_step {
526 my $version_set = (shift @_)->{version_set};
527 Dlog_info { "upgrade_single_step'ing $_" } $version_set;
529 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_up_consume_filenames(
530 $self->storage->sqlt_type,
536 sub prepare_protoschema {
538 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
541 = $self->$to_file($self->schema_version);
543 # we do this because the code that uses this sets parser args,
544 # so we just need to merge in the package
545 $sqltargs->{parser_args}{package} = $self->schema;
546 my $sqlt = SQL::Translator->new({
547 parser => 'SQL::Translator::Parser::DBIx::Class',
548 producer => 'SQL::Translator::Producer::YAML',
552 my $yml = $sqlt->translate;
554 croak("Failed to translate to YAML: " . $sqlt->error)
558 carp "Overwriting existing DDL-YML file - $filename";
562 open my $file, q(>), $filename;
567 __PACKAGE__->meta->make_immutable;
571 # vim: ts=2 sw=2 expandtab
577 This class is the meat of L<DBIx::Class::DeploymentHandler>. It takes care
578 of generating serialized schemata as well as sql files to move from one
579 version of a schema to the rest. One of the hallmark features of this class
580 is that it allows for multiple sql files for deploy and upgrade, allowing
581 developers to fine tune deployment. In addition it also allows for perl
582 files to be run at any stage of the process.
584 For basic usage see L<DBIx::Class::DeploymentHandler::HandlesDeploy>. What's
585 documented here is extra fun stuff or private methods.
587 =head1 DIRECTORY LAYOUT
589 Arguably this is the best feature of L<DBIx::Class::DeploymentHandler>. It's
590 heavily based upon L<DBIx::Migration::Directories>, but has some extensions and
591 modifications, so even if you are familiar with it, please read this. I feel
592 like the best way to describe the layout is with the following example:
610 | | `- 002-remove-customers.pl
613 | `- 002-generate-customers.pl
624 | `- 002-create-stored-procedures.sql
631 | |- 001-create_database.pl
632 | `- 002-create_users_and_permissions.pl
640 So basically, the code
644 on an C<SQLite> database that would simply run
645 C<$sql_migration_dir/SQLite/schema/1/001-auto.sql>. Next,
647 $dm->upgrade_single_step([1,2])
649 would run C<$sql_migration_dir/SQLite/up/1-2/001-auto.sql> followed by
650 C<$sql_migration_dir/_common/up/1-2/002-generate-customers.pl>.
652 C<.pl> files don't have to be in the C<_common> directory, but most of the time
653 they should be, because perl scripts are generally be database independent.
655 C<_generic> exists for when you for some reason are sure that your SQL is
656 generic enough to run on all databases. Good luck with that one.
658 Note that unlike most steps in the process, C<preinstall> will not run SQL, as
659 there may not even be an database at preinstall time. It will run perl scripts
660 just like the other steps in the process, but nothing is passed to them.
661 Until people have used this more it will remain freeform, but a recommended use
662 of preinstall is to have it prompt for username and password, and then call the
663 appropriate C<< CREATE DATABASE >> commands etc.
667 A perl script for this tool is very simple. It merely needs to contain an
668 anonymous sub that takes a L<DBIx::Class::Schema> as it's only argument.
669 A very basic perl script might look like:
679 $schema->resultset('Users')->create({
687 The L<DBIx::Class::Schema> (B<required>) that is used to talk to the database
688 and generate the DDL.
692 The L<DBIx::Class::Storage> that is I<actually> used to talk to the database
693 and generate the DDL. This is automatically created with L</_build_storage>.
695 =attr sql_translator_args
697 The arguments that get passed to L<SQL::Translator> when it's used.
699 =attr script_directory
701 The directory (default C<'sql'>) that scripts are stored in
705 The types of databases (default C<< [qw( MySQL SQLite PostgreSQL )] >>) to
710 Set to true (which is the default) to wrap all upgrades and deploys in a single
715 The version the schema on your harddrive is at. Defaults to
716 C<< $self->schema->schema_version >>.
720 =head2 __ddl_consume_with_prefix
722 $dm->__ddl_consume_with_prefix( 'SQLite', [qw( 1.00 1.01 )], 'up' )
724 This is the meat of the multi-file upgrade/deploy stuff. It returns a list of
725 files in the order that they should be run for a generic "type" of upgrade.
726 You should not be calling this in user code.
728 =head2 _ddl_schema_consume_filenames
730 $dm->__ddl_schema_consume_filenames( 'SQLite', [qw( 1.00 )] )
732 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
735 =head2 _ddl_schema_produce_filename
737 $dm->__ddl_schema_produce_filename( 'SQLite', [qw( 1.00 )] )
739 Returns a single file in which an initial schema will be stored.
741 =head2 _ddl_schema_up_consume_filenames
743 $dm->_ddl_schema_up_consume_filenames( 'SQLite', [qw( 1.00 )] )
745 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
748 =head2 _ddl_schema_down_consume_filenames
750 $dm->_ddl_schema_down_consume_filenames( 'SQLite', [qw( 1.00 )] )
752 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for a
755 =head2 _ddl_schema_up_produce_filenames
757 $dm->_ddl_schema_up_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
759 Returns a single file in which the sql to upgrade from one schema to another
762 =head2 _ddl_schema_down_produce_filename
764 $dm->_ddl_schema_down_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
766 Returns a single file in which the sql to downgrade from one schema to another
769 =head2 _resultsource_install_filename
771 my $filename_fn = $dm->_resultsource_install_filename('User');
772 $dm->$filename_fn('SQLite', '1.00')
774 Returns a function which in turn returns a single filename used to install a
775 single resultsource. Weird interface is convenient for me. Deal with it.
777 =head2 _run_sql_and_perl
779 $dm->_run_sql_and_perl([qw( list of filenames )])
781 Simply put, this runs the list of files passed to it. If the file ends in
782 C<.sql> it runs it as sql and if it ends in C<.pl> it runs it as a perl file.
784 Depending on L</txn_wrap> all of the files run will be wrapped in a single
787 =head2 _prepare_install
789 $dm->_prepare_install({ add_drop_table => 0 }, sub { 'file_to_create' })
791 Generates the sql file for installing the database. First arg is simply
792 L<SQL::Translator> args and the second is a coderef that returns the filename
795 =head2 _prepare_changegrade
797 $dm->_prepare_changegrade('1.00', '1.01', [qw( 1.00 1.01)], 'up')
799 Generates the sql file for migrating from one schema version to another. First
800 arg is the version to start from, second is the version to go to, third is the
801 L<version set|DBIx::Class::DeploymentHandler/VERSION SET>, and last is the
802 direction of the changegrade, be it 'up' or 'down'.
804 =head2 _read_sql_file
806 $dm->_read_sql_file('foo.sql')
808 Reads a sql file and returns lines in an C<ArrayRef>. Strips out comments,
809 transactions, and blank lines.