1 package DBIx::Class::DeploymentHandler::DeployMethod::SQL::Translator;
4 # ABSTRACT: Manage your SQL and Perl migrations in nicely laid out directories
7 use Carp qw( carp croak );
8 use DBIx::Class::DeploymentHandler::Logger;
9 use Log::Contextual qw(:log :dlog), -default_logger =>
10 DBIx::Class::DeploymentHandler::Logger->new({
11 env_prefix => 'DBICDH'
14 use Method::Signatures::Simple;
18 require SQL::Translator::Diff;
20 require DBIx::Class::Storage; # loaded for type constraint
21 use DBIx::Class::DeploymentHandler::Types;
23 use File::Path 'mkpath';
24 use File::Spec::Functions;
26 with 'DBIx::Class::DeploymentHandler::HandlesDeploy';
35 isa => 'DBIx::Class::Schema',
41 isa => 'DBIx::Class::Storage',
46 method _build_storage {
47 my $s = $self->schema->storage;
48 $s->_determine_driver;
52 has sql_translator_args => (
55 default => sub { {} },
57 has script_directory => (
66 isa => 'DBIx::Class::DeploymentHandler::Databases',
68 default => sub { [qw( MySQL SQLite PostgreSQL )] },
77 has schema_version => (
83 # this will probably never get called as the DBICDH
84 # will be passing down a schema_version normally, which
85 # is built the same way, but we leave this in place
86 method _build_schema_version { $self->schema->schema_version }
88 method __ddl_consume_with_prefix($type, $versions, $prefix) {
89 my $base_dir = $self->script_directory;
91 my $main = catfile( $base_dir, $type );
92 my $generic = catfile( $base_dir, '_generic' );
94 catfile( $base_dir, '_common', $prefix, join q(-), @{$versions} );
98 $dir = catfile($main, $prefix, join q(-), @{$versions})
99 } elsif (-d $generic) {
100 $dir = catfile($generic, $prefix, join q(-), @{$versions});
102 croak "neither $main or $generic exist; please write/generate some SQL";
105 opendir my($dh), $dir;
106 my %files = map { $_ => "$dir/$_" } grep { /\.(?:sql|pl|sql-\w+)$/ && -f "$dir/$_" } readdir $dh;
110 opendir my($dh), $common;
111 for my $filename (grep { /\.(?:sql|pl)$/ && -f catfile($common,$_) } readdir $dh) {
112 unless ($files{$filename}) {
113 $files{$filename} = catfile($common,$filename);
119 return [@files{sort keys %files}]
122 method _ddl_preinstall_consume_filenames($type, $version) {
123 $self->__ddl_consume_with_prefix($type, [ $version ], 'preinstall')
126 method _ddl_schema_consume_filenames($type, $version) {
127 $self->__ddl_consume_with_prefix($type, [ $version ], 'schema')
130 method _ddl_protoschema_produce_filename($version) {
131 my $dirname = catfile( $self->script_directory, '_protoschema', $version );
132 mkpath($dirname) unless -d $dirname;
134 return catfile( $dirname, '001-auto.yml' );
137 method _ddl_schema_produce_filename($type, $version) {
138 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
139 mkpath($dirname) unless -d $dirname;
141 return catfile( $dirname, '001-auto.sql' );
144 method _ddl_schema_up_consume_filenames($type, $versions) {
145 $self->__ddl_consume_with_prefix($type, $versions, 'up')
148 method _ddl_schema_down_consume_filenames($type, $versions) {
149 $self->__ddl_consume_with_prefix($type, $versions, 'down')
152 method _ddl_schema_up_produce_filename($type, $versions) {
153 my $dir = $self->script_directory;
155 my $dirname = catfile( $dir, $type, 'up', join q(-), @{$versions});
156 mkpath($dirname) unless -d $dirname;
158 return catfile( $dirname, '001-auto.sql' );
161 method _ddl_schema_down_produce_filename($type, $versions, $dir) {
162 my $dirname = catfile( $dir, $type, 'down', join q(-), @{$versions} );
163 mkpath($dirname) unless -d $dirname;
165 return catfile( $dirname, '001-auto.sql');
168 method _run_sql_array($sql) {
169 my $storage = $self->storage;
172 $_ && # remove blank lines
173 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
175 s/^\s+//; s/\s+$//; # trim whitespace
176 join '', grep { !/^--/ } split /\n/ # remove comments
179 Dlog_trace { "Running SQL $_" } $sql;
180 foreach my $line (@{$sql}) {
181 $storage->_query_start($line);
182 # the whole reason we do this is so that we can see the line that was run
184 $storage->dbh_do (sub { $_[1]->do($line) });
187 die "$_ (running line '$line')"
189 $storage->_query_end($line);
191 return join "\n", @$sql
194 method _run_sql($filename) {
195 log_debug { "Running SQL from $filename" };
196 return $self->_run_sql_array($self->_read_sql_file($filename));
199 method _run_perl($filename) {
200 log_debug { "Running Perl from $filename" };
201 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
203 no warnings 'redefine';
204 my $fn = eval "$filedata";
206 Dlog_trace { "Running Perl $_" } $fn;
209 carp "$filename failed to compile: $@";
210 } elsif (ref $fn eq 'CODE') {
213 carp "$filename should define an anonymouse sub that takes a schema but it didn't!";
217 method _run_sql_and_perl($filenames) {
218 my @files = @{$filenames};
219 my $guard = $self->schema->txn_scope_guard if $self->txn_wrap;
222 for my $filename (@files) {
223 if ($filename =~ /\.sql$/) {
224 $sql .= $self->_run_sql($filename)
225 } elsif ( $filename =~ /\.pl$/ ) {
226 $self->_run_perl($filename)
228 croak "A file ($filename) got to deploy that wasn't sql or perl!";
232 $guard->commit if $self->txn_wrap;
237 method _deploy($version) {
238 if (!$self->ignore_ddl) {
239 return $self->_run_sql_and_perl($self->_ddl_schema_consume_filenames(
240 $self->storage->sqlt_type,
244 my $sqlt = SQL::Translator->new({
246 parser => 'SQL::Translator::Parser::YAML',
247 producer => $self->storage->sqlt_type;
251 my $yaml_filename = $self->$from_file($version);
253 my @sql = $sqlt->translate($yaml_filename);
254 croak("Failed to translate to $db, skipping. (" . $sqlt->error . ")")
261 my $version = (shift @_ || {})->{version} || $self->schema_version;
262 log_info { "deploying version $version" };
263 $self->_deploy($version);
269 my $version = $args->{version} || $self->schema_version;
270 log_info { "preinstalling version $version" };
271 my $storage_type = $args->{storage_type} || $self->storage->sqlt_type;
273 my @files = @{$self->_ddl_preinstall_consume_filenames(
278 for my $filename (@files) {
279 # We ignore sql for now (till I figure out what to do with it)
280 if ( $filename =~ /^(.+)\.pl$/ ) {
281 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
283 no warnings 'redefine';
284 my $fn = eval "$filedata";
288 carp "$filename failed to compile: $@";
289 } elsif (ref $fn eq 'CODE') {
292 carp "$filename should define an anonymous sub but it didn't!";
295 croak "A file ($filename) got to preinstall_scripts that wasn't sql or perl!";
300 sub _prepare_install {
302 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
303 my $from_file = shift;
305 my $schema = $self->schema;
306 my $databases = $self->databases;
307 my $dir = $self->script_directory;
308 my $version = $self->schema_version;
310 return if $self->ignore_ddl;
312 my $sqlt = SQL::Translator->new({
314 parser => 'SQL::Translator::Parser::YAML',
318 my $yaml_filename = $self->$from_file($version);
320 foreach my $db (@$databases) {
322 $sqlt->producer($db);
324 my $filename = $self->$to_file($db, $version, $dir);
326 carp "Overwriting existing DDL file - $filename";
330 my $sql = $sqlt->translate($yaml_filename);
332 carp("Failed to translate to $db, skipping. (" . $sqlt->error . ")");
335 open my $file, q(>), $filename;
341 sub _resultsource_install_filename {
342 my ($self, $source_name) = @_;
344 my ($self, $type, $version) = @_;
345 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
346 mkpath($dirname) unless -d $dirname;
348 return catfile( $dirname, "001-auto-$source_name.sql" );
352 sub _resultsource_protoschema_filename {
353 my ($self, $source_name) = @_;
355 my ($self, $version) = @_;
356 my $dirname = catfile( $self->script_directory, '_protoschema', $version );
357 mkpath($dirname) unless -d $dirname;
359 return catfile( $dirname, "001-auto-$source_name.yml" );
363 sub install_resultsource {
364 my ($self, $args) = @_;
365 my $source = $args->{result_source};
366 my $version = $args->{version};
367 log_info { 'installing_resultsource ' . $source->source_name . ", version $version" };
368 my $rs_install_file =
369 $self->_resultsource_install_filename($source->source_name);
372 $self->$rs_install_file(
373 $self->storage->sqlt_type,
377 $self->_run_sql_and_perl($files);
380 sub prepare_resultsource_install {
382 my $source = (shift @_)->{result_source};
383 log_info { 'preparing install for resultsource ' . $source->source_name };
385 my $install_filename = $self->_resultsource_install_filename($source->source_name);
386 my $proto_filename = $self->_resultsource_protoschema_filename($source->source_name);
387 $self->prepare_protoschema({
388 parser_args => { sources => [$source->source_name], }
390 $self->_prepare_install({}, $proto_filename, $install_filename);
394 log_info { 'preparing deploy' };
396 $self->prepare_protoschema({}, '_ddl_protoschema_produce_filename');
397 $self->_prepare_install({}, '_ddl_protoschema_produce_filename', '_ddl_schema_produce_filename');
400 sub prepare_upgrade {
401 my ($self, $args) = @_;
403 "preparing upgrade from $args->{from_version} to $args->{to_version}"
405 $self->_prepare_changegrade(
406 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'up'
410 sub prepare_downgrade {
411 my ($self, $args) = @_;
413 "preparing downgrade from $args->{from_version} to $args->{to_version}"
415 $self->_prepare_changegrade(
416 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'down'
420 method _prepare_changegrade($from_version, $to_version, $version_set, $direction) {
421 my $schema = $self->schema;
422 my $databases = $self->databases;
423 my $dir = $self->script_directory;
424 my $sqltargs = $self->sql_translator_args;
426 return if $self->ignore_ddl;
428 my $schema_version = $self->schema_version;
432 ignore_constraint_names => 1,
433 ignore_index_names => 1,
437 my $diff_file_method = "_ddl_schema_${direction}_produce_filename";
440 my $prefilename = $self->_ddl_protoschema_produce_filename($from_version, $dir);
442 # should probably be a croak
443 carp("No previous schema file found ($prefilename)")
444 unless -e $prefilename;
446 my $t = SQL::Translator->new({
450 parser => 'SQL::Translator::Parser::YAML',
453 my $out = $t->translate( $prefilename )
456 $source_schema = $t->schema;
458 $source_schema->name( $prefilename )
459 unless $source_schema->name;
464 my $filename = $self->_ddl_protoschema_produce_filename($to_version, $dir);
466 # should probably be a croak
467 carp("No next schema file found ($filename)")
470 my $t = SQL::Translator->new({
474 parser => 'SQL::Translator::Parser::YAML',
477 my $out = $t->translate( $filename )
480 $dest_schema = $t->schema;
482 $dest_schema->name( $filename )
483 unless $dest_schema->name;
485 foreach my $db (@$databases) {
486 my $diff_file = $self->$diff_file_method($db, $version_set, $dir );
488 carp("Overwriting existing $direction-diff file - $diff_file");
492 my $diff = SQL::Translator::Diff::schema_diff(
497 open my $file, q(>), $diff_file;
503 method _read_sql_file($file) {
506 open my $fh, '<', $file;
507 my @data = split /;\n/, join '', <$fh>;
511 $_ && # remove blank lines
512 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
514 s/^\s+//; s/\s+$//; # trim whitespace
515 join '', grep { !/^--/ } split /\n/ # remove comments
521 sub downgrade_single_step {
523 my $version_set = (shift @_)->{version_set};
524 Dlog_info { "downgrade_single_step'ing $_" } $version_set;
526 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_down_consume_filenames(
527 $self->storage->sqlt_type,
534 sub upgrade_single_step {
536 my $version_set = (shift @_)->{version_set};
537 Dlog_info { "upgrade_single_step'ing $_" } $version_set;
539 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_up_consume_filenames(
540 $self->storage->sqlt_type,
546 sub prepare_protoschema {
548 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
551 = $self->$to_file($self->schema_version);
553 # we do this because the code that uses this sets parser args,
554 # so we just need to merge in the package
555 $sqltargs->{parser_args}{package} = $self->schema;
556 my $sqlt = SQL::Translator->new({
557 parser => 'SQL::Translator::Parser::DBIx::Class',
558 producer => 'SQL::Translator::Producer::YAML',
562 my $yml = $sqlt->translate;
564 croak("Failed to translate to YAML: " . $sqlt->error)
568 carp "Overwriting existing DDL-YML file - $filename";
572 open my $file, q(>), $filename;
577 __PACKAGE__->meta->make_immutable;
581 # vim: ts=2 sw=2 expandtab
587 This class is the meat of L<DBIx::Class::DeploymentHandler>. It takes care
588 of generating serialized schemata as well as sql files to move from one
589 version of a schema to the rest. One of the hallmark features of this class
590 is that it allows for multiple sql files for deploy and upgrade, allowing
591 developers to fine tune deployment. In addition it also allows for perl
592 files to be run at any stage of the process.
594 For basic usage see L<DBIx::Class::DeploymentHandler::HandlesDeploy>. What's
595 documented here is extra fun stuff or private methods.
597 =head1 DIRECTORY LAYOUT
599 Arguably this is the best feature of L<DBIx::Class::DeploymentHandler>. It's
600 heavily based upon L<DBIx::Migration::Directories>, but has some extensions and
601 modifications, so even if you are familiar with it, please read this. I feel
602 like the best way to describe the layout is with the following example:
620 | | `- 002-remove-customers.pl
623 | `- 002-generate-customers.pl
634 | `- 002-create-stored-procedures.sql
641 | |- 001-create_database.pl
642 | `- 002-create_users_and_permissions.pl
650 So basically, the code
654 on an C<SQLite> database that would simply run
655 C<$sql_migration_dir/SQLite/schema/1/001-auto.sql>. Next,
657 $dm->upgrade_single_step([1,2])
659 would run C<$sql_migration_dir/SQLite/up/1-2/001-auto.sql> followed by
660 C<$sql_migration_dir/_common/up/1-2/002-generate-customers.pl>.
662 C<.pl> files don't have to be in the C<_common> directory, but most of the time
663 they should be, because perl scripts are generally be database independent.
665 C<_generic> exists for when you for some reason are sure that your SQL is
666 generic enough to run on all databases. Good luck with that one.
668 Note that unlike most steps in the process, C<preinstall> will not run SQL, as
669 there may not even be an database at preinstall time. It will run perl scripts
670 just like the other steps in the process, but nothing is passed to them.
671 Until people have used this more it will remain freeform, but a recommended use
672 of preinstall is to have it prompt for username and password, and then call the
673 appropriate C<< CREATE DATABASE >> commands etc.
677 A perl script for this tool is very simple. It merely needs to contain an
678 anonymous sub that takes a L<DBIx::Class::Schema> as it's only argument.
679 A very basic perl script might look like:
689 $schema->resultset('Users')->create({
697 The L<DBIx::Class::Schema> (B<required>) that is used to talk to the database
698 and generate the DDL.
702 The L<DBIx::Class::Storage> that is I<actually> used to talk to the database
703 and generate the DDL. This is automatically created with L</_build_storage>.
705 =attr sql_translator_args
707 The arguments that get passed to L<SQL::Translator> when it's used.
709 =attr script_directory
711 The directory (default C<'sql'>) that scripts are stored in
715 The types of databases (default C<< [qw( MySQL SQLite PostgreSQL )] >>) to
720 Set to true (which is the default) to wrap all upgrades and deploys in a single
725 The version the schema on your harddrive is at. Defaults to
726 C<< $self->schema->schema_version >>.
730 =head2 __ddl_consume_with_prefix
732 $dm->__ddl_consume_with_prefix( 'SQLite', [qw( 1.00 1.01 )], 'up' )
734 This is the meat of the multi-file upgrade/deploy stuff. It returns a list of
735 files in the order that they should be run for a generic "type" of upgrade.
736 You should not be calling this in user code.
738 =head2 _ddl_schema_consume_filenames
740 $dm->__ddl_schema_consume_filenames( 'SQLite', [qw( 1.00 )] )
742 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
745 =head2 _ddl_schema_produce_filename
747 $dm->__ddl_schema_produce_filename( 'SQLite', [qw( 1.00 )] )
749 Returns a single file in which an initial schema will be stored.
751 =head2 _ddl_schema_up_consume_filenames
753 $dm->_ddl_schema_up_consume_filenames( 'SQLite', [qw( 1.00 )] )
755 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
758 =head2 _ddl_schema_down_consume_filenames
760 $dm->_ddl_schema_down_consume_filenames( 'SQLite', [qw( 1.00 )] )
762 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for a
765 =head2 _ddl_schema_up_produce_filenames
767 $dm->_ddl_schema_up_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
769 Returns a single file in which the sql to upgrade from one schema to another
772 =head2 _ddl_schema_down_produce_filename
774 $dm->_ddl_schema_down_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
776 Returns a single file in which the sql to downgrade from one schema to another
779 =head2 _resultsource_install_filename
781 my $filename_fn = $dm->_resultsource_install_filename('User');
782 $dm->$filename_fn('SQLite', '1.00')
784 Returns a function which in turn returns a single filename used to install a
785 single resultsource. Weird interface is convenient for me. Deal with it.
787 =head2 _run_sql_and_perl
789 $dm->_run_sql_and_perl([qw( list of filenames )])
791 Simply put, this runs the list of files passed to it. If the file ends in
792 C<.sql> it runs it as sql and if it ends in C<.pl> it runs it as a perl file.
794 Depending on L</txn_wrap> all of the files run will be wrapped in a single
797 =head2 _prepare_install
799 $dm->_prepare_install({ add_drop_table => 0 }, sub { 'file_to_create' })
801 Generates the sql file for installing the database. First arg is simply
802 L<SQL::Translator> args and the second is a coderef that returns the filename
805 =head2 _prepare_changegrade
807 $dm->_prepare_changegrade('1.00', '1.01', [qw( 1.00 1.01)], 'up')
809 Generates the sql file for migrating from one schema version to another. First
810 arg is the version to start from, second is the version to go to, third is the
811 L<version set|DBIx::Class::DeploymentHandler/VERSION SET>, and last is the
812 direction of the changegrade, be it 'up' or 'down'.
814 =head2 _read_sql_file
816 $dm->_read_sql_file('foo.sql')
818 Reads a sql file and returns lines in an C<ArrayRef>. Strips out comments,
819 transactions, and blank lines.