1 package DBIx::Class::DeploymentHandler::DeployMethod::SQL::Translator;
4 # ABSTRACT: Manage your SQL and Perl migrations in nicely laid out directories
7 use Carp qw( carp croak );
8 use DBIx::Class::DeploymentHandler::Logger;
9 use Log::Contextual qw(:log :dlog), -package_logger =>
10 DBIx::Class::DeploymentHandler::Logger->new({
11 env_prefix => 'DBICDH'
14 use Method::Signatures::Simple;
18 require SQL::Translator::Diff;
20 require DBIx::Class::Storage; # loaded for type constraint
21 use DBIx::Class::DeploymentHandler::Types;
23 use File::Path 'mkpath';
24 use File::Spec::Functions;
26 with 'DBIx::Class::DeploymentHandler::HandlesDeploy';
34 has force_overwrite => (
41 isa => 'DBIx::Class::Schema',
47 isa => 'DBIx::Class::Storage',
52 method _build_storage {
53 my $s = $self->schema->storage;
54 $s->_determine_driver;
58 has sql_translator_args => (
61 default => sub { {} },
64 has script_directory => (
71 has _filesystem_interface => (
75 _ddl_initialize_consume_filenames => '_ddl_initialize_consume_filenames',
76 _ddl_schema_consume_filenames => '_ddl_schema_consume_filenames',
77 _ddl_protoschema_deploy_consume_filenames => '_ddl_protoschema_deploy_consume_filenames',
78 _ddl_protoschema_upgrade_consume_filenames => '_ddl_protoschema_upgrade_consume_filenames',
79 _ddl_protoschema_downgrade_consume_filenames => '_ddl_protoschema_downgrade_consume_filenames',
80 _ddl_protoschema_produce_filename => '_ddl_protoschema_produce_filename',
81 _ddl_schema_produce_filename => '_ddl_schema_produce_filename',
82 _ddl_schema_upgrade_consume_filenames => '_ddl_schema_upgrade_consume_filenames',
83 _ddl_schema_downgrade_consume_filenames => '_ddl_schema_downgrade_consume_filenames',
84 _ddl_schema_upgrade_produce_filename => '_ddl_schema_upgrade_produce_filename',
85 _ddl_schema_downgrade_produce_filename => '_ddl_schema_downgrade_produce_filename',
87 _read_sql_file => '_read_sql_file',
88 _coderefs_per_files => '_coderefs_per_files',
89 _write_data_string => '_write_data_string',
90 _write_data_list => '_write_data_list',
94 sub _build__filesystem_interface {
95 use DBIx::Class::DeploymentHandler::Filesystem;
96 DBIx::Class::DeploymentHandler::Filesystem->new(
97 script_directory => $_[0]->script_directory,
98 ignore_ddl => $_[0]->ignore_ddl
104 isa => 'DBIx::Class::DeploymentHandler::Databases',
106 default => sub { [qw( MySQL SQLite PostgreSQL )] },
115 has schema_version => (
121 # this will probably never get called as the DBICDH
122 # will be passing down a schema_version normally, which
123 # is built the same way, but we leave this in place
124 method _build_schema_version { $self->schema->schema_version }
126 method _run_sql_array($sql) {
127 my $storage = $self->storage;
130 $_ && # remove blank lines
131 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
133 s/^\s+//; s/\s+$//; # trim whitespace
134 join '', grep { !/^--/ } split /\n/ # remove comments
137 Dlog_trace { "Running SQL $_" } $sql;
138 foreach my $line (@{$sql}) {
139 $storage->_query_start($line);
140 # the whole reason we do this is so that we can see the line that was run
142 $storage->dbh_do (sub { $_[1]->do($line) });
145 die "$_ (running line '$line')"
147 $storage->_query_end($line);
149 return join "\n", @$sql
152 method _run_sql($filename) {
153 log_debug { "Running SQL from $filename" };
154 return $self->_run_sql_array($self->_read_sql_file($filename));
157 method _run_perl($filename, $versions) {
158 log_debug { "Running Perl from $filename" };
159 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
161 no warnings 'redefine';
162 my $fn = eval "$filedata";
164 Dlog_trace { "Running Perl $_" } $fn;
167 carp "$filename failed to compile: $@";
168 } elsif (ref $fn eq 'CODE') {
169 $fn->($self->schema, $versions)
171 carp "$filename should define an anonymouse sub that takes a schema but it didn't!";
175 method _run_sql_and_perl($filenames, $sql_to_run, $versions) {
176 my @files = @{$filenames};
177 my $guard = $self->schema->txn_scope_guard if $self->txn_wrap;
179 $self->_run_sql_array($sql_to_run) if $self->ignore_ddl;
181 my $sql = ($sql_to_run)?join ";\n", @$sql_to_run:'';
183 for my $filename (@files) {
184 if ($self->ignore_ddl && $filename =~ /^[^_]*-auto.*\.sql$/) {
186 } elsif ($filename =~ /\.sql$/) {
187 $sql .= $self->_run_sql($filename)
188 } elsif ( $filename =~ /\.pl$/ ) {
189 $self->_run_perl($filename, $versions)
191 croak "A file ($filename) got to deploy that wasn't sql or perl!";
195 $guard->commit if $self->txn_wrap;
202 my $version = (shift @_ || {})->{version} || $self->schema_version;
203 log_info { "deploying version $version" };
204 my $sqlt_type = $self->storage->sqlt_type;
206 if ($self->ignore_ddl) {
207 $sql = $self->_sql_from_yaml({},
208 '_ddl_protoschema_deploy_consume_filenames', $sqlt_type
211 return $self->_run_sql_and_perl($self->_ddl_schema_consume_filenames(
214 ), $sql, [$version]);
220 my $version = $args->{version} || $self->schema_version;
221 log_info { "initializing version $version" };
222 my $storage_type = $args->{storage_type} || $self->storage->sqlt_type;
224 my @files = @{$self->_ddl_initialize_consume_filenames(
229 for my $filename (@files) {
230 # We ignore sql for now (till I figure out what to do with it)
231 if ( $filename =~ /^(.+)\.pl$/ ) {
232 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
234 no warnings 'redefine';
235 my $fn = eval "$filedata";
239 carp "$filename failed to compile: $@";
240 } elsif (ref $fn eq 'CODE') {
243 carp "$filename should define an anonymous sub but it didn't!";
246 croak "A file ($filename) got to initialize_scripts that wasn't sql or perl!";
251 method _sqldiff_from_yaml($from_version, $to_version, $db, $direction) {
252 my $dir = $self->script_directory;
255 ignore_constraint_names => 1,
256 ignore_index_names => 1,
257 %{$self->sql_translator_args}
262 my $prefilename = $self->_ddl_protoschema_produce_filename($from_version, $dir);
264 # should probably be a croak
265 carp("No previous schema file found ($prefilename)")
266 unless -e $prefilename;
268 my $t = SQL::Translator->new({
272 parser => 'SQL::Translator::Parser::YAML',
275 my $out = $t->translate( $prefilename )
278 $source_schema = $t->schema;
280 $source_schema->name( $prefilename )
281 unless $source_schema->name;
286 my $filename = $self->_ddl_protoschema_produce_filename($to_version, $dir);
288 # should probably be a croak
289 carp("No next schema file found ($filename)")
292 my $t = SQL::Translator->new({
296 parser => 'SQL::Translator::Parser::YAML',
299 my $out = $t->translate( $filename )
302 $dest_schema = $t->schema;
304 $dest_schema->name( $filename )
305 unless $dest_schema->name;
308 my $transform_files_method = "_ddl_protoschema_${direction}_consume_filenames";
309 my $transforms = $self->_coderefs_per_files(
310 $self->$transform_files_method([$from_version, $to_version])
312 $_->($source_schema, $dest_schema) for @$transforms;
314 return [SQL::Translator::Diff::schema_diff(
321 method _sql_from_yaml($sqltargs, $from_file, $db) {
322 my $schema = $self->schema;
323 my $version = $self->schema_version;
327 my $actual_file = $self->$from_file($version);
328 for my $yaml_filename (@{
329 DlogS_trace { "generating SQL from Serialized SQL Files: $_" }
330 (ref $actual_file?$actual_file:[$actual_file])
332 my $sqlt = SQL::Translator->new({
334 parser => 'SQL::Translator::Parser::YAML',
339 push @sql, $sqlt->translate($yaml_filename);
341 carp("Failed to translate to $db, skipping. (" . $sqlt->error . ")");
348 sub _prepare_install {
350 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
351 my $from_file = shift;
353 my $dir = $self->script_directory;
354 my $databases = $self->databases;
355 my $version = $self->schema_version;
357 foreach my $db (@$databases) {
358 my $sql = $self->_sql_from_yaml($sqltargs, $from_file, $db ) or next;
360 my $filename = $self->$to_file($db, $version, $dir);
362 if ($self->force_overwrite) {
363 carp "Overwriting existing DDL file - $filename";
366 die "Cannot overwrite '$filename', either enable force_overwrite or delete it"
370 $self->_write_data_list($filename, $sql);
374 sub _resultsource_install_filename {
375 my ($self, $source_name) = @_;
377 my ($self, $type, $version) = @_;
378 my $dirname = catfile( $self->script_directory, $type, 'deploy', $version );
379 mkpath($dirname) unless -d $dirname;
381 return catfile( $dirname, "001-auto-$source_name.sql" );
385 sub _resultsource_protoschema_filename {
386 my ($self, $source_name) = @_;
388 my ($self, $version) = @_;
389 my $dirname = catfile( $self->script_directory, '_source', 'deploy', $version );
390 mkpath($dirname) unless -d $dirname;
392 return catfile( $dirname, "001-auto-$source_name.yml" );
396 sub install_resultsource {
397 my ($self, $args) = @_;
398 my $source = $args->{result_source}
399 or die 'result_source must be passed to install_resultsource';
400 my $version = $args->{version}
401 or die 'version must be passed to install_resultsource';
402 log_info { 'installing_resultsource ' . $source->source_name . ", version $version" };
403 my $rs_install_file =
404 $self->_resultsource_install_filename($source->source_name);
407 $self->$rs_install_file(
408 $self->storage->sqlt_type,
412 $self->_run_sql_and_perl($files, '', [$version]);
415 sub prepare_resultsource_install {
417 my $source = (shift @_)->{result_source};
418 log_info { 'preparing install for resultsource ' . $source->source_name };
420 my $install_filename = $self->_resultsource_install_filename($source->source_name);
421 my $proto_filename = $self->_resultsource_protoschema_filename($source->source_name);
422 $self->prepare_protoschema({
423 parser_args => { sources => [$source->source_name], }
425 $self->_prepare_install({}, $proto_filename, $install_filename);
429 log_info { 'preparing deploy' };
431 $self->prepare_protoschema({
432 # Exclude __VERSION so that it gets installed separately
433 parser_args => { sources => [grep { $_ ne '__VERSION' } $self->schema->sources], }
434 }, '_ddl_protoschema_produce_filename');
435 $self->_prepare_install({}, '_ddl_protoschema_produce_filename', '_ddl_schema_produce_filename');
438 sub prepare_upgrade {
439 my ($self, $args) = @_;
441 "preparing upgrade from $args->{from_version} to $args->{to_version}"
443 $self->_prepare_changegrade(
444 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'upgrade'
448 sub prepare_downgrade {
449 my ($self, $args) = @_;
451 "preparing downgrade from $args->{from_version} to $args->{to_version}"
453 $self->_prepare_changegrade(
454 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'downgrade'
458 method _prepare_changegrade($from_version, $to_version, $version_set, $direction) {
459 my $schema = $self->schema;
460 my $databases = $self->databases;
461 my $dir = $self->script_directory;
463 my $schema_version = $self->schema_version;
464 my $diff_file_method = "_ddl_schema_${direction}_produce_filename";
465 foreach my $db (@$databases) {
466 my $diff_file = $self->$diff_file_method($db, $version_set, $dir );
468 if ($self->force_overwrite) {
469 carp("Overwriting existing $direction-diff file - $diff_file");
472 die "Cannot overwrite '$diff_file', either enable force_overwrite or delete it"
476 $self->_write_data_list($diff_file,
477 $self->_sqldiff_from_yaml($from_version, $to_version, $db, $direction)
482 sub downgrade_single_step {
484 my $version_set = (shift @_)->{version_set};
485 Dlog_info { "downgrade_single_step'ing $_" } $version_set;
487 my $sqlt_type = $self->storage->sqlt_type;
489 if ($self->ignore_ddl) {
490 $sql_to_run = $self->_sqldiff_from_yaml(
491 $version_set->[0], $version_set->[1], $sqlt_type, 'downgrade',
494 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_downgrade_consume_filenames(
497 ), $sql_to_run, $version_set);
502 sub upgrade_single_step {
504 my $version_set = (shift @_)->{version_set};
505 Dlog_info { "upgrade_single_step'ing $_" } $version_set;
507 my $sqlt_type = $self->storage->sqlt_type;
509 if ($self->ignore_ddl) {
510 $sql_to_run = $self->_sqldiff_from_yaml(
511 $version_set->[0], $version_set->[1], $sqlt_type, 'upgrade',
514 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_upgrade_consume_filenames(
517 ), $sql_to_run, $version_set);
521 sub prepare_protoschema {
523 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
526 = $self->$to_file($self->schema_version);
528 # we do this because the code that uses this sets parser args,
529 # so we just need to merge in the package
530 $sqltargs->{parser_args}{package} = $self->schema;
531 my $sqlt = SQL::Translator->new({
532 parser => 'SQL::Translator::Parser::DBIx::Class',
533 producer => 'SQL::Translator::Producer::YAML',
537 my $yml = $sqlt->translate;
539 croak("Failed to translate to YAML: " . $sqlt->error)
543 if ($self->force_overwrite) {
544 carp "Overwriting existing DDL-YML file - $filename";
547 die "Cannot overwrite '$filename', either enable force_overwrite or delete it"
551 $self->_write_data_string($filename, $yml);
554 __PACKAGE__->meta->make_immutable;
558 # vim: ts=2 sw=2 expandtab
564 This class is the meat of L<DBIx::Class::DeploymentHandler>. It takes care
565 of generating serialized schemata as well as sql files to move from one
566 version of a schema to the rest. One of the hallmark features of this class
567 is that it allows for multiple sql files for deploy and upgrade, allowing
568 developers to fine tune deployment. In addition it also allows for perl
569 files to be run at any stage of the process.
571 For basic usage see L<DBIx::Class::DeploymentHandler::HandlesDeploy>. What's
572 documented here is extra fun stuff or private methods.
574 =head1 DIRECTORY LAYOUT
576 Arguably this is the best feature of L<DBIx::Class::DeploymentHandler>.
577 It's spiritually based upon L<DBIx::Migration::Directories>, but has a
578 lot of extensions and modifications, so even if you are familiar with it,
579 please read this. I feel like the best way to describe the layout is with
580 the following example:
606 | | `- 002-remove-customers.pl
609 | | `- 002-generate-customers.pl
611 | `- 999-bump-action.pl
618 | |- 001-create_database.pl
619 | `- 002-create_users_and_permissions.pl
627 So basically, the code
631 on an C<SQLite> database that would simply run
632 C<$sql_migration_dir/SQLite/deploy/1/001-auto.sql>. Next,
634 $dm->upgrade_single_step([1,2])
636 would run C<$sql_migration_dir/SQLite/upgrade/1-2/001-auto.sql> followed by
637 C<$sql_migration_dir/_common/upgrade/1-2/002-generate-customers.pl>, and
638 finally punctuated by
639 C<$sql_migration_dir/_common/upgrade/_any/999-bump-action.pl>.
641 C<.pl> files don't have to be in the C<_common> directory, but most of the time
642 they should be, because perl scripts are generally database independent.
644 Note that unlike most steps in the process, C<initialize> will not run SQL, as
645 there may not even be an database at initialize time. It will run perl scripts
646 just like the other steps in the process, but nothing is passed to them.
647 Until people have used this more it will remain freeform, but a recommended use
648 of initialize is to have it prompt for username and password, and then call the
649 appropriate C<< CREATE DATABASE >> commands etc.
651 =head2 Directory Specification
653 The following subdirectories are recognized by this DeployMethod:
657 =item C<_source> This directory can contain the following directories:
661 =item C<deploy> This directory merely contains directories named after schema
662 versions, which in turn contain C<yaml> files that are serialized versions
663 of the schema at that version. These files are not for editing by hand.
667 =item C<_preprocess_schema> This directory can contain the following
672 =item C<downgrade> This directory merely contains directories named after
673 migrations, which are of the form C<$from_version-$to_version>. Inside of
674 these directories you may put Perl scripts which are to return a subref
675 that takes the arguments C<< $from_schema, $to_schema >>, which are
676 L<SQL::Translator::Schema> objects.
678 =item C<upgrade> This directory merely contains directories named after
679 migrations, which are of the form C<$from_version-$to_version>. Inside of
680 these directories you may put Perl scripts which are to return a subref
681 that takes the arguments C<< $from_schema, $to_schema >>, which are
682 L<SQL::Translator::Schema> objects.
686 =item C<$storage_type> This is a set of scripts that gets run depending on what
687 your storage type is. If you are not sure what your storage type is, take a
688 look at the producers listed for L<SQL::Translator>. Also note, C<_common>
689 is a special case. C<_common> will get merged into whatever other files you
690 already have. This directory can contain the following directories itself:
694 =item C<initialize> Gets run before the C<deploy> is C<deploy>ed. Has the
695 same structure as the C<deploy> subdirectory as well; that is, it has a
696 directory for each schema version. Unlike C<deploy>, C<upgrade>, and C<downgrade>
697 though, it can only run C<.pl> files, and the coderef in the perl files get
698 no arguments passed to them.
700 =item C<deploy> Gets run when the schema is C<deploy>ed. Structure is a
701 directory per schema version, and then files are merged with C<_common> and run
702 in filename order. C<.sql> files are merely run, as expected. C<.pl> files are
703 run according to L</PERL SCRIPTS>.
705 =item C<upgrade> Gets run when the schema is C<upgrade>d. Structure is a directory
706 per upgrade step, (for example, C<1-2> for upgrading from version 1 to version
707 2,) and then files are merged with C<_common> and run in filename order.
708 C<.sql> files are merely run, as expected. C<.pl> files are run according
711 =item C<downgrade> Gets run when the schema is C<downgrade>d. Structure is a directory
712 per downgrade step, (for example, C<2-1> for downgrading from version 2 to version
713 1,) and then files are merged with C<_common> and run in filename order.
714 C<.sql> files are merely run, as expected. C<.pl> files are run according
722 Note that there can be an C<_any> in the place of any of the versions (like
723 C<1-2> or C<1>), which means those scripts will be run B<every> time. So if
724 you have an C<_any> in C<_common/upgrade>, that script will get run for every
729 A perl script for this tool is very simple. It merely needs to contain an
730 anonymous sub that takes a L<DBIx::Class::Schema> and the version set as it's
733 A very basic perl script might look like:
743 # [1] for deploy, [1,2] for upgrade or downgrade, probably used with _any
744 my $versions = shift;
746 $schema->resultset('Users')->create({
754 This attribute will, when set to true (default is false), cause the DM to use
755 L<SQL::Translator> to use the C<_source>'s serialized SQL::Translator::Schema
756 instead of any pregenerated SQL. If you have a development server this is
757 probably the best plan of action as you will not be putting as many generated
758 files in your version control. Goes well with with C<databases> of C<[]>.
760 =attr force_overwrite
762 When this attribute is true generated files will be overwritten when the
763 methods which create such files are run again. The default is false, in which
764 case the program will die with a message saying which file needs to be deleted.
768 The L<DBIx::Class::Schema> (B<required>) that is used to talk to the database
769 and generate the DDL.
773 The L<DBIx::Class::Storage> that is I<actually> used to talk to the database
774 and generate the DDL. This is automatically created with L</_build_storage>.
776 =attr sql_translator_args
778 The arguments that get passed to L<SQL::Translator> when it's used.
780 =attr script_directory
782 The directory (default C<'sql'>) that scripts are stored in
786 The types of databases (default C<< [qw( MySQL SQLite PostgreSQL )] >>) to
791 Set to true (which is the default) to wrap all upgrades and deploys in a single
796 The version the schema on your harddrive is at. Defaults to
797 C<< $self->schema->schema_version >>.