1 package DBIx::Class::DeploymentHandler::DeployMethod::SQL::Translator;
4 # ABSTRACT: Manage your SQL and Perl migrations in nicely laid out directories
7 use Carp qw( carp croak );
8 use Log::Contextual::WarnLogger;
9 use Log::Contextual qw(:log :dlog), -default_logger => Log::Contextual::WarnLogger->new({
10 env_prefix => 'DBICDH'
12 use Data::Dumper::Concise;
14 use Method::Signatures::Simple;
18 require SQL::Translator::Diff;
20 require DBIx::Class::Storage; # loaded for type constraint
21 use DBIx::Class::DeploymentHandler::Types;
23 use File::Path 'mkpath';
24 use File::Spec::Functions;
26 with 'DBIx::Class::DeploymentHandler::HandlesDeploy';
29 isa => 'DBIx::Class::Schema',
35 isa => 'DBIx::Class::Storage',
40 method _build_storage {
41 my $s = $self->schema->storage;
42 $s->_determine_driver;
46 has sql_translator_args => (
49 default => sub { {} },
51 has script_directory => (
60 isa => 'DBIx::Class::DeploymentHandler::Databases',
62 default => sub { [qw( MySQL SQLite PostgreSQL )] },
71 has schema_version => (
77 # this will probably never get called as the DBICDH
78 # will be passing down a schema_version normally, which
79 # is built the same way, but we leave this in place
80 method _build_schema_version { $self->schema->schema_version }
82 method __ddl_consume_with_prefix($type, $versions, $prefix) {
83 my $base_dir = $self->script_directory;
85 my $main = catfile( $base_dir, $type );
86 my $generic = catfile( $base_dir, '_generic' );
88 catfile( $base_dir, '_common', $prefix, join q(-), @{$versions} );
92 $dir = catfile($main, $prefix, join q(-), @{$versions})
93 } elsif (-d $generic) {
94 $dir = catfile($generic, $prefix, join q(-), @{$versions});
96 croak "neither $main or $generic exist; please write/generate some SQL";
99 opendir my($dh), $dir;
100 my %files = map { $_ => "$dir/$_" } grep { /\.(?:sql|pl|sql-\w+)$/ && -f "$dir/$_" } readdir $dh;
104 opendir my($dh), $common;
105 for my $filename (grep { /\.(?:sql|pl)$/ && -f catfile($common,$_) } readdir $dh) {
106 unless ($files{$filename}) {
107 $files{$filename} = catfile($common,$filename);
113 return [@files{sort keys %files}]
116 method _ddl_preinstall_consume_filenames($type, $version) {
117 $self->__ddl_consume_with_prefix($type, [ $version ], 'preinstall')
120 method _ddl_schema_consume_filenames($type, $version) {
121 $self->__ddl_consume_with_prefix($type, [ $version ], 'schema')
124 method _ddl_schema_produce_filename($type, $version) {
125 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
126 mkpath($dirname) unless -d $dirname;
128 return catfile( $dirname, '001-auto.sql' );
131 method _ddl_schema_up_consume_filenames($type, $versions) {
132 $self->__ddl_consume_with_prefix($type, $versions, 'up')
135 method _ddl_schema_down_consume_filenames($type, $versions) {
136 $self->__ddl_consume_with_prefix($type, $versions, 'down')
139 method _ddl_schema_up_produce_filename($type, $versions) {
140 my $dir = $self->script_directory;
142 my $dirname = catfile( $dir, $type, 'up', join q(-), @{$versions});
143 mkpath($dirname) unless -d $dirname;
145 return catfile( $dirname, '001-auto.sql'
149 method _ddl_schema_down_produce_filename($type, $versions, $dir) {
150 my $dirname = catfile( $dir, $type, 'down', join q(-), @{$versions} );
151 mkpath($dirname) unless -d $dirname;
153 return catfile( $dirname, '001-auto.sql');
156 method _run_sql_array($sql) {
157 my $storage = $self->storage;
159 log_trace { '[DBICDH] Running SQL ' . Dumper($sql) };
160 foreach my $line (@{$sql}) {
161 $storage->_query_start($line);
163 # do a dbh_do cycle here, as we need some error checking in
164 # place (even though we will ignore errors)
165 $storage->dbh_do (sub { $_[1]->do($line) });
168 carp "$_ (running '${line}')"
170 $storage->_query_end($line);
172 return join "\n", @$sql
175 method _run_sql($filename) {
176 log_debug { "[DBICDH] Running SQL from $filename" };
177 return $self->_run_sql_array($self->_read_sql_file($filename));
180 method _run_perl($filename) {
181 log_debug { "[DBICDH] Running Perl from $filename" };
182 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
184 no warnings 'redefine';
185 my $fn = eval "$filedata";
187 log_trace { '[DBICDH] Running Perl ' . Dumper($fn) };
190 carp "$filename failed to compile: $@";
191 } elsif (ref $fn eq 'CODE') {
194 carp "$filename should define an anonymouse sub that takes a schema but it didn't!";
200 method _run_serialized_sql($filename, $type) {
201 if ($type eq 'json') {
203 $json ||= JSON->new->pretty;
204 my @sql = @{$json->decode($filename)};
206 croak "A file ($filename) got to deploy that wasn't sql or perl!";
212 method _run_sql_and_perl($filenames) {
213 my @files = @{$filenames};
214 my $guard = $self->schema->txn_scope_guard if $self->txn_wrap;
217 for my $filename (@files) {
218 if ($filename =~ /\.sql$/) {
219 $sql .= $self->_run_sql($filename)
220 } elsif ( $filename =~ /\.sql-(\w+)$/ ) {
221 $sql .= $self->_run_serialized_sql($filename, $1)
222 } elsif ( $filename =~ /\.pl$/ ) {
223 $self->_run_perl($filename)
225 croak "A file ($filename) got to deploy that wasn't sql or perl!";
229 $guard->commit if $self->txn_wrap;
236 my $version = (shift @_ || {})->{version} || $self->schema_version;
237 log_info { "[DBICDH] deploying version $version" };
239 return $self->_run_sql_and_perl($self->_ddl_schema_consume_filenames(
240 $self->storage->sqlt_type,
248 my $version = $args->{version} || $self->schema_version;
249 log_info { "[DBICDH] preinstalling version $version" };
250 my $storage_type = $args->{storage_type} || $self->storage->sqlt_type;
252 my @files = @{$self->_ddl_preinstall_consume_filenames(
257 for my $filename (@files) {
258 # We ignore sql for now (till I figure out what to do with it)
259 if ( $filename =~ /^(.+)\.pl$/ ) {
260 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
262 no warnings 'redefine';
263 my $fn = eval "$filedata";
267 carp "$filename failed to compile: $@";
268 } elsif (ref $fn eq 'CODE') {
271 carp "$filename should define an anonymous sub but it didn't!";
274 croak "A file ($filename) got to preinstall_scripts that wasn't sql or perl!";
279 sub _prepare_install {
281 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
283 my $schema = $self->schema;
284 my $databases = $self->databases;
285 my $dir = $self->script_directory;
286 my $version = $self->schema_version;
288 my $sqlt = SQL::Translator->new({
290 ignore_constraint_names => 1,
291 ignore_index_names => 1,
292 parser => 'SQL::Translator::Parser::DBIx::Class',
296 my $sqlt_schema = $sqlt->translate( data => $schema )
297 or croak($sqlt->error);
299 foreach my $db (@$databases) {
301 $sqlt->{schema} = $sqlt_schema;
302 $sqlt->producer($db);
304 my $filename = $self->$to_file($db, $version, $dir);
306 carp "Overwriting existing DDL file - $filename";
310 my $output = $sqlt->translate;
312 carp("Failed to translate to $db, skipping. (" . $sqlt->error . ")");
315 open my $file, q(>), $filename;
316 print {$file} $output;
321 sub _resultsource_install_filename {
322 my ($self, $source_name) = @_;
324 my ($self, $type, $version) = @_;
325 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
326 mkpath($dirname) unless -d $dirname;
328 return catfile( $dirname, "001-auto-$source_name.sql" );
332 sub install_resultsource {
333 my ($self, $args) = @_;
334 my $source = $args->{result_source};
335 my $version = $args->{version};
336 log_info { '[DBICDH] installing_resultsource ' . $source->source_name . ", version $version" };
337 my $rs_install_file =
338 $self->_resultsource_install_filename($source->source_name);
341 $self->$rs_install_file(
342 $self->storage->sqlt_type,
346 $self->_run_sql_and_perl($files);
349 sub prepare_resultsource_install {
351 my $source = (shift @_)->{result_source};
352 log_info { '[DBICDH] preparing install for resultsource ' . $source->source_name };
354 my $filename = $self->_resultsource_install_filename($source->source_name);
355 $self->_prepare_install({
356 parser_args => { sources => [$source->source_name], }
361 log_info { '[DBICDH] preparing deploy' };
363 $self->_prepare_install({}, '_ddl_schema_produce_filename');
366 sub prepare_upgrade {
367 my ($self, $args) = @_;
369 '[DBICDH] preparing upgrade ' .
370 "from $args->{from_version} to $args->{to_version}"
372 $self->_prepare_changegrade(
373 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'up'
377 sub prepare_downgrade {
378 my ($self, $args) = @_;
380 '[DBICDH] preparing downgrade ' .
381 "from $args->{from_version} to $args->{to_version}"
383 $self->_prepare_changegrade(
384 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'down'
388 method _prepare_changegrade($from_version, $to_version, $version_set, $direction) {
389 my $schema = $self->schema;
390 my $databases = $self->databases;
391 my $dir = $self->script_directory;
392 my $sqltargs = $self->sql_translator_args;
394 my $schema_version = $self->schema_version;
398 ignore_constraint_names => 1,
399 ignore_index_names => 1,
403 my $sqlt = SQL::Translator->new( $sqltargs );
405 $sqlt->parser('SQL::Translator::Parser::DBIx::Class');
406 my $sqlt_schema = $sqlt->translate( data => $schema )
407 or croak($sqlt->error);
409 foreach my $db (@$databases) {
411 $sqlt->{schema} = $sqlt_schema;
412 $sqlt->producer($db);
414 my $prefilename = $self->_ddl_schema_produce_filename($db, $from_version, $dir);
415 unless(-e $prefilename) {
416 carp("No previous schema file found ($prefilename)");
419 my $diff_file_method = "_ddl_schema_${direction}_produce_filename";
420 my $diff_file = $self->$diff_file_method($db, $version_set, $dir );
422 carp("Overwriting existing $direction-diff file - $diff_file");
428 my $t = SQL::Translator->new({
434 $t->parser( $db ) # could this really throw an exception?
437 my $out = $t->translate( $prefilename )
440 $source_schema = $t->schema;
442 $source_schema->name( $prefilename )
443 unless $source_schema->name;
446 # The "new" style of producers have sane normalization and can support
447 # diffing a SQL file against a DBIC->SQLT schema. Old style ones don't
448 # And we have to diff parsed SQL against parsed SQL.
449 my $dest_schema = $sqlt_schema;
451 unless ( "SQL::Translator::Producer::$db"->can('preprocess_schema') ) {
452 my $t = SQL::Translator->new({
458 $t->parser( $db ) # could this really throw an exception?
461 my $filename = $self->_ddl_schema_produce_filename($db, $to_version, $dir);
462 my $out = $t->translate( $filename )
465 $dest_schema = $t->schema;
467 $dest_schema->name( $filename )
468 unless $dest_schema->name;
471 my $diff = SQL::Translator::Diff::schema_diff(
476 open my $file, q(>), $diff_file;
482 method _read_sql_file($file) {
485 open my $fh, '<', $file;
486 my @data = split /;\n/, join '', <$fh>;
490 $_ && # remove blank lines
491 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
493 s/^\s+//; s/\s+$//; # trim whitespace
494 join '', grep { !/^--/ } split /\n/ # remove comments
500 sub downgrade_single_step {
502 my $version_set = (shift @_)->{version_set};
503 log_info { qq([DBICDH] downgrade_single_step'ing ) . Dumper($version_set) };
505 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_down_consume_filenames(
506 $self->storage->sqlt_type,
513 sub upgrade_single_step {
515 my $version_set = (shift @_)->{version_set};
516 log_info { qq([DBICDH] upgrade_single_step'ing ) . Dumper($version_set) };
518 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_up_consume_filenames(
519 $self->storage->sqlt_type,
525 __PACKAGE__->meta->make_immutable;
529 # vim: ts=2 sw=2 expandtab
535 This class is the meat of L<DBIx::Class::DeploymentHandler>. It takes care of
536 generating sql files representing schemata as well as sql files to move from
537 one version of a schema to the rest. One of the hallmark features of this
538 class is that it allows for multiple sql files for deploy and upgrade, allowing
539 developers to fine tune deployment. In addition it also allows for perl files
540 to be run at any stage of the process.
542 For basic usage see L<DBIx::Class::DeploymentHandler::HandlesDeploy>. What's
543 documented here is extra fun stuff or private methods.
545 =head1 DIRECTORY LAYOUT
547 Arguably this is the best feature of L<DBIx::Class::DeploymentHandler>. It's
548 heavily based upon L<DBIx::Migration::Directories>, but has some extensions and
549 modifications, so even if you are familiar with it, please read this. I feel
550 like the best way to describe the layout is with the following example:
568 | | `- 002-remove-customers.pl
571 | `- 002-generate-customers.pl
582 | `- 002-create-stored-procedures.sql
589 | |- 001-create_database.pl
590 | `- 002-create_users_and_permissions.pl
598 So basically, the code
602 on an C<SQLite> database that would simply run
603 C<$sql_migration_dir/SQLite/schema/1/001-auto.sql>. Next,
605 $dm->upgrade_single_step([1,2])
607 would run C<$sql_migration_dir/SQLite/up/1-2/001-auto.sql> followed by
608 C<$sql_migration_dir/_common/up/1-2/002-generate-customers.pl>.
610 Now, a C<.pl> file doesn't have to be in the C<_common> directory, but most of
611 the time it probably should be, since perl scripts will mostly be database
614 C<_generic> exists for when you for some reason are sure that your SQL is
615 generic enough to run on all databases. Good luck with that one.
617 Note that unlike most steps in the process, C<preinstall> will not run SQL, as
618 there may not even be an database at preinstall time. It will run perl scripts
619 just like the other steps in the process, but nothing is passed to them.
620 Until people have used this more it will remain freeform, but a recommended use
621 of preinstall is to have it prompt for username and password, and then call the
622 appropriate C<< CREATE DATABASE >> commands etc.
626 A perl script for this tool is very simple. It merely needs to contain an
627 anonymous sub that takes a L<DBIx::Class::Schema> as it's only argument.
628 A very basic perl script might look like:
638 $schema->resultset('Users')->create({
646 The L<DBIx::Class::Schema> (B<required>) that is used to talk to the database
647 and generate the DDL.
651 The L<DBIx::Class::Storage> that is I<actually> used to talk to the database
652 and generate the DDL. This is automatically created with L</_build_storage>.
654 =attr sql_translator_args
656 The arguments that get passed to L<SQL::Translator> when it's used.
658 =attr script_directory
660 The directory (default C<'sql'>) that scripts are stored in
664 The types of databases (default C<< [qw( MySQL SQLite PostgreSQL )] >>) to
669 Set to true (which is the default) to wrap all upgrades and deploys in a single
674 The version the schema on your harddrive is at. Defaults to
675 C<< $self->schema->schema_version >>.
679 =head2 __ddl_consume_with_prefix
681 $dm->__ddl_consume_with_prefix( 'SQLite', [qw( 1.00 1.01 )], 'up' )
683 This is the meat of the multi-file upgrade/deploy stuff. It returns a list of
684 files in the order that they should be run for a generic "type" of upgrade.
685 You should not be calling this in user code.
687 =head2 _ddl_schema_consume_filenames
689 $dm->__ddl_schema_consume_filenames( 'SQLite', [qw( 1.00 )] )
691 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
694 =head2 _ddl_schema_produce_filename
696 $dm->__ddl_schema_produce_filename( 'SQLite', [qw( 1.00 )] )
698 Returns a single file in which an initial schema will be stored.
700 =head2 _ddl_schema_up_consume_filenames
702 $dm->_ddl_schema_up_consume_filenames( 'SQLite', [qw( 1.00 )] )
704 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
707 =head2 _ddl_schema_down_consume_filenames
709 $dm->_ddl_schema_down_consume_filenames( 'SQLite', [qw( 1.00 )] )
711 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for a
714 =head2 _ddl_schema_up_produce_filenames
716 $dm->_ddl_schema_up_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
718 Returns a single file in which the sql to upgrade from one schema to another
721 =head2 _ddl_schema_down_produce_filename
723 $dm->_ddl_schema_down_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
725 Returns a single file in which the sql to downgrade from one schema to another
728 =head2 _resultsource_install_filename
730 my $filename_fn = $dm->_resultsource_install_filename('User');
731 $dm->$filename_fn('SQLite', '1.00')
733 Returns a function which in turn returns a single filename used to install a
734 single resultsource. Weird interface is convenient for me. Deal with it.
736 =head2 _run_sql_and_perl
738 $dm->_run_sql_and_perl([qw( list of filenames )])
740 Simply put, this runs the list of files passed to it. If the file ends in
741 C<.sql> it runs it as sql and if it ends in C<.pl> it runs it as a perl file.
743 Depending on L</txn_wrap> all of the files run will be wrapped in a single
746 =head2 _prepare_install
748 $dm->_prepare_install({ add_drop_table => 0 }, sub { 'file_to_create' })
750 Generates the sql file for installing the database. First arg is simply
751 L<SQL::Translator> args and the second is a coderef that returns the filename
754 =head2 _prepare_changegrade
756 $dm->_prepare_changegrade('1.00', '1.01', [qw( 1.00 1.01)], 'up')
758 Generates the sql file for migrating from one schema version to another. First
759 arg is the version to start from, second is the version to go to, third is the
760 L<version set|DBIx::Class::DeploymentHandler/VERSION SET>, and last is the
761 direction of the changegrade, be it 'up' or 'down'.
763 =head2 _read_sql_file
765 $dm->_read_sql_file('foo.sql')
767 Reads a sql file and returns lines in an C<ArrayRef>. Strips out comments,
768 transactions, and blank lines.