1 package DBIx::Class::DeploymentHandler::DeployMethod::SQL::Translator;
4 # ABSTRACT: Manage your SQL and Perl migrations in nicely laid out directories
7 use Carp qw( carp croak );
8 use Log::Contextual::WarnLogger;
9 use Log::Contextual qw(:log :dlog), -default_logger => Log::Contextual::WarnLogger->new({
10 env_prefix => 'DBICDH'
12 use Data::Dumper::Concise;
14 use Method::Signatures::Simple;
18 require SQL::Translator::Diff;
20 require DBIx::Class::Storage; # loaded for type constraint
21 use DBIx::Class::DeploymentHandler::Types;
23 use File::Path 'mkpath';
24 use File::Spec::Functions;
26 with 'DBIx::Class::DeploymentHandler::HandlesDeploy';
29 isa => 'DBIx::Class::Schema',
35 isa => 'DBIx::Class::Storage',
40 method _build_storage {
41 my $s = $self->schema->storage;
42 $s->_determine_driver;
46 has sql_translator_args => (
49 default => sub { {} },
51 has script_directory => (
60 isa => 'DBIx::Class::DeploymentHandler::Databases',
62 default => sub { [qw( MySQL SQLite PostgreSQL )] },
71 has schema_version => (
77 # this will probably never get called as the DBICDH
78 # will be passing down a schema_version normally, which
79 # is built the same way, but we leave this in place
80 method _build_schema_version { $self->schema->schema_version }
82 method __ddl_consume_with_prefix($type, $versions, $prefix) {
83 my $base_dir = $self->script_directory;
85 my $main = catfile( $base_dir, $type );
86 my $generic = catfile( $base_dir, '_generic' );
88 catfile( $base_dir, '_common', $prefix, join q(-), @{$versions} );
92 $dir = catfile($main, $prefix, join q(-), @{$versions})
93 } elsif (-d $generic) {
94 $dir = catfile($generic, $prefix, join q(-), @{$versions});
96 croak "neither $main or $generic exist; please write/generate some SQL";
99 opendir my($dh), $dir;
100 my %files = map { $_ => "$dir/$_" } grep { /\.(?:sql|pl|sql-\w+)$/ && -f "$dir/$_" } readdir $dh;
104 opendir my($dh), $common;
105 for my $filename (grep { /\.(?:sql|pl)$/ && -f catfile($common,$_) } readdir $dh) {
106 unless ($files{$filename}) {
107 $files{$filename} = catfile($common,$filename);
113 return [@files{sort keys %files}]
116 method _ddl_preinstall_consume_filenames($type, $version) {
117 $self->__ddl_consume_with_prefix($type, [ $version ], 'preinstall')
120 method _ddl_schema_consume_filenames($type, $version) {
121 $self->__ddl_consume_with_prefix($type, [ $version ], 'schema')
124 method _ddl_schema_produce_filename($type, $version) {
125 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
126 mkpath($dirname) unless -d $dirname;
128 return catfile( $dirname, '001-auto.sql' );
131 method _ddl_schema_up_consume_filenames($type, $versions) {
132 $self->__ddl_consume_with_prefix($type, $versions, 'up')
135 method _ddl_schema_down_consume_filenames($type, $versions) {
136 $self->__ddl_consume_with_prefix($type, $versions, 'down')
139 method _ddl_schema_up_produce_filename($type, $versions) {
140 my $dir = $self->script_directory;
142 my $dirname = catfile( $dir, $type, 'up', join q(-), @{$versions});
143 mkpath($dirname) unless -d $dirname;
145 return catfile( $dirname, '001-auto.sql'
149 method _ddl_schema_down_produce_filename($type, $versions, $dir) {
150 my $dirname = catfile( $dir, $type, 'down', join q(-), @{$versions} );
151 mkpath($dirname) unless -d $dirname;
153 return catfile( $dirname, '001-auto.sql');
156 method _run_sql_array($sql) {
157 my $storage = $self->storage;
159 my $ret = join "\n", @$sql;
160 log_trace { "[DBICDH] Running SQL $sql" };
161 foreach my $line (@{$sql}) {
162 $storage->_query_start($line);
164 # do a dbh_do cycle here, as we need some error checking in
165 # place (even though we will ignore errors)
166 $storage->dbh_do (sub { $_[1]->do($line) });
169 carp "$_ (running '${line}')"
171 $storage->_query_end($line);
176 method _run_sql($filename) {
177 log_debug { "[DBICDH] Running SQL from $filename" };
178 return $self->_run_sql_array($self->_read_sql_file($filename));
181 method _run_perl($filename) {
182 log_debug { "[DBICDH] Running Perl from $filename" };
183 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
185 no warnings 'redefine';
186 my $fn = eval "$filedata";
188 log_trace { '[DBICDH] Running Perl ' . Dumper($fn) };
191 carp "$filename failed to compile: $@";
192 } elsif (ref $fn eq 'CODE') {
195 carp "$filename should define an anonymouse sub that takes a schema but it didn't!";
201 method _run_serialized_sql($filename, $type) {
202 if ($type eq 'json') {
204 $json ||= JSON->new->pretty;
205 my @sql = @{$json->decode($filename)};
207 croak "A file ($filename) got to deploy that wasn't sql or perl!";
213 method _run_sql_and_perl($filenames) {
214 my @files = @{$filenames};
215 my $guard = $self->schema->txn_scope_guard if $self->txn_wrap;
218 for my $filename (@files) {
219 if ($filename =~ /\.sql$/) {
220 $sql .= $self->_run_sql($filename)
221 } elsif ( $filename =~ /\.sql-(\w+)$/ ) {
222 $sql .= $self->_run_serialized_sql($filename, $1)
223 } elsif ( $filename =~ /\.pl$/ ) {
224 $self->_run_perl($filename)
226 croak "A file ($filename) got to deploy that wasn't sql or perl!";
230 $guard->commit if $self->txn_wrap;
237 my $version = (shift @_ || {})->{version} || $self->schema_version;
238 log_info { "[DBICDH] deploying version $version" };
240 return $self->_run_sql_and_perl($self->_ddl_schema_consume_filenames(
241 $self->storage->sqlt_type,
249 my $version = $args->{version} || $self->schema_version;
250 log_info { "[DBICDH] preinstalling version $version" };
251 my $storage_type = $args->{storage_type} || $self->storage->sqlt_type;
253 my @files = @{$self->_ddl_preinstall_consume_filenames(
258 for my $filename (@files) {
259 # We ignore sql for now (till I figure out what to do with it)
260 if ( $filename =~ /^(.+)\.pl$/ ) {
261 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
263 no warnings 'redefine';
264 my $fn = eval "$filedata";
268 carp "$filename failed to compile: $@";
269 } elsif (ref $fn eq 'CODE') {
272 carp "$filename should define an anonymous sub but it didn't!";
275 croak "A file ($filename) got to preinstall_scripts that wasn't sql or perl!";
280 sub _prepare_install {
282 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
284 my $schema = $self->schema;
285 my $databases = $self->databases;
286 my $dir = $self->script_directory;
287 my $version = $self->schema_version;
289 my $sqlt = SQL::Translator->new({
291 ignore_constraint_names => 1,
292 ignore_index_names => 1,
293 parser => 'SQL::Translator::Parser::DBIx::Class',
297 my $sqlt_schema = $sqlt->translate( data => $schema )
298 or croak($sqlt->error);
300 foreach my $db (@$databases) {
302 $sqlt->{schema} = $sqlt_schema;
303 $sqlt->producer($db);
305 my $filename = $self->$to_file($db, $version, $dir);
307 carp "Overwriting existing DDL file - $filename";
311 my $output = $sqlt->translate;
313 carp("Failed to translate to $db, skipping. (" . $sqlt->error . ")");
316 open my $file, q(>), $filename;
317 print {$file} $output;
322 sub _resultsource_install_filename {
323 my ($self, $source_name) = @_;
325 my ($self, $type, $version) = @_;
326 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
327 mkpath($dirname) unless -d $dirname;
329 return catfile( $dirname, "001-auto-$source_name.sql" );
333 sub install_resultsource {
334 my ($self, $args) = @_;
335 my $source = $args->{result_source};
336 my $version = $args->{version};
337 log_info { '[DBICDH] installing_resultsource ' . $source->source_name . ", version $version" };
338 my $rs_install_file =
339 $self->_resultsource_install_filename($source->source_name);
342 $self->$rs_install_file(
343 $self->storage->sqlt_type,
347 $self->_run_sql_and_perl($files);
350 sub prepare_resultsource_install {
352 my $source = (shift @_)->{result_source};
353 log_info { '[DBICDH] preparing install for resultsource ' . $source->source_name };
355 my $filename = $self->_resultsource_install_filename($source->source_name);
356 $self->_prepare_install({
357 parser_args => { sources => [$source->source_name], }
362 log_info { '[DBICDH] preparing deploy' };
364 $self->_prepare_install({}, '_ddl_schema_produce_filename');
367 sub prepare_upgrade {
368 my ($self, $args) = @_;
370 '[DBICDH] preparing upgrade ' .
371 "from $args->{from_version} to $args->{to_version}"
373 $self->_prepare_changegrade(
374 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'up'
378 sub prepare_downgrade {
379 my ($self, $args) = @_;
381 '[DBICDH] preparing downgrade ' .
382 "from $args->{from_version} to $args->{to_version}"
384 $self->_prepare_changegrade(
385 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'down'
389 method _prepare_changegrade($from_version, $to_version, $version_set, $direction) {
390 my $schema = $self->schema;
391 my $databases = $self->databases;
392 my $dir = $self->script_directory;
393 my $sqltargs = $self->sql_translator_args;
395 my $schema_version = $self->schema_version;
399 ignore_constraint_names => 1,
400 ignore_index_names => 1,
404 my $sqlt = SQL::Translator->new( $sqltargs );
406 $sqlt->parser('SQL::Translator::Parser::DBIx::Class');
407 my $sqlt_schema = $sqlt->translate( data => $schema )
408 or croak($sqlt->error);
410 foreach my $db (@$databases) {
412 $sqlt->{schema} = $sqlt_schema;
413 $sqlt->producer($db);
415 my $prefilename = $self->_ddl_schema_produce_filename($db, $from_version, $dir);
416 unless(-e $prefilename) {
417 carp("No previous schema file found ($prefilename)");
420 my $diff_file_method = "_ddl_schema_${direction}_produce_filename";
421 my $diff_file = $self->$diff_file_method($db, $version_set, $dir );
423 carp("Overwriting existing $direction-diff file - $diff_file");
429 my $t = SQL::Translator->new({
435 $t->parser( $db ) # could this really throw an exception?
438 my $out = $t->translate( $prefilename )
441 $source_schema = $t->schema;
443 $source_schema->name( $prefilename )
444 unless $source_schema->name;
447 # The "new" style of producers have sane normalization and can support
448 # diffing a SQL file against a DBIC->SQLT schema. Old style ones don't
449 # And we have to diff parsed SQL against parsed SQL.
450 my $dest_schema = $sqlt_schema;
452 unless ( "SQL::Translator::Producer::$db"->can('preprocess_schema') ) {
453 my $t = SQL::Translator->new({
459 $t->parser( $db ) # could this really throw an exception?
462 my $filename = $self->_ddl_schema_produce_filename($db, $to_version, $dir);
463 my $out = $t->translate( $filename )
466 $dest_schema = $t->schema;
468 $dest_schema->name( $filename )
469 unless $dest_schema->name;
472 my $diff = SQL::Translator::Diff::schema_diff(
477 open my $file, q(>), $diff_file;
483 method _read_sql_file($file) {
486 open my $fh, '<', $file;
487 my @data = split /;\n/, join '', <$fh>;
491 $_ && # remove blank lines
492 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
494 s/^\s+//; s/\s+$//; # trim whitespace
495 join '', grep { !/^--/ } split /\n/ # remove comments
501 sub downgrade_single_step {
503 my $version_set = (shift @_)->{version_set};
504 log_info { qq([DBICDH] downgrade_single_step'ing ) . Dumper($version_set) };
506 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_down_consume_filenames(
507 $self->storage->sqlt_type,
514 sub upgrade_single_step {
516 my $version_set = (shift @_)->{version_set};
517 log_info { qq([DBICDH] upgrade_single_step'ing ) . Dumper($version_set) };
519 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_up_consume_filenames(
520 $self->storage->sqlt_type,
526 __PACKAGE__->meta->make_immutable;
530 # vim: ts=2 sw=2 expandtab
536 This class is the meat of L<DBIx::Class::DeploymentHandler>. It takes care of
537 generating sql files representing schemata as well as sql files to move from
538 one version of a schema to the rest. One of the hallmark features of this
539 class is that it allows for multiple sql files for deploy and upgrade, allowing
540 developers to fine tune deployment. In addition it also allows for perl files
541 to be run at any stage of the process.
543 For basic usage see L<DBIx::Class::DeploymentHandler::HandlesDeploy>. What's
544 documented here is extra fun stuff or private methods.
546 =head1 DIRECTORY LAYOUT
548 Arguably this is the best feature of L<DBIx::Class::DeploymentHandler>. It's
549 heavily based upon L<DBIx::Migration::Directories>, but has some extensions and
550 modifications, so even if you are familiar with it, please read this. I feel
551 like the best way to describe the layout is with the following example:
569 | | `- 002-remove-customers.pl
572 | `- 002-generate-customers.pl
583 | `- 002-create-stored-procedures.sql
590 | |- 001-create_database.pl
591 | `- 002-create_users_and_permissions.pl
599 So basically, the code
603 on an C<SQLite> database that would simply run
604 C<$sql_migration_dir/SQLite/schema/1/001-auto.sql>. Next,
606 $dm->upgrade_single_step([1,2])
608 would run C<$sql_migration_dir/SQLite/up/1-2/001-auto.sql> followed by
609 C<$sql_migration_dir/_common/up/1-2/002-generate-customers.pl>.
611 Now, a C<.pl> file doesn't have to be in the C<_common> directory, but most of
612 the time it probably should be, since perl scripts will mostly be database
615 C<_generic> exists for when you for some reason are sure that your SQL is
616 generic enough to run on all databases. Good luck with that one.
618 Note that unlike most steps in the process, C<preinstall> will not run SQL, as
619 there may not even be an database at preinstall time. It will run perl scripts
620 just like the other steps in the process, but nothing is passed to them.
621 Until people have used this more it will remain freeform, but a recommended use
622 of preinstall is to have it prompt for username and password, and then call the
623 appropriate C<< CREATE DATABASE >> commands etc.
627 A perl script for this tool is very simple. It merely needs to contain an
628 anonymous sub that takes a L<DBIx::Class::Schema> as it's only argument.
629 A very basic perl script might look like:
639 $schema->resultset('Users')->create({
647 The L<DBIx::Class::Schema> (B<required>) that is used to talk to the database
648 and generate the DDL.
652 The L<DBIx::Class::Storage> that is I<actually> used to talk to the database
653 and generate the DDL. This is automatically created with L</_build_storage>.
655 =attr sql_translator_args
657 The arguments that get passed to L<SQL::Translator> when it's used.
659 =attr script_directory
661 The directory (default C<'sql'>) that scripts are stored in
665 The types of databases (default C<< [qw( MySQL SQLite PostgreSQL )] >>) to
670 Set to true (which is the default) to wrap all upgrades and deploys in a single
675 The version the schema on your harddrive is at. Defaults to
676 C<< $self->schema->schema_version >>.
680 =head2 __ddl_consume_with_prefix
682 $dm->__ddl_consume_with_prefix( 'SQLite', [qw( 1.00 1.01 )], 'up' )
684 This is the meat of the multi-file upgrade/deploy stuff. It returns a list of
685 files in the order that they should be run for a generic "type" of upgrade.
686 You should not be calling this in user code.
688 =head2 _ddl_schema_consume_filenames
690 $dm->__ddl_schema_consume_filenames( 'SQLite', [qw( 1.00 )] )
692 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
695 =head2 _ddl_schema_produce_filename
697 $dm->__ddl_schema_produce_filename( 'SQLite', [qw( 1.00 )] )
699 Returns a single file in which an initial schema will be stored.
701 =head2 _ddl_schema_up_consume_filenames
703 $dm->_ddl_schema_up_consume_filenames( 'SQLite', [qw( 1.00 )] )
705 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
708 =head2 _ddl_schema_down_consume_filenames
710 $dm->_ddl_schema_down_consume_filenames( 'SQLite', [qw( 1.00 )] )
712 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for a
715 =head2 _ddl_schema_up_produce_filenames
717 $dm->_ddl_schema_up_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
719 Returns a single file in which the sql to upgrade from one schema to another
722 =head2 _ddl_schema_down_produce_filename
724 $dm->_ddl_schema_down_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
726 Returns a single file in which the sql to downgrade from one schema to another
729 =head2 _resultsource_install_filename
731 my $filename_fn = $dm->_resultsource_install_filename('User');
732 $dm->$filename_fn('SQLite', '1.00')
734 Returns a function which in turn returns a single filename used to install a
735 single resultsource. Weird interface is convenient for me. Deal with it.
737 =head2 _run_sql_and_perl
739 $dm->_run_sql_and_perl([qw( list of filenames )])
741 Simply put, this runs the list of files passed to it. If the file ends in
742 C<.sql> it runs it as sql and if it ends in C<.pl> it runs it as a perl file.
744 Depending on L</txn_wrap> all of the files run will be wrapped in a single
747 =head2 _prepare_install
749 $dm->_prepare_install({ add_drop_table => 0 }, sub { 'file_to_create' })
751 Generates the sql file for installing the database. First arg is simply
752 L<SQL::Translator> args and the second is a coderef that returns the filename
755 =head2 _prepare_changegrade
757 $dm->_prepare_changegrade('1.00', '1.01', [qw( 1.00 1.01)], 'up')
759 Generates the sql file for migrating from one schema version to another. First
760 arg is the version to start from, second is the version to go to, third is the
761 L<version set|DBIx::Class::DeploymentHandler/VERSION SET>, and last is the
762 direction of the changegrade, be it 'up' or 'down'.
764 =head2 _read_sql_file
766 $dm->_read_sql_file('foo.sql')
768 Reads a sql file and returns lines in an C<ArrayRef>. Strips out comments,
769 transactions, and blank lines.