1 package DBIx::Class::DeploymentHandler::DeployMethod::SQL::Translator;
4 # ABSTRACT: Manage your SQL and Perl migrations in nicely laid out directories
7 use Carp qw( carp croak );
8 use Log::Contextual::WarnLogger;
9 use Log::Contextual qw(:log :dlog), -default_logger => Log::Contextual::WarnLogger->new({
10 env_prefix => 'DBICDH'
12 use Data::Dumper::Concise;
14 use Method::Signatures::Simple;
18 require SQL::Translator::Diff;
20 require DBIx::Class::Storage; # loaded for type constraint
21 use DBIx::Class::DeploymentHandler::Types;
23 use File::Path 'mkpath';
24 use File::Spec::Functions;
26 with 'DBIx::Class::DeploymentHandler::HandlesDeploy';
29 isa => 'DBIx::Class::Schema',
35 isa => 'DBIx::Class::Storage',
40 method _build_storage {
41 my $s = $self->schema->storage;
42 $s->_determine_driver;
46 has sql_translator_args => (
49 default => sub { {} },
51 has script_directory => (
60 isa => 'DBIx::Class::DeploymentHandler::Databases',
62 default => sub { [qw( MySQL SQLite PostgreSQL )] },
71 has schema_version => (
77 # this will probably never get called as the DBICDH
78 # will be passing down a schema_version normally, which
79 # is built the same way
80 method _build_schema_version { $self->schema->schema_version }
82 method __ddl_consume_with_prefix($type, $versions, $prefix) {
83 my $base_dir = $self->script_directory;
85 my $main = catfile( $base_dir, $type );
86 my $generic = catfile( $base_dir, '_generic' );
88 catfile( $base_dir, '_common', $prefix, join q(-), @{$versions} );
92 $dir = catfile($main, $prefix, join q(-), @{$versions})
93 } elsif (-d $generic) {
94 $dir = catfile($generic, $prefix, join q(-), @{$versions});
96 croak "neither $main or $generic exist; please write/generate some SQL";
99 opendir my($dh), $dir;
100 my %files = map { $_ => "$dir/$_" } grep { /\.(?:sql|pl)$/ && -f "$dir/$_" } readdir $dh;
104 opendir my($dh), $common;
105 for my $filename (grep { /\.(?:sql|pl)$/ && -f catfile($common,$_) } readdir $dh) {
106 unless ($files{$filename}) {
107 $files{$filename} = catfile($common,$filename);
113 return [@files{sort keys %files}]
116 method _ddl_preinstall_consume_filenames($type, $version) {
117 $self->__ddl_consume_with_prefix($type, [ $version ], 'preinstall')
120 method _ddl_schema_consume_filenames($type, $version) {
121 $self->__ddl_consume_with_prefix($type, [ $version ], 'schema')
124 method _ddl_schema_produce_filename($type, $version) {
125 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
126 mkpath($dirname) unless -d $dirname;
128 return catfile( $dirname, '001-auto.sql' );
131 method _ddl_schema_up_consume_filenames($type, $versions) {
132 $self->__ddl_consume_with_prefix($type, $versions, 'up')
135 method _ddl_schema_down_consume_filenames($type, $versions) {
136 $self->__ddl_consume_with_prefix($type, $versions, 'down')
139 method _ddl_schema_up_produce_filename($type, $versions) {
140 my $dir = $self->script_directory;
142 my $dirname = catfile( $dir, $type, 'up', join q(-), @{$versions});
143 mkpath($dirname) unless -d $dirname;
145 return catfile( $dirname, '001-auto.sql'
149 method _ddl_schema_down_produce_filename($type, $versions, $dir) {
150 my $dirname = catfile( $dir, $type, 'down', join q(-), @{$versions} );
151 mkpath($dirname) unless -d $dirname;
153 return catfile( $dirname, '001-auto.sql');
156 method _run_sql_and_perl($filenames) {
157 my @files = @{$filenames};
158 my $storage = $self->storage;
161 my $guard = $self->schema->txn_scope_guard if $self->txn_wrap;
164 for my $filename (@files) {
165 if ($filename =~ /\.sql$/) {
166 log_debug { "[DBICDH] Running SQL from $filename" };
167 my @sql = @{$self->_read_sql_file($filename)};
168 $sql .= join "\n", @sql;
169 log_trace { "[DBICDH] Running SQL $sql" };
171 foreach my $line (@sql) {
172 $storage->_query_start($line);
174 # do a dbh_do cycle here, as we need some error checking in
175 # place (even though we will ignore errors)
176 $storage->dbh_do (sub { $_[1]->do($line) });
179 carp "$_ (running '${line}')"
181 $storage->_query_end($line);
183 } elsif ( $filename =~ /^(.+)\.pl$/ ) {
184 log_debug { "[DBICDH] Running Perl from $filename" };
185 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
187 no warnings 'redefine';
188 my $fn = eval "$filedata";
190 log_trace { '[DBICDH] Running Perl ' . Dumper($fn) };
193 carp "$filename failed to compile: $@";
194 } elsif (ref $fn eq 'CODE') {
197 carp "$filename should define an anonymouse sub that takes a schema but it didn't!";
200 croak "A file ($filename) got to deploy that wasn't sql or perl!";
204 $guard->commit if $self->txn_wrap;
211 my $version = (shift @_ || {})->{version} || $self->schema_version;
212 log_info { "[DBICDH] deploying version $version" };
214 return $self->_run_sql_and_perl($self->_ddl_schema_consume_filenames(
215 $self->storage->sqlt_type,
223 my $version = $args->{version} || $self->schema_version;
224 log_info { "[DBICDH] preinstalling version $version" };
225 my $storage_type = $args->{storage_type} || $self->storage->sqlt_type;
227 my @files = @{$self->_ddl_preinstall_consume_filenames(
232 for my $filename (@files) {
233 # We ignore sql for now (till I figure out what to do with it)
234 if ( $filename =~ /^(.+)\.pl$/ ) {
235 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
237 no warnings 'redefine';
238 my $fn = eval "$filedata";
242 carp "$filename failed to compile: $@";
243 } elsif (ref $fn eq 'CODE') {
246 carp "$filename should define an anonymous sub but it didn't!";
249 croak "A file ($filename) got to preinstall_scripts that wasn't sql or perl!";
254 sub _prepare_install {
256 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
258 my $schema = $self->schema;
259 my $databases = $self->databases;
260 my $dir = $self->script_directory;
261 my $version = $self->schema_version;
263 my $sqlt = SQL::Translator->new({
265 ignore_constraint_names => 1,
266 ignore_index_names => 1,
267 parser => 'SQL::Translator::Parser::DBIx::Class',
271 my $sqlt_schema = $sqlt->translate( data => $schema )
272 or croak($sqlt->error);
274 foreach my $db (@$databases) {
276 $sqlt->{schema} = $sqlt_schema;
277 $sqlt->producer($db);
279 my $filename = $self->$to_file($db, $version, $dir);
281 carp "Overwriting existing DDL file - $filename";
285 my $output = $sqlt->translate;
287 carp("Failed to translate to $db, skipping. (" . $sqlt->error . ")");
290 open my $file, q(>), $filename;
291 print {$file} $output;
296 sub _resultsource_install_filename {
297 my ($self, $source_name) = @_;
299 my ($self, $type, $version) = @_;
300 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
301 mkpath($dirname) unless -d $dirname;
303 return catfile( $dirname, "001-auto-$source_name.sql" );
307 sub install_resultsource {
308 my ($self, $args) = @_;
309 my $source = $args->{result_source};
310 my $version = $args->{version};
311 log_info { '[DBICDH] installing_resultsource ' . $source->source_name . ", version $version" };
312 my $rs_install_file =
313 $self->_resultsource_install_filename($source->source_name);
316 $self->$rs_install_file(
317 $self->storage->sqlt_type,
321 $self->_run_sql_and_perl($files);
324 sub prepare_resultsource_install {
326 my $source = (shift @_)->{result_source};
327 log_info { '[DBICDH] preparing install for resultsource ' . $source->source_name };
329 my $filename = $self->_resultsource_install_filename($source->source_name);
330 $self->_prepare_install({
331 parser_args => { sources => [$source->source_name], }
336 log_info { '[DBICDH] preparing deploy' };
338 $self->_prepare_install({}, '_ddl_schema_produce_filename');
341 sub prepare_upgrade {
342 my ($self, $args) = @_;
344 '[DBICDH] preparing upgrade ' .
345 "from $args->{from_version} to $args->{to_version}"
347 $self->_prepare_changegrade(
348 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'up'
352 sub prepare_downgrade {
353 my ($self, $args) = @_;
355 '[DBICDH] preparing downgrade ' .
356 "from $args->{from_version} to $args->{to_version}"
358 $self->_prepare_changegrade(
359 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'down'
363 method _prepare_changegrade($from_version, $to_version, $version_set, $direction) {
364 my $schema = $self->schema;
365 my $databases = $self->databases;
366 my $dir = $self->script_directory;
367 my $sqltargs = $self->sql_translator_args;
369 my $schema_version = $self->schema_version;
373 ignore_constraint_names => 1,
374 ignore_index_names => 1,
378 my $sqlt = SQL::Translator->new( $sqltargs );
380 $sqlt->parser('SQL::Translator::Parser::DBIx::Class');
381 my $sqlt_schema = $sqlt->translate( data => $schema )
382 or croak($sqlt->error);
384 foreach my $db (@$databases) {
386 $sqlt->{schema} = $sqlt_schema;
387 $sqlt->producer($db);
389 my $prefilename = $self->_ddl_schema_produce_filename($db, $from_version, $dir);
390 unless(-e $prefilename) {
391 carp("No previous schema file found ($prefilename)");
394 my $diff_file_method = "_ddl_schema_${direction}_produce_filename";
395 my $diff_file = $self->$diff_file_method($db, $version_set, $dir );
397 carp("Overwriting existing $direction-diff file - $diff_file");
403 my $t = SQL::Translator->new({
409 $t->parser( $db ) # could this really throw an exception?
412 my $out = $t->translate( $prefilename )
415 $source_schema = $t->schema;
417 $source_schema->name( $prefilename )
418 unless $source_schema->name;
421 # The "new" style of producers have sane normalization and can support
422 # diffing a SQL file against a DBIC->SQLT schema. Old style ones don't
423 # And we have to diff parsed SQL against parsed SQL.
424 my $dest_schema = $sqlt_schema;
426 unless ( "SQL::Translator::Producer::$db"->can('preprocess_schema') ) {
427 my $t = SQL::Translator->new({
433 $t->parser( $db ) # could this really throw an exception?
436 my $filename = $self->_ddl_schema_produce_filename($db, $to_version, $dir);
437 my $out = $t->translate( $filename )
440 $dest_schema = $t->schema;
442 $dest_schema->name( $filename )
443 unless $dest_schema->name;
446 my $diff = SQL::Translator::Diff::schema_diff(
451 open my $file, q(>), $diff_file;
457 method _read_sql_file($file) {
460 open my $fh, '<', $file;
461 my @data = split /;\n/, join '', <$fh>;
465 $_ && # remove blank lines
466 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
468 s/^\s+//; s/\s+$//; # trim whitespace
469 join '', grep { !/^--/ } split /\n/ # remove comments
475 sub downgrade_single_step {
477 my $version_set = (shift @_)->{version_set};
478 log_info { qq([DBICDH] downgrade_single_step'ing ) . Dumper($version_set) };
480 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_down_consume_filenames(
481 $self->storage->sqlt_type,
488 sub upgrade_single_step {
490 my $version_set = (shift @_)->{version_set};
491 log_info { qq([DBICDH] upgrade_single_step'ing ) . Dumper($version_set) };
493 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_up_consume_filenames(
494 $self->storage->sqlt_type,
500 __PACKAGE__->meta->make_immutable;
504 # vim: ts=2 sw=2 expandtab
510 This class is the meat of L<DBIx::Class::DeploymentHandler>. It takes care of
511 generating sql files representing schemata as well as sql files to move from
512 one version of a schema to the rest. One of the hallmark features of this
513 class is that it allows for multiple sql files for deploy and upgrade, allowing
514 developers to fine tune deployment. In addition it also allows for perl files
515 to be run at any stage of the process.
517 For basic usage see L<DBIx::Class::DeploymentHandler::HandlesDeploy>. What's
518 documented here is extra fun stuff or private methods.
520 =head1 DIRECTORY LAYOUT
522 Arguably this is the best feature of L<DBIx::Class::DeploymentHandler>. It's
523 heavily based upon L<DBIx::Migration::Directories>, but has some extensions and
524 modifications, so even if you are familiar with it, please read this. I feel
525 like the best way to describe the layout is with the following example:
543 | | `- 002-remove-customers.pl
546 | `- 002-generate-customers.pl
557 | `- 002-create-stored-procedures.sql
564 | |- 001-create_database.pl
565 | `- 002-create_users_and_permissions.pl
573 So basically, the code
577 on an C<SQLite> database that would simply run
578 C<$sql_migration_dir/SQLite/schema/1/001-auto.sql>. Next,
580 $dm->upgrade_single_step([1,2])
582 would run C<$sql_migration_dir/SQLite/up/1-2/001-auto.sql> followed by
583 C<$sql_migration_dir/_common/up/1-2/002-generate-customers.pl>.
585 Now, a C<.pl> file doesn't have to be in the C<_common> directory, but most of
586 the time it probably should be, since perl scripts will mostly be database
589 C<_generic> exists for when you for some reason are sure that your SQL is
590 generic enough to run on all databases. Good luck with that one.
592 Note that unlike most steps in the process, C<preinstall> will not run SQL, as
593 there may not even be an database at preinstall time. It will run perl scripts
594 just like the other steps in the process, but nothing is passed to them.
595 Until people have used this more it will remain freeform, but a recommended use
596 of preinstall is to have it prompt for username and password, and then call the
597 appropriate C<< CREATE DATABASE >> commands etc.
601 A perl script for this tool is very simple. It merely needs to contain an
602 anonymous sub that takes a L<DBIx::Class::Schema> as it's only argument.
603 A very basic perl script might look like:
613 $schema->resultset('Users')->create({
621 The L<DBIx::Class::Schema> (B<required>) that is used to talk to the database
622 and generate the DDL.
626 The L<DBIx::Class::Storage> that is I<actually> used to talk to the database
627 and generate the DDL. This is automatically created with L</_build_storage>.
629 =attr sql_translator_args
631 The arguments that get passed to L<SQL::Translator> when it's used.
633 =attr script_directory
635 The directory (default C<'sql'>) that scripts are stored in
639 The types of databases (default C<< [qw( MySQL SQLite PostgreSQL )] >>) to
644 Set to true (which is the default) to wrap all upgrades and deploys in a single
649 The version the schema on your harddrive is at. Defaults to
650 C<< $self->schema->schema_version >>.
654 =head2 __ddl_consume_with_prefix
656 $dm->__ddl_consume_with_prefix( 'SQLite', [qw( 1.00 1.01 )], 'up' )
658 This is the meat of the multi-file upgrade/deploy stuff. It returns a list of
659 files in the order that they should be run for a generic "type" of upgrade.
660 You should not be calling this in user code.
662 =head2 _ddl_schema_consume_filenames
664 $dm->__ddl_schema_consume_filenames( 'SQLite', [qw( 1.00 )] )
666 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
669 =head2 _ddl_schema_produce_filename
671 $dm->__ddl_schema_produce_filename( 'SQLite', [qw( 1.00 )] )
673 Returns a single file in which an initial schema will be stored.
675 =head2 _ddl_schema_up_consume_filenames
677 $dm->_ddl_schema_up_consume_filenames( 'SQLite', [qw( 1.00 )] )
679 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
682 =head2 _ddl_schema_down_consume_filenames
684 $dm->_ddl_schema_down_consume_filenames( 'SQLite', [qw( 1.00 )] )
686 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for a
689 =head2 _ddl_schema_up_produce_filenames
691 $dm->_ddl_schema_up_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
693 Returns a single file in which the sql to upgrade from one schema to another
696 =head2 _ddl_schema_down_produce_filename
698 $dm->_ddl_schema_down_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
700 Returns a single file in which the sql to downgrade from one schema to another
703 =head2 _resultsource_install_filename
705 my $filename_fn = $dm->_resultsource_install_filename('User');
706 $dm->$filename_fn('SQLite', '1.00')
708 Returns a function which in turn returns a single filename used to install a
709 single resultsource. Weird interface is convenient for me. Deal with it.
711 =head2 _run_sql_and_perl
713 $dm->_run_sql_and_perl([qw( list of filenames )])
715 Simply put, this runs the list of files passed to it. If the file ends in
716 C<.sql> it runs it as sql and if it ends in C<.pl> it runs it as a perl file.
718 Depending on L</txn_wrap> all of the files run will be wrapped in a single
721 =head2 _prepare_install
723 $dm->_prepare_install({ add_drop_table => 0 }, sub { 'file_to_create' })
725 Generates the sql file for installing the database. First arg is simply
726 L<SQL::Translator> args and the second is a coderef that returns the filename
729 =head2 _prepare_changegrade
731 $dm->_prepare_changegrade('1.00', '1.01', [qw( 1.00 1.01)], 'up')
733 Generates the sql file for migrating from one schema version to another. First
734 arg is the version to start from, second is the version to go to, third is the
735 L<version set|DBIx::Class::DeploymentHandler/VERSION SET>, and last is the
736 direction of the changegrade, be it 'up' or 'down'.
738 =head2 _read_sql_file
740 $dm->_read_sql_file('foo.sql')
742 Reads a sql file and returns lines in an C<ArrayRef>. Strips out comments,
743 transactions, and blank lines.