1 package DBIx::Class::DeploymentHandler::DeployMethod::SQL::Translator;
4 # ABSTRACT: Manage your SQL and Perl migrations in nicely laid out directories
7 use Carp qw( carp croak );
8 use DBIx::Class::DeploymentHandler::Logger;
9 use Log::Contextual qw(:log :dlog), -default_logger =>
10 DBIx::Class::DeploymentHandler::Logger->new({
11 env_prefix => 'DBICDH'
14 use Method::Signatures::Simple;
18 require SQL::Translator::Diff;
20 require DBIx::Class::Storage; # loaded for type constraint
21 use DBIx::Class::DeploymentHandler::Types;
23 use File::Path 'mkpath';
24 use File::Spec::Functions;
26 with 'DBIx::Class::DeploymentHandler::HandlesDeploy';
29 isa => 'DBIx::Class::Schema',
35 isa => 'DBIx::Class::Storage',
40 method _build_storage {
41 my $s = $self->schema->storage;
42 $s->_determine_driver;
46 has sql_translator_args => (
49 default => sub { {} },
51 has script_directory => (
60 isa => 'DBIx::Class::DeploymentHandler::Databases',
62 default => sub { [qw( MySQL SQLite PostgreSQL )] },
71 has schema_version => (
77 # this will probably never get called as the DBICDH
78 # will be passing down a schema_version normally, which
79 # is built the same way, but we leave this in place
80 method _build_schema_version { $self->schema->schema_version }
82 method __ddl_consume_with_prefix($type, $versions, $prefix) {
83 my $base_dir = $self->script_directory;
85 my $main = catfile( $base_dir, $type );
86 my $generic = catfile( $base_dir, '_generic' );
88 catfile( $base_dir, '_common', $prefix, join q(-), @{$versions} );
92 $dir = catfile($main, $prefix, join q(-), @{$versions})
93 } elsif (-d $generic) {
94 $dir = catfile($generic, $prefix, join q(-), @{$versions});
96 croak "neither $main or $generic exist; please write/generate some SQL";
99 opendir my($dh), $dir;
100 my %files = map { $_ => "$dir/$_" } grep { /\.(?:sql|pl|sql-\w+)$/ && -f "$dir/$_" } readdir $dh;
104 opendir my($dh), $common;
105 for my $filename (grep { /\.(?:sql|pl)$/ && -f catfile($common,$_) } readdir $dh) {
106 unless ($files{$filename}) {
107 $files{$filename} = catfile($common,$filename);
113 return [@files{sort keys %files}]
116 method _ddl_preinstall_consume_filenames($type, $version) {
117 $self->__ddl_consume_with_prefix($type, [ $version ], 'preinstall')
120 method _ddl_schema_consume_filenames($type, $version) {
121 $self->__ddl_consume_with_prefix($type, [ $version ], 'schema')
124 method _ddl_protoschema_produce_filename($version) {
125 my $dirname = catfile( $self->script_directory, '_protoschema', $version );
126 mkpath($dirname) unless -d $dirname;
128 return catfile( $dirname, '001-auto.yml' );
131 method _ddl_schema_produce_filename($type, $version) {
132 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
133 mkpath($dirname) unless -d $dirname;
135 return catfile( $dirname, '001-auto.sql' );
138 method _ddl_schema_up_consume_filenames($type, $versions) {
139 $self->__ddl_consume_with_prefix($type, $versions, 'up')
142 method _ddl_schema_down_consume_filenames($type, $versions) {
143 $self->__ddl_consume_with_prefix($type, $versions, 'down')
146 method _ddl_schema_up_produce_filename($type, $versions) {
147 my $dir = $self->script_directory;
149 my $dirname = catfile( $dir, $type, 'up', join q(-), @{$versions});
150 mkpath($dirname) unless -d $dirname;
152 return catfile( $dirname, '001-auto.sql'
156 method _ddl_schema_down_produce_filename($type, $versions, $dir) {
157 my $dirname = catfile( $dir, $type, 'down', join q(-), @{$versions} );
158 mkpath($dirname) unless -d $dirname;
160 return catfile( $dirname, '001-auto.sql');
163 method _run_sql_array($sql) {
164 my $storage = $self->storage;
167 $_ && # remove blank lines
168 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
170 s/^\s+//; s/\s+$//; # trim whitespace
171 join '', grep { !/^--/ } split /\n/ # remove comments
174 Dlog_trace { "Running SQL $_" } $sql;
175 foreach my $line (@{$sql}) {
176 $storage->_query_start($line);
177 # the whole reason we do this is so that we can see the line that was run
179 $storage->dbh_do (sub { $_[1]->do($line) });
182 die "$_ (running line '$line')"
184 $storage->_query_end($line);
186 return join "\n", @$sql
189 method _run_sql($filename) {
190 log_debug { "Running SQL from $filename" };
191 return $self->_run_sql_array($self->_read_sql_file($filename));
194 method _run_perl($filename) {
195 log_debug { "Running Perl from $filename" };
196 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
198 no warnings 'redefine';
199 my $fn = eval "$filedata";
201 Dlog_trace { "Running Perl $_" } $fn;
204 carp "$filename failed to compile: $@";
205 } elsif (ref $fn eq 'CODE') {
208 carp "$filename should define an anonymouse sub that takes a schema but it didn't!";
214 method _run_serialized_sql($filename, $type) {
215 if ($type eq 'json') {
217 $json ||= JSON->new->pretty;
218 my @sql = @{$json->decode($filename)};
220 croak "A file ($filename) got to deploy that wasn't sql or perl!";
226 method _run_sql_and_perl($filenames) {
227 my @files = @{$filenames};
228 my $guard = $self->schema->txn_scope_guard if $self->txn_wrap;
231 for my $filename (@files) {
232 if ($filename =~ /\.sql$/) {
233 $sql .= $self->_run_sql($filename)
234 } elsif ( $filename =~ /\.sql-(\w+)$/ ) {
235 $sql .= $self->_run_serialized_sql($filename, $1)
236 } elsif ( $filename =~ /\.pl$/ ) {
237 $self->_run_perl($filename)
239 croak "A file ($filename) got to deploy that wasn't sql or perl!";
243 $guard->commit if $self->txn_wrap;
250 my $version = (shift @_ || {})->{version} || $self->schema_version;
251 log_info { "deploying version $version" };
253 return $self->_run_sql_and_perl($self->_ddl_schema_consume_filenames(
254 $self->storage->sqlt_type,
262 my $version = $args->{version} || $self->schema_version;
263 log_info { "preinstalling version $version" };
264 my $storage_type = $args->{storage_type} || $self->storage->sqlt_type;
266 my @files = @{$self->_ddl_preinstall_consume_filenames(
271 for my $filename (@files) {
272 # We ignore sql for now (till I figure out what to do with it)
273 if ( $filename =~ /^(.+)\.pl$/ ) {
274 my $filedata = do { local( @ARGV, $/ ) = $filename; <> };
276 no warnings 'redefine';
277 my $fn = eval "$filedata";
281 carp "$filename failed to compile: $@";
282 } elsif (ref $fn eq 'CODE') {
285 carp "$filename should define an anonymous sub but it didn't!";
288 croak "A file ($filename) got to preinstall_scripts that wasn't sql or perl!";
293 sub _prepare_install {
295 my $sqltargs = { %{$self->sql_translator_args}, %{shift @_} };
297 my $schema = $self->schema;
298 my $databases = $self->databases;
299 my $dir = $self->script_directory;
300 my $version = $self->schema_version;
302 my $sqlt = SQL::Translator->new({
304 parser => 'SQL::Translator::Parser::YAML',
308 my $yaml_filename = $self->_ddl_protoschema_produce_filename($version);
310 foreach my $db (@$databases) {
312 $sqlt->producer($db);
314 my $filename = $self->$to_file($db, $version, $dir);
316 carp "Overwriting existing DDL file - $filename";
320 my $sql = $sqlt->translate($yaml_filename);
322 carp("Failed to translate to $db, skipping. (" . $sqlt->error . ")");
325 open my $file, q(>), $filename;
331 sub _resultsource_install_filename {
332 my ($self, $source_name) = @_;
334 my ($self, $type, $version) = @_;
335 my $dirname = catfile( $self->script_directory, $type, 'schema', $version );
336 mkpath($dirname) unless -d $dirname;
338 return catfile( $dirname, "001-auto-$source_name.sql" );
342 sub install_resultsource {
343 my ($self, $args) = @_;
344 my $source = $args->{result_source};
345 my $version = $args->{version};
346 log_info { 'installing_resultsource ' . $source->source_name . ", version $version" };
347 my $rs_install_file =
348 $self->_resultsource_install_filename($source->source_name);
351 $self->$rs_install_file(
352 $self->storage->sqlt_type,
356 $self->_run_sql_and_perl($files);
359 sub prepare_resultsource_install {
361 my $source = (shift @_)->{result_source};
362 log_info { 'preparing install for resultsource ' . $source->source_name };
364 my $filename = $self->_resultsource_install_filename($source->source_name);
365 $self->_prepare_install({
366 parser_args => { sources => [$source->source_name], }
371 log_info { 'preparing deploy' };
373 $self->_generate_protoschema;
374 $self->_prepare_install({}, '_ddl_schema_produce_filename');
377 sub prepare_upgrade {
378 my ($self, $args) = @_;
380 "preparing upgrade from $args->{from_version} to $args->{to_version}"
382 $self->_prepare_changegrade(
383 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'up'
387 sub prepare_downgrade {
388 my ($self, $args) = @_;
390 "preparing downgrade from $args->{from_version} to $args->{to_version}"
392 $self->_prepare_changegrade(
393 $args->{from_version}, $args->{to_version}, $args->{version_set}, 'down'
397 method _prepare_changegrade($from_version, $to_version, $version_set, $direction) {
398 my $schema = $self->schema;
399 my $databases = $self->databases;
400 my $dir = $self->script_directory;
401 my $sqltargs = $self->sql_translator_args;
403 my $schema_version = $self->schema_version;
407 ignore_constraint_names => 1,
408 ignore_index_names => 1,
412 my $sqlt = SQL::Translator->new( $sqltargs );
414 $sqlt->parser('SQL::Translator::Parser::DBIx::Class');
415 my $sqlt_schema = $sqlt->translate( data => $schema )
416 or croak($sqlt->error);
418 foreach my $db (@$databases) {
420 $sqlt->{schema} = $sqlt_schema;
421 $sqlt->producer($db);
423 my $prefilename = $self->_ddl_schema_produce_filename($db, $from_version, $dir);
424 unless(-e $prefilename) {
425 carp("No previous schema file found ($prefilename)");
428 my $diff_file_method = "_ddl_schema_${direction}_produce_filename";
429 my $diff_file = $self->$diff_file_method($db, $version_set, $dir );
431 carp("Overwriting existing $direction-diff file - $diff_file");
437 my $t = SQL::Translator->new({
443 $t->parser( $db ) # could this really throw an exception?
446 my $out = $t->translate( $prefilename )
449 $source_schema = $t->schema;
451 $source_schema->name( $prefilename )
452 unless $source_schema->name;
455 # The "new" style of producers have sane normalization and can support
456 # diffing a SQL file against a DBIC->SQLT schema. Old style ones don't
457 # And we have to diff parsed SQL against parsed SQL.
458 my $dest_schema = $sqlt_schema;
460 unless ( "SQL::Translator::Producer::$db"->can('preprocess_schema') ) {
461 my $t = SQL::Translator->new({
467 $t->parser( $db ) # could this really throw an exception?
470 my $filename = $self->_ddl_schema_produce_filename($db, $to_version, $dir);
471 my $out = $t->translate( $filename )
474 $dest_schema = $t->schema;
476 $dest_schema->name( $filename )
477 unless $dest_schema->name;
480 my $diff = SQL::Translator::Diff::schema_diff(
485 open my $file, q(>), $diff_file;
491 method _read_sql_file($file) {
494 open my $fh, '<', $file;
495 my @data = split /;\n/, join '', <$fh>;
499 $_ && # remove blank lines
500 !/^(BEGIN|BEGIN TRANSACTION|COMMIT)/ # strip txn's
502 s/^\s+//; s/\s+$//; # trim whitespace
503 join '', grep { !/^--/ } split /\n/ # remove comments
509 sub downgrade_single_step {
511 my $version_set = (shift @_)->{version_set};
512 Dlog_info { "downgrade_single_step'ing $_" } $version_set;
514 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_down_consume_filenames(
515 $self->storage->sqlt_type,
522 sub upgrade_single_step {
524 my $version_set = (shift @_)->{version_set};
525 Dlog_info { "upgrade_single_step'ing $_" } $version_set;
527 my $sql = $self->_run_sql_and_perl($self->_ddl_schema_up_consume_filenames(
528 $self->storage->sqlt_type,
534 sub _generate_protoschema {
537 = $self->_ddl_protoschema_produce_filename($self->schema_version);
539 my $sqlt = SQL::Translator->new({
540 parser => 'SQL::Translator::Parser::DBIx::Class',
541 producer => 'SQL::Translator::Producer::YAML',
542 parser_args => { package => $self->schema },
543 %{ $self->sql_translator_args }
546 my $yml = $sqlt->translate;
548 croak("Failed to translate to YAML: " . $sqlt->error)
552 carp "Overwriting existing DDL-YML file - $filename";
556 open my $file, q(>), $filename;
561 __PACKAGE__->meta->make_immutable;
565 # vim: ts=2 sw=2 expandtab
571 This class is the meat of L<DBIx::Class::DeploymentHandler>. It takes
572 care of generating serialized sql files representing schemata as well
573 as serialized sql files to move from one version of a schema to the rest.
574 One of the hallmark features of this class is that it allows for multiple sql
575 files for deploy and upgrade, allowing developers to fine tune deployment.
576 In addition it also allows for perl files to be run
577 at any stage of the process.
579 For basic usage see L<DBIx::Class::DeploymentHandler::HandlesDeploy>. What's
580 documented here is extra fun stuff or private methods.
582 =head1 DIRECTORY LAYOUT
584 Arguably this is the best feature of L<DBIx::Class::DeploymentHandler>. It's
585 heavily based upon L<DBIx::Migration::Directories>, but has some extensions and
586 modifications, so even if you are familiar with it, please read this. I feel
587 like the best way to describe the layout is with the following example:
593 | | `- 001-auto.sql-json
596 | | `- 001-auto.sql-json
599 | | `- 001-auto.sql-json
601 | `- 001-auto.sql-json
605 | | `- 002-remove-customers.pl
608 | `- 002-generate-customers.pl
612 | | `- 001-auto.sql-json
615 | | `- 001-auto.sql-json
618 | |- 001-auto.sql-json
619 | `- 002-create-stored-procedures.sql
623 | `- 001-auto.sql-json
626 | |- 001-create_database.pl
627 | `- 002-create_users_and_permissions.pl
630 | `- 001-auto.sql-json
635 So basically, the code
639 on an C<SQLite> database that would simply run
640 C<$sql_migration_dir/SQLite/schema/1/001-auto.sql-json>. Next,
642 $dm->upgrade_single_step([1,2])
644 would run C<$sql_migration_dir/SQLite/up/1-2/001-auto.sql-json> followed by
645 C<$sql_migration_dir/_common/up/1-2/002-generate-customers.pl>.
647 C<.pl> files don't have to be in the C<_common> directory, but most of the time
648 they should be, because perl scripts are generally be database independent.
650 C<_generic> exists for when you for some reason are sure that your SQL is
651 generic enough to run on all databases. Good luck with that one.
653 Note that unlike most steps in the process, C<preinstall> will not run SQL, as
654 there may not even be an database at preinstall time. It will run perl scripts
655 just like the other steps in the process, but nothing is passed to them.
656 Until people have used this more it will remain freeform, but a recommended use
657 of preinstall is to have it prompt for username and password, and then call the
658 appropriate C<< CREATE DATABASE >> commands etc.
660 =head1 SERIALIZED SQL
662 The SQL that this module generates and uses is serialized into an array of
663 SQL statements. The reason being that some databases handle multiple
664 statements in a single execution differently. Generally you do not need to
665 worry about this as these are scripts generated for you. If you find that
666 you are editing them on a regular basis something is wrong and you either need
667 to submit a bug or consider writing extra serialized SQL or Perl scripts to run
668 before or after the automatically generated script.
670 B<NOTE:> Currently the SQL is serialized into JSON. I am willing to merge in
671 patches that will allow more serialization formats if you want that feature,
672 but if you do send me a patch for that realize that I do not want to add YAML
673 support or whatever, I would rather add a generic method of adding any
674 serialization format.
678 A perl script for this tool is very simple. It merely needs to contain an
679 anonymous sub that takes a L<DBIx::Class::Schema> as it's only argument.
680 A very basic perl script might look like:
690 $schema->resultset('Users')->create({
698 The L<DBIx::Class::Schema> (B<required>) that is used to talk to the database
699 and generate the DDL.
703 The L<DBIx::Class::Storage> that is I<actually> used to talk to the database
704 and generate the DDL. This is automatically created with L</_build_storage>.
706 =attr sql_translator_args
708 The arguments that get passed to L<SQL::Translator> when it's used.
710 =attr script_directory
712 The directory (default C<'sql'>) that scripts are stored in
716 The types of databases (default C<< [qw( MySQL SQLite PostgreSQL )] >>) to
721 Set to true (which is the default) to wrap all upgrades and deploys in a single
726 The version the schema on your harddrive is at. Defaults to
727 C<< $self->schema->schema_version >>.
731 =head2 __ddl_consume_with_prefix
733 $dm->__ddl_consume_with_prefix( 'SQLite', [qw( 1.00 1.01 )], 'up' )
735 This is the meat of the multi-file upgrade/deploy stuff. It returns a list of
736 files in the order that they should be run for a generic "type" of upgrade.
737 You should not be calling this in user code.
739 =head2 _ddl_schema_consume_filenames
741 $dm->__ddl_schema_consume_filenames( 'SQLite', [qw( 1.00 )] )
743 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
746 =head2 _ddl_schema_produce_filename
748 $dm->__ddl_schema_produce_filename( 'SQLite', [qw( 1.00 )] )
750 Returns a single file in which an initial schema will be stored.
752 =head2 _ddl_schema_up_consume_filenames
754 $dm->_ddl_schema_up_consume_filenames( 'SQLite', [qw( 1.00 )] )
756 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for an
759 =head2 _ddl_schema_down_consume_filenames
761 $dm->_ddl_schema_down_consume_filenames( 'SQLite', [qw( 1.00 )] )
763 Just a curried L</__ddl_consume_with_prefix>. Get's a list of files for a
766 =head2 _ddl_schema_up_produce_filenames
768 $dm->_ddl_schema_up_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
770 Returns a single file in which the sql to upgrade from one schema to another
773 =head2 _ddl_schema_down_produce_filename
775 $dm->_ddl_schema_down_produce_filename( 'SQLite', [qw( 1.00 1.01 )] )
777 Returns a single file in which the sql to downgrade from one schema to another
780 =head2 _resultsource_install_filename
782 my $filename_fn = $dm->_resultsource_install_filename('User');
783 $dm->$filename_fn('SQLite', '1.00')
785 Returns a function which in turn returns a single filename used to install a
786 single resultsource. Weird interface is convenient for me. Deal with it.
788 =head2 _run_sql_and_perl
790 $dm->_run_sql_and_perl([qw( list of filenames )])
792 Simply put, this runs the list of files passed to it. If the file ends in
793 C<.sql> it runs it as sql and if it ends in C<.pl> it runs it as a perl file.
795 Depending on L</txn_wrap> all of the files run will be wrapped in a single
798 =head2 _prepare_install
800 $dm->_prepare_install({ add_drop_table => 0 }, sub { 'file_to_create' })
802 Generates the sql file for installing the database. First arg is simply
803 L<SQL::Translator> args and the second is a coderef that returns the filename
806 =head2 _prepare_changegrade
808 $dm->_prepare_changegrade('1.00', '1.01', [qw( 1.00 1.01)], 'up')
810 Generates the sql file for migrating from one schema version to another. First
811 arg is the version to start from, second is the version to go to, third is the
812 L<version set|DBIx::Class::DeploymentHandler/VERSION SET>, and last is the
813 direction of the changegrade, be it 'up' or 'down'.
815 =head2 _read_sql_file
817 $dm->_read_sql_file('foo.sql')
819 Reads a sql file and returns lines in an C<ArrayRef>. Strips out comments,
820 transactions, and blank lines.