Initial code written for transactional isolation
[dbsrgits/DBM-Deep.git] / lib / DBM / Deep.pm
CommitLineData
ffed8b01 1package DBM::Deep;
2
3##
4# DBM::Deep
5#
6# Description:
d0b74c17 7# Multi-level database module for storing hash trees, arrays and simple
8# key/value pairs into FTP-able, cross-platform binary database files.
ffed8b01 9#
d0b74c17 10# Type `perldoc DBM::Deep` for complete documentation.
ffed8b01 11#
12# Usage Examples:
d0b74c17 13# my %db;
14# tie %db, 'DBM::Deep', 'my_database.db'; # standard tie() method
ffed8b01 15#
d0b74c17 16# my $db = new DBM::Deep( 'my_database.db' ); # preferred OO method
17#
18# $db->{my_scalar} = 'hello world';
19# $db->{my_hash} = { larry => 'genius', hashes => 'fast' };
20# $db->{my_array} = [ 1, 2, 3, time() ];
21# $db->{my_complex} = [ 'hello', { perl => 'rules' }, 42, 99 ];
22# push @{$db->{my_array}}, 'another value';
23# my @key_list = keys %{$db->{my_hash}};
24# print "This module " . $db->{my_complex}->[1]->{perl} . "!\n";
ffed8b01 25#
26# Copyright:
d0b74c17 27# (c) 2002-2006 Joseph Huckaby. All Rights Reserved.
28# This program is free software; you can redistribute it and/or
29# modify it under the same terms as Perl itself.
ffed8b01 30##
31
460b1067 32use 5.6.0;
33
ffed8b01 34use strict;
460b1067 35use warnings;
8b957036 36
d8db2929 37our $VERSION = q(0.99_03);
86867f3a 38
596e9574 39use Fcntl qw( :DEFAULT :flock :seek );
12b96196 40
41use Clone::Any '_clone_data';
ffed8b01 42use Digest::MD5 ();
a8fdabda 43use FileHandle::Fmode ();
ffed8b01 44use Scalar::Util ();
ffed8b01 45
696cadb7 46use DBM::Deep::Engine3;
460b1067 47use DBM::Deep::File;
95967a5e 48
ffed8b01 49##
50# Setup constants for users to pass to new()
51##
696cadb7 52sub TYPE_HASH () { DBM::Deep::Engine3->SIG_HASH }
53sub TYPE_ARRAY () { DBM::Deep::Engine3->SIG_ARRAY }
ffed8b01 54
696cadb7 55# This is used in all the children of this class in their TIE<type> methods.
0ca7ea98 56sub _get_args {
57 my $proto = shift;
58
59 my $args;
60 if (scalar(@_) > 1) {
61 if ( @_ % 2 ) {
62 $proto->_throw_error( "Odd number of parameters to " . (caller(1))[2] );
63 }
64 $args = {@_};
65 }
d0b74c17 66 elsif ( ref $_[0] ) {
4d35d856 67 unless ( eval { local $SIG{'__DIE__'}; %{$_[0]} || 1 } ) {
0ca7ea98 68 $proto->_throw_error( "Not a hashref in args to " . (caller(1))[2] );
69 }
70 $args = $_[0];
71 }
d0b74c17 72 else {
0ca7ea98 73 $args = { file => shift };
74 }
75
76 return $args;
77}
78
ffed8b01 79sub new {
d0b74c17 80 ##
81 # Class constructor method for Perl OO interface.
82 # Calls tie() and returns blessed reference to tied hash or array,
83 # providing a hybrid OO/tie interface.
84 ##
85 my $class = shift;
86 my $args = $class->_get_args( @_ );
87
88 ##
89 # Check if we want a tied hash or array.
90 ##
91 my $self;
92 if (defined($args->{type}) && $args->{type} eq TYPE_ARRAY) {
6fe26b29 93 $class = 'DBM::Deep::Array';
94 require DBM::Deep::Array;
d0b74c17 95 tie @$self, $class, %$args;
96 }
97 else {
6fe26b29 98 $class = 'DBM::Deep::Hash';
99 require DBM::Deep::Hash;
d0b74c17 100 tie %$self, $class, %$args;
101 }
ffed8b01 102
d0b74c17 103 return bless $self, $class;
ffed8b01 104}
105
96041a25 106# This initializer is called from the various TIE* methods. new() calls tie(),
107# which allows for a single point of entry.
0795f290 108sub _init {
0795f290 109 my $class = shift;
994ccd8e 110 my ($args) = @_;
0795f290 111
83371fe3 112 $args->{storage} = DBM::Deep::File->new( $args )
113 unless exists $args->{storage};
460b1067 114
115 # locking implicitly enables autoflush
116 if ($args->{locking}) { $args->{autoflush} = 1; }
117
0795f290 118 # These are the defaults to be optionally overridden below
119 my $self = bless {
95967a5e 120 type => TYPE_HASH,
e06824f8 121 base_offset => undef,
359a01ac 122
123 parent => undef,
124 parent_key => undef,
125
83371fe3 126 storage => undef,
c9f02899 127 engine => undef,
0795f290 128 }, $class;
c9f02899 129
130 $args->{engine} = DBM::Deep::Engine3->new( { %{$args}, obj => $self } )
131 unless exists $args->{engine};
8db25060 132
fde3db1a 133 # Grab the parameters we want to use
0795f290 134 foreach my $param ( keys %$self ) {
135 next unless exists $args->{$param};
3e9498a1 136 $self->{$param} = $args->{$param};
ffed8b01 137 }
d0b74c17 138
696cadb7 139 eval {
140 local $SIG{'__DIE__'};
3ed26433 141
696cadb7 142 $self->lock;
143 $self->_engine->setup_fh( $self );
3ed26433 144 $self->_storage->set_inode;
696cadb7 145 $self->unlock;
146 }; if ( $@ ) {
147 my $e = $@;
148 eval { local $SIG{'__DIE__'}; $self->unlock; };
149 die $e;
150 }
359a01ac 151
0795f290 152 return $self;
ffed8b01 153}
154
ffed8b01 155sub TIEHASH {
6fe26b29 156 shift;
157 require DBM::Deep::Hash;
158 return DBM::Deep::Hash->TIEHASH( @_ );
ffed8b01 159}
160
161sub TIEARRAY {
6fe26b29 162 shift;
163 require DBM::Deep::Array;
164 return DBM::Deep::Array->TIEARRAY( @_ );
ffed8b01 165}
166
ffed8b01 167sub lock {
994ccd8e 168 my $self = shift->_get_self;
83371fe3 169 return $self->_storage->lock( $self, @_ );
ffed8b01 170}
171
172sub unlock {
994ccd8e 173 my $self = shift->_get_self;
83371fe3 174 return $self->_storage->unlock( $self, @_ );
ffed8b01 175}
176
906c8e01 177sub _copy_value {
178 my $self = shift->_get_self;
179 my ($spot, $value) = @_;
180
181 if ( !ref $value ) {
182 ${$spot} = $value;
183 }
184 elsif ( eval { local $SIG{__DIE__}; $value->isa( 'DBM::Deep' ) } ) {
f9c33187 185 ${$spot} = $value->_repr;
906c8e01 186 $value->_copy_node( ${$spot} );
187 }
188 else {
189 my $r = Scalar::Util::reftype( $value );
190 my $c = Scalar::Util::blessed( $value );
191 if ( $r eq 'ARRAY' ) {
192 ${$spot} = [ @{$value} ];
193 }
194 else {
195 ${$spot} = { %{$value} };
196 }
95bbd935 197 ${$spot} = bless ${$spot}, $c
906c8e01 198 if defined $c;
199 }
200
201 return 1;
202}
203
261d1296 204sub _copy_node {
f9c33187 205 die "Must be implemented in a child class\n";
206}
906c8e01 207
f9c33187 208sub _repr {
209 die "Must be implemented in a child class\n";
ffed8b01 210}
211
212sub export {
d0b74c17 213 ##
214 # Recursively export into standard Perl hashes and arrays.
215 ##
994ccd8e 216 my $self = shift->_get_self;
d0b74c17 217
f9c33187 218 my $temp = $self->_repr;
d0b74c17 219
220 $self->lock();
221 $self->_copy_node( $temp );
222 $self->unlock();
223
c9f02899 224 my $classname = $self->_engine->get_classname( $self );
84467b9f 225 if ( defined $classname ) {
226 bless $temp, $classname;
68f943b3 227 }
228
d0b74c17 229 return $temp;
ffed8b01 230}
231
232sub import {
d0b74c17 233 ##
234 # Recursively import Perl hash/array structure
235 ##
d0b74c17 236 if (!ref($_[0])) { return; } # Perl calls import() on use -- ignore
237
994ccd8e 238 my $self = shift->_get_self;
239 my ($struct) = @_;
d0b74c17 240
c9cec40e 241 # struct is not a reference, so just import based on our type
d0b74c17 242 if (!ref($struct)) {
f9c33187 243 $struct = $self->_repr( @_ );
d0b74c17 244 }
245
12b96196 246 #XXX This isn't the best solution. Better would be to use Data::Walker,
247 #XXX but that's a lot more thinking than I want to do right now.
7a960a12 248 eval {
84467b9f 249 #$self->begin_work;
12b96196 250 $self->_import( _clone_data( $struct ) );
84467b9f 251 #$self->commit;
7a960a12 252 }; if ( $@ ) {
84467b9f 253 #$self->rollback;
7a960a12 254 die $@;
255 }
256
257 return 1;
ffed8b01 258}
259
13ff93d5 260#XXX Need to keep track of who has a fh to this file in order to
261#XXX close them all prior to optimize on Win32/cygwin
ffed8b01 262sub optimize {
d0b74c17 263 ##
264 # Rebuild entire database into new file, then move
265 # it back on top of original.
266 ##
994ccd8e 267 my $self = shift->_get_self;
cc4bef86 268
269#XXX Need to create a new test for this
83371fe3 270# if ($self->_storage->{links} > 1) {
1400a48e 271# $self->_throw_error("Cannot optimize: reference count is greater than 1");
d0b74c17 272# }
273
7a960a12 274 #XXX Do we have to lock the tempfile?
275
d0b74c17 276 my $db_temp = DBM::Deep->new(
83371fe3 277 file => $self->_storage->{file} . '.tmp',
d0b74c17 278 type => $self->_type
279 );
d0b74c17 280
281 $self->lock();
282 $self->_copy_node( $db_temp );
283 undef $db_temp;
284
285 ##
286 # Attempt to copy user, group and permissions over to new file
287 ##
288 my @stats = stat($self->_fh);
289 my $perms = $stats[2] & 07777;
290 my $uid = $stats[4];
291 my $gid = $stats[5];
83371fe3 292 chown( $uid, $gid, $self->_storage->{file} . '.tmp' );
293 chmod( $perms, $self->_storage->{file} . '.tmp' );
d0b74c17 294
ffed8b01 295 # q.v. perlport for more information on this variable
90f93b43 296 if ( $^O eq 'MSWin32' || $^O eq 'cygwin' ) {
d0b74c17 297 ##
298 # Potential race condition when optmizing on Win32 with locking.
299 # The Windows filesystem requires that the filehandle be closed
300 # before it is overwritten with rename(). This could be redone
301 # with a soft copy.
302 ##
303 $self->unlock();
83371fe3 304 $self->_storage->close;
d0b74c17 305 }
306
83371fe3 307 if (!rename $self->_storage->{file} . '.tmp', $self->_storage->{file}) {
308 unlink $self->_storage->{file} . '.tmp';
d0b74c17 309 $self->unlock();
1400a48e 310 $self->_throw_error("Optimize failed: Cannot copy temp file over original: $!");
d0b74c17 311 }
312
313 $self->unlock();
83371fe3 314 $self->_storage->close;
315 $self->_storage->open;
72e315ac 316 $self->_engine->setup_fh( $self );
70b55428 317
d0b74c17 318 return 1;
ffed8b01 319}
320
321sub clone {
d0b74c17 322 ##
323 # Make copy of object and return
324 ##
994ccd8e 325 my $self = shift->_get_self;
d0b74c17 326
327 return DBM::Deep->new(
c3aafc14 328 type => $self->_type,
d0b74c17 329 base_offset => $self->_base_offset,
83371fe3 330 storage => $self->_storage,
c9f02899 331 engine => $self->_engine,
c3aafc14 332 parent => $self->{parent},
333 parent_key => $self->{parent_key},
d0b74c17 334 );
ffed8b01 335}
336
337{
338 my %is_legal_filter = map {
339 $_ => ~~1,
340 } qw(
341 store_key store_value
342 fetch_key fetch_value
343 );
344
345 sub set_filter {
346 ##
347 # Setup filter function for storing or fetching the key or value
348 ##
994ccd8e 349 my $self = shift->_get_self;
350 my $type = lc shift;
351 my $func = shift;
d0b74c17 352
ffed8b01 353 if ( $is_legal_filter{$type} ) {
83371fe3 354 $self->_storage->{"filter_$type"} = $func;
ffed8b01 355 return 1;
356 }
357
358 return;
359 }
360}
361
fee0243f 362sub begin_work {
363 my $self = shift->_get_self;
8cb9205a 364 return $self->_engine->begin_work( $self, @_ );
fee0243f 365}
366
367sub rollback {
368 my $self = shift->_get_self;
8cb9205a 369 return $self->_engine->rollback( $self, @_ );
fee0243f 370}
371
359a01ac 372sub commit {
373 my $self = shift->_get_self;
8cb9205a 374 return $self->_engine->commit( $self, @_ );
359a01ac 375}
fee0243f 376
ffed8b01 377##
378# Accessor methods
379##
380
72e315ac 381sub _engine {
382 my $self = $_[0]->_get_self;
383 return $self->{engine};
384}
385
83371fe3 386sub _storage {
2ac02042 387 my $self = $_[0]->_get_self;
83371fe3 388 return $self->{storage};
ffed8b01 389}
390
4d35d856 391sub _type {
2ac02042 392 my $self = $_[0]->_get_self;
d0b74c17 393 return $self->{type};
ffed8b01 394}
395
4d35d856 396sub _base_offset {
2ac02042 397 my $self = $_[0]->_get_self;
d0b74c17 398 return $self->{base_offset};
ffed8b01 399}
400
994ccd8e 401sub _fh {
994ccd8e 402 my $self = $_[0]->_get_self;
83371fe3 403 return $self->_storage->{fh};
994ccd8e 404}
405
ffed8b01 406##
407# Utility methods
408##
409
261d1296 410sub _throw_error {
95967a5e 411 die "DBM::Deep: $_[1]\n";
ffed8b01 412}
413
359a01ac 414sub _find_parent {
415 my $self = shift;
cfd97a7f 416
417 my $base = '';
633df1fd 418 #XXX This if() is redundant
cfd97a7f 419 if ( my $parent = $self->{parent} ) {
420 my $child = $self;
25c7c8d6 421 while ( $parent->{parent} ) {
cfd97a7f 422 $base = (
423 $parent->_type eq TYPE_HASH
415dcbb7 424 ? "\{q{$child->{parent_key}}\}"
cfd97a7f 425 : "\[$child->{parent_key}\]"
426 ) . $base;
427
428 $child = $parent;
429 $parent = $parent->{parent};
25c7c8d6 430 }
1ad1fc2b 431
25c7c8d6 432 if ( $base ) {
415dcbb7 433 $base = "\$db->get( q{$child->{parent_key}} )->" . $base;
25c7c8d6 434 }
435 else {
415dcbb7 436 $base = "\$db->get( q{$child->{parent_key}} )";
359a01ac 437 }
359a01ac 438 }
25c7c8d6 439 return $base;
359a01ac 440}
441
ffed8b01 442sub STORE {
d0b74c17 443 ##
444 # Store single hash key/value or array element in database.
445 ##
446 my $self = shift->_get_self;
359a01ac 447 my ($key, $value, $orig_key) = @_;
c3aafc14 448 $orig_key = $key unless defined $orig_key;
81d3d316 449
a8fdabda 450 if ( !FileHandle::Fmode::is_W( $self->_fh ) ) {
acd4faf2 451 $self->_throw_error( 'Cannot write to a readonly filehandle' );
452 }
d0b74c17 453
504185fb 454 #XXX The second condition needs to disappear
c3aafc14 455 if ( !( $self->_type eq TYPE_ARRAY && $orig_key eq 'length') ) {
4768a580 456 my $rhs;
457
458 my $r = Scalar::Util::reftype( $value ) || '';
459 if ( $r eq 'HASH' ) {
460 $rhs = '{}';
461 }
462 elsif ( $r eq 'ARRAY' ) {
463 $rhs = '[]';
464 }
465 elsif ( defined $value ) {
466 $rhs = "'$value'";
467 }
468 else {
469 $rhs = "undef";
470 }
471
472 if ( my $c = Scalar::Util::blessed( $value ) ) {
473 $rhs = "bless $rhs, '$c'";
474 }
475
25c7c8d6 476 my $lhs = $self->_find_parent;
477 if ( $lhs ) {
478 if ( $self->_type eq TYPE_HASH ) {
415dcbb7 479 $lhs .= "->\{q{$orig_key}\}";
25c7c8d6 480 }
481 else {
482 $lhs .= "->\[$orig_key\]";
483 }
484
485 $lhs .= "=$rhs;";
486 }
487 else {
415dcbb7 488 $lhs = "\$db->put(q{$orig_key},$rhs);";
25c7c8d6 489 }
490
83371fe3 491 $self->_storage->audit($lhs);
4768a580 492 }
359a01ac 493
d0b74c17 494 ##
495 # Request exclusive lock for writing
496 ##
497 $self->lock( LOCK_EX );
498
0cb639bd 499 # User may be storing a complex value, in which case we do not want it run
500 # through the filtering system.
83371fe3 501 if ( !ref($value) && $self->_storage->{filter_store_value} ) {
502 $value = $self->_storage->{filter_store_value}->( $value );
d0b74c17 503 }
504
c9f02899 505 $self->_engine->write_value( $self, $key, $value, $orig_key );
d0b74c17 506
507 $self->unlock();
508
86867f3a 509 return 1;
ffed8b01 510}
511
512sub FETCH {
d0b74c17 513 ##
514 # Fetch single value or element given plain key or array index
515 ##
cb79ec85 516 my $self = shift->_get_self;
a97c8f67 517 my ($key, $orig_key) = @_;
0cb639bd 518 $orig_key = $key unless defined $orig_key;
ffed8b01 519
d0b74c17 520 ##
521 # Request shared lock for reading
522 ##
523 $self->lock( LOCK_SH );
524
c9f02899 525 my $result = $self->_engine->read_value( $self, $key, $orig_key );
d0b74c17 526
527 $self->unlock();
528
a86430bd 529 # Filters only apply to scalar values, so the ref check is making
530 # sure the fetched bucket is a scalar, not a child hash or array.
83371fe3 531 return ($result && !ref($result) && $self->_storage->{filter_fetch_value})
532 ? $self->_storage->{filter_fetch_value}->($result)
cb79ec85 533 : $result;
ffed8b01 534}
535
536sub DELETE {
d0b74c17 537 ##
538 # Delete single key/value pair or element given plain key or array index
539 ##
a97c8f67 540 my $self = shift->_get_self;
541 my ($key, $orig_key) = @_;
c3aafc14 542 $orig_key = $key unless defined $orig_key;
d0b74c17 543
a8fdabda 544 if ( !FileHandle::Fmode::is_W( $self->_fh ) ) {
a86430bd 545 $self->_throw_error( 'Cannot write to a readonly filehandle' );
546 }
d0b74c17 547
4768a580 548 if ( defined $orig_key ) {
549 my $lhs = $self->_find_parent;
25c7c8d6 550 if ( $lhs ) {
83371fe3 551 $self->_storage->audit( "delete $lhs;" );
a97c8f67 552 }
4768a580 553 else {
83371fe3 554 $self->_storage->audit( "\$db->delete('$orig_key');" );
4768a580 555 }
a97c8f67 556 }
557
d0b74c17 558 ##
559 # Request exclusive lock for writing
560 ##
561 $self->lock( LOCK_EX );
562
d0b74c17 563 ##
564 # Delete bucket
565 ##
c9f02899 566 my $value = $self->_engine->delete_key( $self, $key, $orig_key );
a86430bd 567
83371fe3 568 if (defined $value && !ref($value) && $self->_storage->{filter_fetch_value}) {
569 $value = $self->_storage->{filter_fetch_value}->($value);
3b6a5056 570 }
571
d0b74c17 572 $self->unlock();
573
574 return $value;
ffed8b01 575}
576
577sub EXISTS {
d0b74c17 578 ##
579 # Check if a single key or element exists given plain key or array index
580 ##
a97c8f67 581 my $self = shift->_get_self;
582 my ($key) = @_;
d0b74c17 583
d0b74c17 584 ##
585 # Request shared lock for reading
586 ##
587 $self->lock( LOCK_SH );
588
c9f02899 589 my $result = $self->_engine->key_exists( $self, $key );
d0b74c17 590
591 $self->unlock();
592
593 return $result;
ffed8b01 594}
595
596sub CLEAR {
d0b74c17 597 ##
598 # Clear all keys from hash, or all elements from array.
599 ##
a97c8f67 600 my $self = shift->_get_self;
ffed8b01 601
a8fdabda 602 if ( !FileHandle::Fmode::is_W( $self->_fh ) ) {
a86430bd 603 $self->_throw_error( 'Cannot write to a readonly filehandle' );
604 }
605
4768a580 606 {
a97c8f67 607 my $lhs = $self->_find_parent;
608
a97c8f67 609 if ( $self->_type eq TYPE_HASH ) {
e82621dd 610 $lhs = '%{' . $lhs . '}';
a97c8f67 611 }
612 else {
e82621dd 613 $lhs = '@{' . $lhs . '}';
a97c8f67 614 }
615
83371fe3 616 $self->_storage->audit( "$lhs = ();" );
a97c8f67 617 }
618
d0b74c17 619 ##
620 # Request exclusive lock for writing
621 ##
622 $self->lock( LOCK_EX );
623
f9a320bb 624 if ( $self->_type eq TYPE_HASH ) {
625 my $key = $self->first_key;
626 while ( $key ) {
83c43bb5 627 # Retrieve the key before deleting because we depend on next_key
f9a320bb 628 my $next_key = $self->next_key( $key );
c9f02899 629 $self->_engine->delete_key( $self, $key, $key );
f9a320bb 630 $key = $next_key;
631 }
632 }
633 else {
634 my $size = $self->FETCHSIZE;
c3aafc14 635 for my $key ( 0 .. $size - 1 ) {
c9f02899 636 $self->_engine->delete_key( $self, $key, $key );
f9a320bb 637 }
638 $self->STORESIZE( 0 );
639 }
f9c33187 640#XXX This needs updating to use _release_space
f9a320bb 641# $self->_engine->write_tag(
642# $self->_base_offset, $self->_type,
643# chr(0)x$self->_engine->{index_size},
644# );
d0b74c17 645
646 $self->unlock();
647
648 return 1;
ffed8b01 649}
650
ffed8b01 651##
652# Public method aliases
653##
7f441181 654sub put { (shift)->STORE( @_ ) }
655sub store { (shift)->STORE( @_ ) }
656sub get { (shift)->FETCH( @_ ) }
657sub fetch { (shift)->FETCH( @_ ) }
baa27ab6 658sub delete { (shift)->DELETE( @_ ) }
659sub exists { (shift)->EXISTS( @_ ) }
660sub clear { (shift)->CLEAR( @_ ) }
ffed8b01 661
6621;
ffed8b01 663__END__
664
665=head1 NAME
666
667DBM::Deep - A pure perl multi-level hash/array DBM
668
669=head1 SYNOPSIS
670
671 use DBM::Deep;
672 my $db = DBM::Deep->new( "foo.db" );
d0b74c17 673
eff6a245 674 $db->{key} = 'value';
ffed8b01 675 print $db->{key};
d0b74c17 676
eff6a245 677 $db->put('key' => 'value');
ffed8b01 678 print $db->get('key');
d0b74c17 679
ffed8b01 680 # true multi-level support
681 $db->{my_complex} = [
d0b74c17 682 'hello', { perl => 'rules' },
683 42, 99,
90f93b43 684 ];
ffed8b01 685
eff6a245 686 tie my %db, 'DBM::Deep', 'foo.db';
687 $db{key} = 'value';
688 print $db{key};
ffed8b01 689
eff6a245 690 tied(%db)->put('key' => 'value');
691 print tied(%db)->get('key');
8db25060 692
eff6a245 693=head1 DESCRIPTION
8db25060 694
eff6a245 695A unique flat-file database module, written in pure perl. True multi-level
696hash/array support (unlike MLDBM, which is faked), hybrid OO / tie()
697interface, cross-platform FTPable files, ACID transactions, and is quite fast.
698Can handle millions of keys and unlimited levels without significant
699slow-down. Written from the ground-up in pure perl -- this is NOT a wrapper
700around a C-based DBM. Out-of-the-box compatibility with Unix, Mac OS X and
701Windows.
ffed8b01 702
eff6a245 703=head1 VERSION DIFFERENCES
ffed8b01 704
eff6a245 705B<NOTE>: 0.99_01 and above have significant file format differences from 0.983 and
706before. There will be a backwards-compatibility layer in 1.00, but that is
707slated for a later 0.99_x release. This version is B<NOT> backwards compatible
708with 0.983 and before.
ffed8b01 709
710=head1 SETUP
711
d0b74c17 712Construction can be done OO-style (which is the recommended way), or using
ffed8b01 713Perl's tie() function. Both are examined here.
714
715=head2 OO CONSTRUCTION
716
717The recommended way to construct a DBM::Deep object is to use the new()
eff6a245 718method, which gets you a blessed I<and> tied hash (or array) reference.
ffed8b01 719
a8fdabda 720 my $db = DBM::Deep->new( "foo.db" );
ffed8b01 721
722This opens a new database handle, mapped to the file "foo.db". If this
d0b74c17 723file does not exist, it will automatically be created. DB files are
ffed8b01 724opened in "r+" (read/write) mode, and the type of object returned is a
725hash, unless otherwise specified (see L<OPTIONS> below).
726
ffed8b01 727You can pass a number of options to the constructor to specify things like
eff6a245 728locking, autoflush, etc. This is done by passing an inline hash (or hashref):
ffed8b01 729
a8fdabda 730 my $db = DBM::Deep->new(
731 file => "foo.db",
732 locking => 1,
733 autoflush => 1
734 );
ffed8b01 735
736Notice that the filename is now specified I<inside> the hash with
d0b74c17 737the "file" parameter, as opposed to being the sole argument to the
ffed8b01 738constructor. This is required if any options are specified.
739See L<OPTIONS> below for the complete list.
740
ffed8b01 741You can also start with an array instead of a hash. For this, you must
742specify the C<type> parameter:
743
a8fdabda 744 my $db = DBM::Deep->new(
745 file => "foo.db",
746 type => DBM::Deep->TYPE_ARRAY
747 );
ffed8b01 748
749B<Note:> Specifing the C<type> parameter only takes effect when beginning
750a new DB file. If you create a DBM::Deep object with an existing file, the
90f93b43 751C<type> will be loaded from the file header, and an error will be thrown if
752the wrong type is passed in.
ffed8b01 753
754=head2 TIE CONSTRUCTION
755
90f93b43 756Alternately, you can create a DBM::Deep handle by using Perl's built-in
757tie() function. The object returned from tie() can be used to call methods,
eff6a245 758such as lock() and unlock(). (That object can be retrieved from the tied
759variable at any time using tied() - please see L<perltie/> for more info.
ffed8b01 760
a8fdabda 761 my %hash;
762 my $db = tie %hash, "DBM::Deep", "foo.db";
d0b74c17 763
a8fdabda 764 my @array;
765 my $db = tie @array, "DBM::Deep", "bar.db";
ffed8b01 766
767As with the OO constructor, you can replace the DB filename parameter with
768a hash containing one or more options (see L<OPTIONS> just below for the
769complete list).
770
a8fdabda 771 tie %hash, "DBM::Deep", {
772 file => "foo.db",
773 locking => 1,
774 autoflush => 1
775 };
ffed8b01 776
777=head2 OPTIONS
778
779There are a number of options that can be passed in when constructing your
780DBM::Deep objects. These apply to both the OO- and tie- based approaches.
781
782=over
783
784=item * file
785
786Filename of the DB file to link the handle to. You can pass a full absolute
d0b74c17 787filesystem path, partial path, or a plain filename if the file is in the
714618f0 788current working directory. This is a required parameter (though q.v. fh).
789
790=item * fh
791
792If you want, you can pass in the fh instead of the file. This is most useful for doing
793something like:
794
795 my $db = DBM::Deep->new( { fh => \*DATA } );
796
797You are responsible for making sure that the fh has been opened appropriately for your
798needs. If you open it read-only and attempt to write, an exception will be thrown. If you
799open it write-only or append-only, an exception will be thrown immediately as DBM::Deep
800needs to read from the fh.
801
eff6a245 802=item * audit_file / audit_fh
803
804These are just like file/fh, except for auditing. Please see L</AUDITING> for
805more information.
806
714618f0 807=item * file_offset
808
809This is the offset within the file that the DBM::Deep db starts. Most of the time, you will
810not need to set this. However, it's there if you want it.
811
812If you pass in fh and do not set this, it will be set appropriately.
ffed8b01 813
ffed8b01 814=item * type
815
816This parameter specifies what type of object to create, a hash or array. Use
359a01ac 817one of these two constants:
818
819=over 4
820
821=item * C<DBM::Deep-E<gt>TYPE_HASH>
822
823=item * C<DBM::Deep-E<gt>TYPE_ARRAY>.
824
825=back
826
d0b74c17 827This only takes effect when beginning a new file. This is an optional
ffed8b01 828parameter, and defaults to C<DBM::Deep-E<gt>TYPE_HASH>.
829
830=item * locking
831
eff6a245 832Specifies whether locking is to be enabled. DBM::Deep uses Perl's flock()
833function to lock the database in exclusive mode for writes, and shared mode
834for reads. Pass any true value to enable. This affects the base DB handle
835I<and any child hashes or arrays> that use the same DB file. This is an
836optional parameter, and defaults to 0 (disabled). See L<LOCKING> below for
837more.
ffed8b01 838
839=item * autoflush
840
d0b74c17 841Specifies whether autoflush is to be enabled on the underlying filehandle.
842This obviously slows down write operations, but is required if you may have
843multiple processes accessing the same DB file (also consider enable I<locking>).
844Pass any true value to enable. This is an optional parameter, and defaults to 0
ffed8b01 845(disabled).
846
847=item * autobless
848
359a01ac 849If I<autobless> mode is enabled, DBM::Deep will preserve the class something
850is blessed into, and restores it when fetched. This is an optional parameter, and defaults to 1 (enabled).
851
852B<Note:> If you use the OO-interface, you will not be able to call any methods
853of DBM::Deep on the blessed item. This is considered to be a feature.
ffed8b01 854
855=item * filter_*
856
359a01ac 857See L</FILTERS> below.
ffed8b01 858
ffed8b01 859=back
860
861=head1 TIE INTERFACE
862
863With DBM::Deep you can access your databases using Perl's standard hash/array
90f93b43 864syntax. Because all DBM::Deep objects are I<tied> to hashes or arrays, you can
865treat them as such. DBM::Deep will intercept all reads/writes and direct them
866to the right place -- the DB file. This has nothing to do with the
867L<TIE CONSTRUCTION> section above. This simply tells you how to use DBM::Deep
868using regular hashes and arrays, rather than calling functions like C<get()>
869and C<put()> (although those work too). It is entirely up to you how to want
870to access your databases.
ffed8b01 871
872=head2 HASHES
873
874You can treat any DBM::Deep object like a normal Perl hash reference. Add keys,
875or even nested hashes (or arrays) using standard Perl syntax:
876
a8fdabda 877 my $db = DBM::Deep->new( "foo.db" );
d0b74c17 878
a8fdabda 879 $db->{mykey} = "myvalue";
880 $db->{myhash} = {};
881 $db->{myhash}->{subkey} = "subvalue";
ffed8b01 882
a8fdabda 883 print $db->{myhash}->{subkey} . "\n";
ffed8b01 884
885You can even step through hash keys using the normal Perl C<keys()> function:
886
a8fdabda 887 foreach my $key (keys %$db) {
888 print "$key: " . $db->{$key} . "\n";
889 }
ffed8b01 890
891Remember that Perl's C<keys()> function extracts I<every> key from the hash and
d0b74c17 892pushes them onto an array, all before the loop even begins. If you have an
eff6a245 893extremely large hash, this may exhaust Perl's memory. Instead, consider using
d0b74c17 894Perl's C<each()> function, which pulls keys/values one at a time, using very
ffed8b01 895little memory:
896
a8fdabda 897 while (my ($key, $value) = each %$db) {
898 print "$key: $value\n";
899 }
ffed8b01 900
901Please note that when using C<each()>, you should always pass a direct
902hash reference, not a lookup. Meaning, you should B<never> do this:
903
a8fdabda 904 # NEVER DO THIS
905 while (my ($key, $value) = each %{$db->{foo}}) { # BAD
ffed8b01 906
907This causes an infinite loop, because for each iteration, Perl is calling
908FETCH() on the $db handle, resulting in a "new" hash for foo every time, so
d0b74c17 909it effectively keeps returning the first key over and over again. Instead,
ffed8b01 910assign a temporary variable to C<$db->{foo}>, then pass that to each().
911
912=head2 ARRAYS
913
914As with hashes, you can treat any DBM::Deep object like a normal Perl array
d0b74c17 915reference. This includes inserting, removing and manipulating elements,
ffed8b01 916and the C<push()>, C<pop()>, C<shift()>, C<unshift()> and C<splice()> functions.
d0b74c17 917The object must have first been created using type C<DBM::Deep-E<gt>TYPE_ARRAY>,
ffed8b01 918or simply be a nested array reference inside a hash. Example:
919
a8fdabda 920 my $db = DBM::Deep->new(
921 file => "foo-array.db",
922 type => DBM::Deep->TYPE_ARRAY
923 );
d0b74c17 924
a8fdabda 925 $db->[0] = "foo";
926 push @$db, "bar", "baz";
927 unshift @$db, "bah";
d0b74c17 928
a8fdabda 929 my $last_elem = pop @$db; # baz
930 my $first_elem = shift @$db; # bah
931 my $second_elem = $db->[1]; # bar
d0b74c17 932
a8fdabda 933 my $num_elements = scalar @$db;
ffed8b01 934
935=head1 OO INTERFACE
936
937In addition to the I<tie()> interface, you can also use a standard OO interface
938to manipulate all aspects of DBM::Deep databases. Each type of object (hash or
d0b74c17 939array) has its own methods, but both types share the following common methods:
eff6a245 940C<put()>, C<get()>, C<exists()>, C<delete()> and C<clear()>. C<fetch()> and
941C<store(> are aliases to C<put()> and C<get()>, respectively.
ffed8b01 942
943=over
944
4d35d856 945=item * new() / clone()
946
947These are the constructor and copy-functions.
948
90f93b43 949=item * put() / store()
ffed8b01 950
951Stores a new hash key/value pair, or sets an array element value. Takes two
952arguments, the hash key or array index, and the new value. The value can be
953a scalar, hash ref or array ref. Returns true on success, false on failure.
954
a8fdabda 955 $db->put("foo", "bar"); # for hashes
956 $db->put(1, "bar"); # for arrays
ffed8b01 957
90f93b43 958=item * get() / fetch()
ffed8b01 959
960Fetches the value of a hash key or array element. Takes one argument: the hash
d0b74c17 961key or array index. Returns a scalar, hash ref or array ref, depending on the
ffed8b01 962data type stored.
963
a8fdabda 964 my $value = $db->get("foo"); # for hashes
965 my $value = $db->get(1); # for arrays
ffed8b01 966
967=item * exists()
968
d0b74c17 969Checks if a hash key or array index exists. Takes one argument: the hash key
ffed8b01 970or array index. Returns true if it exists, false if not.
971
a8fdabda 972 if ($db->exists("foo")) { print "yay!\n"; } # for hashes
973 if ($db->exists(1)) { print "yay!\n"; } # for arrays
ffed8b01 974
975=item * delete()
976
977Deletes one hash key/value pair or array element. Takes one argument: the hash
978key or array index. Returns true on success, false if not found. For arrays,
979the remaining elements located after the deleted element are NOT moved over.
980The deleted element is essentially just undefined, which is exactly how Perl's
d0b74c17 981internal arrays work. Please note that the space occupied by the deleted
982key/value or element is B<not> reused again -- see L<UNUSED SPACE RECOVERY>
ffed8b01 983below for details and workarounds.
984
a8fdabda 985 $db->delete("foo"); # for hashes
986 $db->delete(1); # for arrays
ffed8b01 987
988=item * clear()
989
d0b74c17 990Deletes B<all> hash keys or array elements. Takes no arguments. No return
991value. Please note that the space occupied by the deleted keys/values or
992elements is B<not> reused again -- see L<UNUSED SPACE RECOVERY> below for
ffed8b01 993details and workarounds.
994
a8fdabda 995 $db->clear(); # hashes or arrays
ffed8b01 996
4d35d856 997=item * lock() / unlock()
998
999q.v. Locking.
1000
1001=item * optimize()
1002
eff6a245 1003Recover lost disk space. This is important to do, especially if you use
1004transactions.
4d35d856 1005
1006=item * import() / export()
1007
1008Data going in and out.
1009
ffed8b01 1010=back
1011
1012=head2 HASHES
1013
d0b74c17 1014For hashes, DBM::Deep supports all the common methods described above, and the
ffed8b01 1015following additional methods: C<first_key()> and C<next_key()>.
1016
1017=over
1018
1019=item * first_key()
1020
d0b74c17 1021Returns the "first" key in the hash. As with built-in Perl hashes, keys are
1022fetched in an undefined order (which appears random). Takes no arguments,
ffed8b01 1023returns the key as a scalar value.
1024
a8fdabda 1025 my $key = $db->first_key();
ffed8b01 1026
1027=item * next_key()
1028
1029Returns the "next" key in the hash, given the previous one as the sole argument.
1030Returns undef if there are no more keys to be fetched.
1031
a8fdabda 1032 $key = $db->next_key($key);
ffed8b01 1033
1034=back
1035
1036Here are some examples of using hashes:
1037
a8fdabda 1038 my $db = DBM::Deep->new( "foo.db" );
d0b74c17 1039
a8fdabda 1040 $db->put("foo", "bar");
1041 print "foo: " . $db->get("foo") . "\n";
d0b74c17 1042
a8fdabda 1043 $db->put("baz", {}); # new child hash ref
1044 $db->get("baz")->put("buz", "biz");
1045 print "buz: " . $db->get("baz")->get("buz") . "\n";
d0b74c17 1046
a8fdabda 1047 my $key = $db->first_key();
1048 while ($key) {
1049 print "$key: " . $db->get($key) . "\n";
1050 $key = $db->next_key($key);
1051 }
d0b74c17 1052
a8fdabda 1053 if ($db->exists("foo")) { $db->delete("foo"); }
ffed8b01 1054
1055=head2 ARRAYS
1056
d0b74c17 1057For arrays, DBM::Deep supports all the common methods described above, and the
1058following additional methods: C<length()>, C<push()>, C<pop()>, C<shift()>,
ffed8b01 1059C<unshift()> and C<splice()>.
1060
1061=over
1062
1063=item * length()
1064
1065Returns the number of elements in the array. Takes no arguments.
1066
a8fdabda 1067 my $len = $db->length();
ffed8b01 1068
1069=item * push()
1070
d0b74c17 1071Adds one or more elements onto the end of the array. Accepts scalars, hash
ffed8b01 1072refs or array refs. No return value.
1073
a8fdabda 1074 $db->push("foo", "bar", {});
ffed8b01 1075
1076=item * pop()
1077
1078Fetches the last element in the array, and deletes it. Takes no arguments.
1079Returns undef if array is empty. Returns the element value.
1080
a8fdabda 1081 my $elem = $db->pop();
ffed8b01 1082
1083=item * shift()
1084
d0b74c17 1085Fetches the first element in the array, deletes it, then shifts all the
1086remaining elements over to take up the space. Returns the element value. This
1087method is not recommended with large arrays -- see L<LARGE ARRAYS> below for
ffed8b01 1088details.
1089
a8fdabda 1090 my $elem = $db->shift();
ffed8b01 1091
1092=item * unshift()
1093
d0b74c17 1094Inserts one or more elements onto the beginning of the array, shifting all
1095existing elements over to make room. Accepts scalars, hash refs or array refs.
1096No return value. This method is not recommended with large arrays -- see
ffed8b01 1097<LARGE ARRAYS> below for details.
1098
a8fdabda 1099 $db->unshift("foo", "bar", {});
ffed8b01 1100
1101=item * splice()
1102
d0b74c17 1103Performs exactly like Perl's built-in function of the same name. See L<perldoc
ffed8b01 1104-f splice> for usage -- it is too complicated to document here. This method is
1105not recommended with large arrays -- see L<LARGE ARRAYS> below for details.
1106
1107=back
1108
1109Here are some examples of using arrays:
1110
a8fdabda 1111 my $db = DBM::Deep->new(
1112 file => "foo.db",
1113 type => DBM::Deep->TYPE_ARRAY
1114 );
d0b74c17 1115
a8fdabda 1116 $db->push("bar", "baz");
1117 $db->unshift("foo");
1118 $db->put(3, "buz");
d0b74c17 1119
a8fdabda 1120 my $len = $db->length();
1121 print "length: $len\n"; # 4
d0b74c17 1122
a8fdabda 1123 for (my $k=0; $k<$len; $k++) {
1124 print "$k: " . $db->get($k) . "\n";
1125 }
d0b74c17 1126
a8fdabda 1127 $db->splice(1, 2, "biz", "baf");
d0b74c17 1128
a8fdabda 1129 while (my $elem = shift @$db) {
1130 print "shifted: $elem\n";
1131 }
ffed8b01 1132
1133=head1 LOCKING
1134
d0b74c17 1135Enable automatic file locking by passing a true value to the C<locking>
ffed8b01 1136parameter when constructing your DBM::Deep object (see L<SETUP> above).
1137
a8fdabda 1138 my $db = DBM::Deep->new(
1139 file => "foo.db",
1140 locking => 1
1141 );
ffed8b01 1142
d0b74c17 1143This causes DBM::Deep to C<flock()> the underlying filehandle with exclusive
1144mode for writes, and shared mode for reads. This is required if you have
1145multiple processes accessing the same database file, to avoid file corruption.
1146Please note that C<flock()> does NOT work for files over NFS. See L<DB OVER
ffed8b01 1147NFS> below for more.
1148
1149=head2 EXPLICIT LOCKING
1150
d0b74c17 1151You can explicitly lock a database, so it remains locked for multiple
1152transactions. This is done by calling the C<lock()> method, and passing an
90f93b43 1153optional lock mode argument (defaults to exclusive mode). This is particularly
d0b74c17 1154useful for things like counters, where the current value needs to be fetched,
ffed8b01 1155then incremented, then stored again.
1156
a8fdabda 1157 $db->lock();
1158 my $counter = $db->get("counter");
1159 $counter++;
1160 $db->put("counter", $counter);
1161 $db->unlock();
d0b74c17 1162
a8fdabda 1163 # or...
ffed8b01 1164
a8fdabda 1165 $db->lock();
1166 $db->{counter}++;
1167 $db->unlock();
ffed8b01 1168
1169You can pass C<lock()> an optional argument, which specifies which mode to use
68f943b3 1170(exclusive or shared). Use one of these two constants:
1171C<DBM::Deep-E<gt>LOCK_EX> or C<DBM::Deep-E<gt>LOCK_SH>. These are passed
1172directly to C<flock()>, and are the same as the constants defined in Perl's
1173L<Fcntl/> module.
ffed8b01 1174
a8fdabda 1175 $db->lock( $db->LOCK_SH );
1176 # something here
1177 $db->unlock();
ffed8b01 1178
ffed8b01 1179=head1 IMPORTING/EXPORTING
1180
1181You can import existing complex structures by calling the C<import()> method,
1182and export an entire database into an in-memory structure using the C<export()>
1183method. Both are examined here.
1184
1185=head2 IMPORTING
1186
1187Say you have an existing hash with nested hashes/arrays inside it. Instead of
d0b74c17 1188walking the structure and adding keys/elements to the database as you go,
1189simply pass a reference to the C<import()> method. This recursively adds
ffed8b01 1190everything to an existing DBM::Deep object for you. Here is an example:
1191
a8fdabda 1192 my $struct = {
1193 key1 => "value1",
1194 key2 => "value2",
1195 array1 => [ "elem0", "elem1", "elem2" ],
1196 hash1 => {
1197 subkey1 => "subvalue1",
1198 subkey2 => "subvalue2"
1199 }
1200 };
d0b74c17 1201
a8fdabda 1202 my $db = DBM::Deep->new( "foo.db" );
1203 $db->import( $struct );
d0b74c17 1204
a8fdabda 1205 print $db->{key1} . "\n"; # prints "value1"
d0b74c17 1206
1207This recursively imports the entire C<$struct> object into C<$db>, including
ffed8b01 1208all nested hashes and arrays. If the DBM::Deep object contains exsiting data,
d0b74c17 1209keys are merged with the existing ones, replacing if they already exist.
1210The C<import()> method can be called on any database level (not just the base
ffed8b01 1211level), and works with both hash and array DB types.
1212
ffed8b01 1213B<Note:> Make sure your existing structure has no circular references in it.
eff6a245 1214These will cause an infinite loop when importing. There are plans to fix this
1215in a later release.
ffed8b01 1216
1217=head2 EXPORTING
1218
d0b74c17 1219Calling the C<export()> method on an existing DBM::Deep object will return
1220a reference to a new in-memory copy of the database. The export is done
ffed8b01 1221recursively, so all nested hashes/arrays are all exported to standard Perl
1222objects. Here is an example:
1223
a8fdabda 1224 my $db = DBM::Deep->new( "foo.db" );
d0b74c17 1225
a8fdabda 1226 $db->{key1} = "value1";
1227 $db->{key2} = "value2";
1228 $db->{hash1} = {};
1229 $db->{hash1}->{subkey1} = "subvalue1";
1230 $db->{hash1}->{subkey2} = "subvalue2";
d0b74c17 1231
a8fdabda 1232 my $struct = $db->export();
d0b74c17 1233
a8fdabda 1234 print $struct->{key1} . "\n"; # prints "value1"
ffed8b01 1235
1236This makes a complete copy of the database in memory, and returns a reference
d0b74c17 1237to it. The C<export()> method can be called on any database level (not just
1238the base level), and works with both hash and array DB types. Be careful of
1239large databases -- you can store a lot more data in a DBM::Deep object than an
ffed8b01 1240in-memory Perl structure.
1241
ffed8b01 1242B<Note:> Make sure your database has no circular references in it.
eff6a245 1243These will cause an infinite loop when exporting. There are plans to fix this
1244in a later release.
ffed8b01 1245
1246=head1 FILTERS
1247
1248DBM::Deep has a number of hooks where you can specify your own Perl function
1249to perform filtering on incoming or outgoing data. This is a perfect
1250way to extend the engine, and implement things like real-time compression or
d0b74c17 1251encryption. Filtering applies to the base DB level, and all child hashes /
1252arrays. Filter hooks can be specified when your DBM::Deep object is first
1253constructed, or by calling the C<set_filter()> method at any time. There are
ffed8b01 1254four available filter hooks, described below:
1255
1256=over
1257
1258=item * filter_store_key
1259
d0b74c17 1260This filter is called whenever a hash key is stored. It
ffed8b01 1261is passed the incoming key, and expected to return a transformed key.
1262
1263=item * filter_store_value
1264
d0b74c17 1265This filter is called whenever a hash key or array element is stored. It
ffed8b01 1266is passed the incoming value, and expected to return a transformed value.
1267
1268=item * filter_fetch_key
1269
d0b74c17 1270This filter is called whenever a hash key is fetched (i.e. via
ffed8b01 1271C<first_key()> or C<next_key()>). It is passed the transformed key,
1272and expected to return the plain key.
1273
1274=item * filter_fetch_value
1275
d0b74c17 1276This filter is called whenever a hash key or array element is fetched.
ffed8b01 1277It is passed the transformed value, and expected to return the plain value.
1278
1279=back
1280
1281Here are the two ways to setup a filter hook:
1282
a8fdabda 1283 my $db = DBM::Deep->new(
1284 file => "foo.db",
1285 filter_store_value => \&my_filter_store,
1286 filter_fetch_value => \&my_filter_fetch
1287 );
d0b74c17 1288
a8fdabda 1289 # or...
d0b74c17 1290
a8fdabda 1291 $db->set_filter( "filter_store_value", \&my_filter_store );
1292 $db->set_filter( "filter_fetch_value", \&my_filter_fetch );
ffed8b01 1293
1294Your filter function will be called only when dealing with SCALAR keys or
1295values. When nested hashes and arrays are being stored/fetched, filtering
d0b74c17 1296is bypassed. Filters are called as static functions, passed a single SCALAR
ffed8b01 1297argument, and expected to return a single SCALAR value. If you want to
1298remove a filter, set the function reference to C<undef>:
1299
a8fdabda 1300 $db->set_filter( "filter_store_value", undef );
ffed8b01 1301
1302=head2 REAL-TIME ENCRYPTION EXAMPLE
1303
d0b74c17 1304Here is a working example that uses the I<Crypt::Blowfish> module to
ffed8b01 1305do real-time encryption / decryption of keys & values with DBM::Deep Filters.
d0b74c17 1306Please visit L<http://search.cpan.org/search?module=Crypt::Blowfish> for more
ffed8b01 1307on I<Crypt::Blowfish>. You'll also need the I<Crypt::CBC> module.
1308
a8fdabda 1309 use DBM::Deep;
1310 use Crypt::Blowfish;
1311 use Crypt::CBC;
1312
1313 my $cipher = Crypt::CBC->new({
1314 'key' => 'my secret key',
1315 'cipher' => 'Blowfish',
1316 'iv' => '$KJh#(}q',
1317 'regenerate_key' => 0,
1318 'padding' => 'space',
1319 'prepend_iv' => 0
1320 });
1321
1322 my $db = DBM::Deep->new(
1323 file => "foo-encrypt.db",
1324 filter_store_key => \&my_encrypt,
1325 filter_store_value => \&my_encrypt,
1326 filter_fetch_key => \&my_decrypt,
1327 filter_fetch_value => \&my_decrypt,
1328 );
1329
1330 $db->{key1} = "value1";
1331 $db->{key2} = "value2";
1332 print "key1: " . $db->{key1} . "\n";
1333 print "key2: " . $db->{key2} . "\n";
1334
1335 undef $db;
1336 exit;
1337
1338 sub my_encrypt {
1339 return $cipher->encrypt( $_[0] );
1340 }
1341 sub my_decrypt {
1342 return $cipher->decrypt( $_[0] );
1343 }
ffed8b01 1344
1345=head2 REAL-TIME COMPRESSION EXAMPLE
1346
1347Here is a working example that uses the I<Compress::Zlib> module to do real-time
1348compression / decompression of keys & values with DBM::Deep Filters.
d0b74c17 1349Please visit L<http://search.cpan.org/search?module=Compress::Zlib> for
ffed8b01 1350more on I<Compress::Zlib>.
1351
a8fdabda 1352 use DBM::Deep;
1353 use Compress::Zlib;
1354
1355 my $db = DBM::Deep->new(
1356 file => "foo-compress.db",
1357 filter_store_key => \&my_compress,
1358 filter_store_value => \&my_compress,
1359 filter_fetch_key => \&my_decompress,
1360 filter_fetch_value => \&my_decompress,
1361 );
1362
1363 $db->{key1} = "value1";
1364 $db->{key2} = "value2";
1365 print "key1: " . $db->{key1} . "\n";
1366 print "key2: " . $db->{key2} . "\n";
1367
1368 undef $db;
1369 exit;
1370
1371 sub my_compress {
1372 return Compress::Zlib::memGzip( $_[0] ) ;
1373 }
1374 sub my_decompress {
1375 return Compress::Zlib::memGunzip( $_[0] ) ;
1376 }
ffed8b01 1377
1378B<Note:> Filtering of keys only applies to hashes. Array "keys" are
1379actually numerical index numbers, and are not filtered.
1380
1381=head1 ERROR HANDLING
1382
1383Most DBM::Deep methods return a true value for success, and call die() on
95967a5e 1384failure. You can wrap calls in an eval block to catch the die.
ffed8b01 1385
a8fdabda 1386 my $db = DBM::Deep->new( "foo.db" ); # create hash
1387 eval { $db->push("foo"); }; # ILLEGAL -- push is array-only call
d0b74c17 1388
a8fdabda 1389 print $@; # prints error message
429e4192 1390
ffed8b01 1391=head1 LARGEFILE SUPPORT
1392
1393If you have a 64-bit system, and your Perl is compiled with both LARGEFILE
1394and 64-bit support, you I<may> be able to create databases larger than 2 GB.
1395DBM::Deep by default uses 32-bit file offset tags, but these can be changed
044e6288 1396by specifying the 'pack_size' parameter when constructing the file.
ffed8b01 1397
a8fdabda 1398 DBM::Deep->new(
1399 filename => $filename,
1400 pack_size => 'large',
1401 );
ffed8b01 1402
d0b74c17 1403This tells DBM::Deep to pack all file offsets with 8-byte (64-bit) quad words
1404instead of 32-bit longs. After setting these values your DB files have a
ffed8b01 1405theoretical maximum size of 16 XB (exabytes).
1406
044e6288 1407You can also use C<pack_size =E<gt> 'small'> in order to use 16-bit file
1408offsets.
1409
ffed8b01 1410B<Note:> Changing these values will B<NOT> work for existing database files.
044e6288 1411Only change this for new files. Once the value has been set, it is stored in
1412the file's header and cannot be changed for the life of the file. These
1413parameters are per-file, meaning you can access 32-bit and 64-bit files, as
1414you chose.
ffed8b01 1415
044e6288 1416B<Note:> We have not personally tested files larger than 2 GB -- all my
1417systems have only a 32-bit Perl. However, I have received user reports that
1418this does indeed work!
ffed8b01 1419
1420=head1 LOW-LEVEL ACCESS
1421
90f93b43 1422If you require low-level access to the underlying filehandle that DBM::Deep uses,
4d35d856 1423you can call the C<_fh()> method, which returns the handle:
ffed8b01 1424
a8fdabda 1425 my $fh = $db->_fh();
ffed8b01 1426
1427This method can be called on the root level of the datbase, or any child
1428hashes or arrays. All levels share a I<root> structure, which contains things
90f93b43 1429like the filehandle, a reference counter, and all the options specified
460b1067 1430when you created the object. You can get access to this file object by
83371fe3 1431calling the C<_storage()> method.
ffed8b01 1432
83371fe3 1433 my $file_obj = $db->_storage();
ffed8b01 1434
1435This is useful for changing options after the object has already been created,
f5be9b03 1436such as enabling/disabling locking. You can also store your own temporary user
1437data in this structure (be wary of name collision), which is then accessible from
1438any child hash or array.
ffed8b01 1439
1440=head1 CUSTOM DIGEST ALGORITHM
1441
1442DBM::Deep by default uses the I<Message Digest 5> (MD5) algorithm for hashing
1443keys. However you can override this, and use another algorithm (such as SHA-256)
d0b74c17 1444or even write your own. But please note that DBM::Deep currently expects zero
044e6288 1445collisions, so your algorithm has to be I<perfect>, so to speak. Collision
1446detection may be introduced in a later version.
ffed8b01 1447
044e6288 1448You can specify a custom digest algorithm by passing it into the parameter
1449list for new(), passing a reference to a subroutine as the 'digest' parameter,
1450and the length of the algorithm's hashes (in bytes) as the 'hash_size'
1451parameter. Here is a working example that uses a 256-bit hash from the
d0b74c17 1452I<Digest::SHA256> module. Please see
044e6288 1453L<http://search.cpan.org/search?module=Digest::SHA256> for more information.
ffed8b01 1454
a8fdabda 1455 use DBM::Deep;
1456 use Digest::SHA256;
d0b74c17 1457
a8fdabda 1458 my $context = Digest::SHA256::new(256);
d0b74c17 1459
a8fdabda 1460 my $db = DBM::Deep->new(
1461 filename => "foo-sha.db",
1462 digest => \&my_digest,
1463 hash_size => 32,
1464 );
d0b74c17 1465
a8fdabda 1466 $db->{key1} = "value1";
1467 $db->{key2} = "value2";
1468 print "key1: " . $db->{key1} . "\n";
1469 print "key2: " . $db->{key2} . "\n";
d0b74c17 1470
a8fdabda 1471 undef $db;
1472 exit;
d0b74c17 1473
a8fdabda 1474 sub my_digest {
1475 return substr( $context->hash($_[0]), 0, 32 );
1476 }
ffed8b01 1477
1478B<Note:> Your returned digest strings must be B<EXACTLY> the number
044e6288 1479of bytes you specify in the hash_size parameter (in this case 32).
ffed8b01 1480
260a80b4 1481B<Note:> If you do choose to use a custom digest algorithm, you must set it
1482every time you access this file. Otherwise, the default (MD5) will be used.
1483
ffed8b01 1484=head1 CIRCULAR REFERENCES
1485
1486DBM::Deep has B<experimental> support for circular references. Meaning you
1487can have a nested hash key or array element that points to a parent object.
1488This relationship is stored in the DB file, and is preserved between sessions.
1489Here is an example:
1490
a8fdabda 1491 my $db = DBM::Deep->new( "foo.db" );
d0b74c17 1492
a8fdabda 1493 $db->{foo} = "bar";
1494 $db->{circle} = $db; # ref to self
d0b74c17 1495
a8fdabda 1496 print $db->{foo} . "\n"; # prints "bar"
1497 print $db->{circle}->{foo} . "\n"; # prints "bar" again
ffed8b01 1498
69c94980 1499B<Note>: Passing the object to a function that recursively walks the
ffed8b01 1500object tree (such as I<Data::Dumper> or even the built-in C<optimize()> or
69c94980 1501C<export()> methods) will result in an infinite loop. This will be fixed in
1502a future release.
ffed8b01 1503
eff6a245 1504=head1 AUDITING
1505
1506New in 0.99_01 is the ability to audit your databases actions. By passing in
1507audit_file (or audit_fh) to the constructor, all actions will be logged to
1508that file. The format is one that is suitable for eval'ing against the
1509database to replay the actions. Please see t/33_audit_trail.t for an example
1510of how to do this.
1511
1512=head1 TRANSACTIONS
1513
1514New in 0.99_01 is ACID transactions. Every DBM::Deep object is completely
1515transaction-ready - it is not an option you have to turn on. Three new methods
1516have been added to support them. They are:
1517
1518=over 4
1519
1520=item * begin_work()
1521
1522This starts a transaction.
1523
1524=item * commit()
1525
1526This applies the changes done within the transaction to the mainline and ends
1527the transaction.
1528
1529=item * rollback()
1530
1531This discards the changes done within the transaction to the mainline and ends
1532the transaction.
1533
1534=back
1535
1536Transactions in DBM::Deep are done using the MVCC method, the same method used
1537by the InnoDB MySQL table type.
1538
ffed8b01 1539=head1 CAVEATS / ISSUES / BUGS
1540
1541This section describes all the known issues with DBM::Deep. It you have found
1542something that is not listed here, please send e-mail to L<jhuckaby@cpan.org>.
1543
1544=head2 UNUSED SPACE RECOVERY
1545
14a3acb6 1546One major caveat with DBM::Deep is that space occupied by existing keys and
ffed8b01 1547values is not recovered when they are deleted. Meaning if you keep deleting
1548and adding new keys, your file will continuously grow. I am working on this,
d0b74c17 1549but in the meantime you can call the built-in C<optimize()> method from time to
ffed8b01 1550time (perhaps in a crontab or something) to recover all your unused space.
1551
a8fdabda 1552 $db->optimize(); # returns true on success
ffed8b01 1553
1554This rebuilds the ENTIRE database into a new file, then moves it on top of
1555the original. The new file will have no unused space, thus it will take up as
d0b74c17 1556little disk space as possible. Please note that this operation can take
1557a long time for large files, and you need enough disk space to temporarily hold
15582 copies of your DB file. The temporary file is created in the same directory
1559as the original, named with a ".tmp" extension, and is deleted when the
1560operation completes. Oh, and if locking is enabled, the DB is automatically
ffed8b01 1561locked for the entire duration of the copy.
1562
d0b74c17 1563B<WARNING:> Only call optimize() on the top-level node of the database, and
1564make sure there are no child references lying around. DBM::Deep keeps a reference
ffed8b01 1565counter, and if it is greater than 1, optimize() will abort and return undef.
1566
eea0d863 1567=head2 REFERENCES
1568
1569(The reasons given assume a high level of Perl understanding, specifically of
1570references. You can safely skip this section.)
1571
1572Currently, the only references supported are HASH and ARRAY. The other reference
1573types (SCALAR, CODE, GLOB, and REF) cannot be supported for various reasons.
1574
1575=over 4
1576
1577=item * GLOB
1578
1579These are things like filehandles and other sockets. They can't be supported
1580because it's completely unclear how DBM::Deep should serialize them.
1581
1582=item * SCALAR / REF
1583
1584The discussion here refers to the following type of example:
1585
1586 my $x = 25;
1587 $db->{key1} = \$x;
1588
1589 $x = 50;
1590
1591 # In some other process ...
1592
1593 my $val = ${ $db->{key1} };
1594
1595 is( $val, 50, "What actually gets stored in the DB file?" );
1596
1597The problem is one of synchronization. When the variable being referred to
1598changes value, the reference isn't notified. This means that the new value won't
1599be stored in the datafile for other processes to read. There is no TIEREF.
1600
1601It is theoretically possible to store references to values already within a
1602DBM::Deep object because everything already is synchronized, but the change to
1603the internals would be quite large. Specifically, DBM::Deep would have to tie
1604every single value that is stored. This would bloat the RAM footprint of
1605DBM::Deep at least twofold (if not more) and be a significant performance drain,
1606all to support a feature that has never been requested.
1607
1608=item * CODE
1609
1990c72d 1610L<Data::Dump::Streamer/> provides a mechanism for serializing coderefs,
1611including saving off all closure state. However, just as for SCALAR and REF,
1612that closure state may change without notifying the DBM::Deep object storing
1613the reference.
eea0d863 1614
1615=back
1616
ffed8b01 1617=head2 FILE CORRUPTION
1618
14a3acb6 1619The current level of error handling in DBM::Deep is minimal. Files I<are> checked
1620for a 32-bit signature when opened, but other corruption in files can cause
1621segmentation faults. DBM::Deep may try to seek() past the end of a file, or get
ffed8b01 1622stuck in an infinite loop depending on the level of corruption. File write
1623operations are not checked for failure (for speed), so if you happen to run
d0b74c17 1624out of disk space, DBM::Deep will probably fail in a bad way. These things will
ffed8b01 1625be addressed in a later version of DBM::Deep.
1626
1627=head2 DB OVER NFS
1628
d8db2929 1629Beware of using DBM::Deep files over NFS. DBM::Deep uses flock(), which works
1630well on local filesystems, but will NOT protect you from file corruption over
1631NFS. I've heard about setting up your NFS server with a locking daemon, then
1632using lockf() to lock your files, but your mileage may vary there as well.
1633From what I understand, there is no real way to do it. However, if you need
1634access to the underlying filehandle in DBM::Deep for using some other kind of
1635locking scheme like lockf(), see the L<LOW-LEVEL ACCESS> section above.
ffed8b01 1636
1637=head2 COPYING OBJECTS
1638
d0b74c17 1639Beware of copying tied objects in Perl. Very strange things can happen.
1640Instead, use DBM::Deep's C<clone()> method which safely copies the object and
ffed8b01 1641returns a new, blessed, tied hash or array to the same level in the DB.
1642
a8fdabda 1643 my $copy = $db->clone();
ffed8b01 1644
90f93b43 1645B<Note>: Since clone() here is cloning the object, not the database location, any
d8db2929 1646modifications to either $db or $copy will be visible to both.
90f93b43 1647
ffed8b01 1648=head2 LARGE ARRAYS
1649
1650Beware of using C<shift()>, C<unshift()> or C<splice()> with large arrays.
1651These functions cause every element in the array to move, which can be murder
1652on DBM::Deep, as every element has to be fetched from disk, then stored again in
90f93b43 1653a different location. This will be addressed in the forthcoming version 1.00.
ffed8b01 1654
9be51a89 1655=head2 WRITEONLY FILES
1656
1657If you pass in a filehandle to new(), you may have opened it in either a readonly or
1658writeonly mode. STORE will verify that the filehandle is writable. However, there
1659doesn't seem to be a good way to determine if a filehandle is readable. And, if the
1660filehandle isn't readable, it's not clear what will happen. So, don't do that.
1661
261d1296 1662=head1 CODE COVERAGE
1663
eff6a245 1664B<Devel::Cover> is used to test the code coverage of the tests. Below is the
1665B<Devel::Cover> report on this distribution's test suite.
7910cf68 1666
eff6a245 1667 ---------------------------- ------ ------ ------ ------ ------ ------ ------
1668 File stmt bran cond sub pod time total
1669 ---------------------------- ------ ------ ------ ------ ------ ------ ------
1670 blib/lib/DBM/Deep.pm 96.2 89.0 75.0 95.8 89.5 36.0 92.9
1671 blib/lib/DBM/Deep/Array.pm 96.1 88.3 100.0 96.4 100.0 15.9 94.7
1672 blib/lib/DBM/Deep/Engine.pm 96.6 86.6 89.5 100.0 0.0 20.0 91.0
1673 blib/lib/DBM/Deep/File.pm 99.4 88.3 55.6 100.0 0.0 19.6 89.5
1674 blib/lib/DBM/Deep/Hash.pm 98.5 83.3 100.0 100.0 100.0 8.5 96.3
1675 Total 96.9 87.4 81.2 98.0 38.5 100.0 92.1
1676 ---------------------------- ------ ------ ------ ------ ------ ------ ------
37c5bcf0 1677
1678=head1 MORE INFORMATION
1679
1680Check out the DBM::Deep Google Group at L<http://groups.google.com/group/DBM-Deep>
eff6a245 1681or send email to L<DBM-Deep@googlegroups.com>. You can also visit #dbm-deep on
1682irc.perl.org
ffed8b01 1683
d8db2929 1684The source code repository is at L<http://svn.perl.org/modules/DBM-Deep>
1685
eff6a245 1686=head1 MAINTAINERS
37c5bcf0 1687
aeeb5497 1688Rob Kinyon, L<rkinyon@cpan.org>
ffed8b01 1689
eff6a245 1690Originally written by Joseph Huckaby, L<jhuckaby@cpan.org>
1691
ffed8b01 1692Special thanks to Adam Sah and Rich Gaushell! You know why :-)
1693
1694=head1 SEE ALSO
1695
1696perltie(1), Tie::Hash(3), Digest::MD5(3), Fcntl(3), flock(2), lockf(3), nfs(5),
1697Digest::SHA256(3), Crypt::Blowfish(3), Compress::Zlib(3)
1698
1699=head1 LICENSE
1700
aeeb5497 1701Copyright (c) 2002-2006 Joseph Huckaby. All Rights Reserved.
ffed8b01 1702This is free software, you may use it and distribute it under the
1703same terms as Perl itself.
1704
1705=cut