test of svn on new machine
[dbsrgits/DBM-Deep.git] / lib / DBM / Deep / Engine.pm
CommitLineData
a20d9a3f 1package DBM::Deep::Engine;
2
3use strict;
4
5use Fcntl qw( :DEFAULT :flock :seek );
6
8db25060 7##
8# Setup file and tag signatures. These should never change.
9##
10sub SIG_FILE () { 'DPDB' }
11sub SIG_INTERNAL () { 'i' }
12sub SIG_HASH () { 'H' }
13sub SIG_ARRAY () { 'A' }
8db25060 14sub SIG_NULL () { 'N' }
15sub SIG_DATA () { 'D' }
16sub SIG_INDEX () { 'I' }
17sub SIG_BLIST () { 'B' }
7b1e1aa1 18sub SIG_FREE () { 'F' }
8db25060 19sub SIG_SIZE () { 1 }
20
612969fb 21sub precalc_sizes {
beac1dff 22 ##
23 # Precalculate index, bucket and bucket list sizes
24 ##
251dfd0e 25 my $self = shift;
1bf65be7 26
251dfd0e 27 $self->{index_size} = (2**8) * $self->{long_size};
6ed2f3df 28 $self->{bucket_size} = $self->{hash_size} + $self->{long_size} * 2;
251dfd0e 29 $self->{bucket_list_size} = $self->{max_buckets} * $self->{bucket_size};
1bf65be7 30
251dfd0e 31 return 1;
1bf65be7 32}
33
34sub set_pack {
beac1dff 35 ##
36 # Set pack/unpack modes (see file header for more)
37 ##
251dfd0e 38 my $self = shift;
1bf65be7 39 my ($long_s, $long_p, $data_s, $data_p) = @_;
40
81d16922 41 ##
75be6413 42 # Set to 4 and 'N' for 32-bit offset tags (default). Theoretical limit of 4
43 # GB per file.
beac1dff 44 # (Perl must be compiled with largefile support for files > 2 GB)
81d16922 45 #
46 # Set to 8 and 'Q' for 64-bit offsets. Theoretical limit of 16 XB per file.
beac1dff 47 # (Perl must be compiled with largefile and 64-bit long support)
81d16922 48 ##
251dfd0e 49 $self->{long_size} = $long_s ? $long_s : 4;
50 $self->{long_pack} = $long_p ? $long_p : 'N';
1bf65be7 51
81d16922 52 ##
75be6413 53 # Set to 4 and 'N' for 32-bit data length prefixes. Limit of 4 GB for each
d5d7c51d 54 # key/value. Upgrading this is possible (see above) but probably not
55 # necessary. If you need more than 4 GB for a single key or value, this
56 # module is really not for you :-)
81d16922 57 ##
251dfd0e 58 $self->{data_size} = $data_s ? $data_s : 4;
59 $self->{data_pack} = $data_p ? $data_p : 'N';
1bf65be7 60
beac1dff 61 return $self->precalc_sizes();
1bf65be7 62}
63
64sub set_digest {
beac1dff 65 ##
66 # Set key digest function (default is MD5)
67 ##
251dfd0e 68 my $self = shift;
1bf65be7 69 my ($digest_func, $hash_size) = @_;
70
d0b74c17 71 $self->{digest} = $digest_func ? $digest_func : \&Digest::MD5::md5;
251dfd0e 72 $self->{hash_size} = $hash_size ? $hash_size : 16;
612969fb 73
beac1dff 74 return $self->precalc_sizes();
612969fb 75}
76
77sub new {
78 my $class = shift;
79 my ($args) = @_;
80
81 my $self = bless {
82 long_size => 4,
83 long_pack => 'N',
84 data_size => 4,
85 data_pack => 'N',
251dfd0e 86
612969fb 87 digest => \&Digest::MD5::md5,
88 hash_size => 16,
251dfd0e 89
81d16922 90 ##
d5d7c51d 91 # Maximum number of buckets per list before another level of indexing is
92 # done.
93 # Increase this value for slightly greater speed, but larger database
94 # files. DO NOT decrease this value below 16, due to risk of recursive
95 # reindex overrun.
81d16922 96 ##
612969fb 97 max_buckets => 16,
98 }, $class;
99
251dfd0e 100 $self->precalc_sizes;
612969fb 101
102 return $self;
1bf65be7 103}
104
70b55428 105sub setup_fh {
106 my $self = shift;
107 my ($obj) = @_;
108
109 $self->open( $obj ) if !defined $obj->_fh;
110
6fde4ed2 111 my $fh = $obj->_fh;
112 flock $fh, LOCK_EX;
118ba343 113
6fde4ed2 114 unless ( $obj->{base_offset} ) {
118ba343 115 seek($fh, 0 + $obj->_root->{file_offset}, SEEK_SET);
116 my $signature;
117 my $bytes_read = read( $fh, $signature, length(SIG_FILE));
118
119 ##
120 # File is empty -- write signature and master index
121 ##
122 if (!$bytes_read) {
c9ec091a 123 my $loc = $self->_request_space( $obj, length( SIG_FILE ) );
124 seek($fh, $loc + $obj->_root->{file_offset}, SEEK_SET);
118ba343 125 print( $fh SIG_FILE);
126
c9ec091a 127 $obj->{base_offset} = $self->_request_space(
16d1ad9b 128 $obj, $self->tag_size( $self->{index_size} ),
c9ec091a 129 );
118ba343 130
9e4f83a0 131 $self->write_tag(
c9ec091a 132 $obj, $obj->_base_offset, $obj->_type,
f37c15ab 133 chr(0)x$self->{index_size},
118ba343 134 );
135
136 # Flush the filehandle
137 my $old_fh = select $fh;
138 my $old_af = $|; $| = 1; $| = $old_af;
139 select $old_fh;
140 }
141 else {
142 $obj->{base_offset} = $bytes_read;
143
144 ##
145 # Check signature was valid
146 ##
147 unless ($signature eq SIG_FILE) {
148 $self->close_fh( $obj );
149 $obj->_throw_error("Signature not found -- file is not a Deep DB");
150 }
151
152 ##
153 # Get our type from master index signature
154 ##
155 my $tag = $self->load_tag($obj, $obj->_base_offset)
156 or $obj->_throw_error("Corrupted file, no master index record");
157
158 unless ($obj->{type} eq $tag->{signature}) {
159 $obj->_throw_error("File type mismatch");
160 }
161 }
118ba343 162 }
e06824f8 163
673464d9 164 #XXX We have to make sure we don't mess up when autoflush isn't turned on
70b55428 165 unless ( $obj->_root->{inode} ) {
166 my @stats = stat($obj->_fh);
167 $obj->_root->{inode} = $stats[1];
168 $obj->_root->{end} = $stats[7];
169 }
170
6fde4ed2 171 flock $fh, LOCK_UN;
172
70b55428 173 return 1;
174}
175
a20d9a3f 176sub open {
20f7b20c 177 ##
178 # Open a fh to the database, create if nonexistent.
179 # Make sure file signature matches DBM::Deep spec.
180 ##
a20d9a3f 181 my $self = shift;
70b55428 182 my ($obj) = @_;
a20d9a3f 183
673464d9 184 # Theoretically, adding O_BINARY should remove the need for the binmode
185 # Of course, testing it is going to be ... interesting.
186 my $flags = O_RDWR | O_CREAT | O_BINARY;
a20d9a3f 187
673464d9 188 my $fh;
d5d7c51d 189 my $filename = $obj->_root->{file};
190 sysopen( $fh, $filename, $flags )
191 or $obj->_throw_error("Cannot sysopen file '$filename': $!");
673464d9 192 $obj->_root->{fh} = $fh;
a20d9a3f 193
194 #XXX Can we remove this by using the right sysopen() flags?
195 # Maybe ... q.v. above
196 binmode $fh; # for win32
197
cd59cad8 198 if ($obj->_root->{autoflush}) {
a20d9a3f 199 my $old = select $fh;
200 $|=1;
201 select $old;
202 }
20f7b20c 203
a20d9a3f 204 return 1;
205}
206
3d1b8be9 207sub close_fh {
cd59cad8 208 my $self = shift;
a21f2d90 209 my ($obj) = @_;
cd59cad8 210
211 if ( my $fh = $obj->_root->{fh} ) {
212 close $fh;
213 }
214 $obj->_root->{fh} = undef;
215
216 return 1;
217}
218
16d1ad9b 219sub tag_size {
220 my $self = shift;
221 my ($size) = @_;
222 return SIG_SIZE + $self->{data_size} + $size;
223}
224
9e4f83a0 225sub write_tag {
20f7b20c 226 ##
227 # Given offset, signature and content, create tag and write to disk
228 ##
d4b1166e 229 my $self = shift;
20f7b20c 230 my ($obj, $offset, $sig, $content) = @_;
f37c15ab 231 my $size = length( $content );
20f7b20c 232
d4b1166e 233 my $fh = $obj->_fh;
234
f37c15ab 235 if ( defined $offset ) {
236 seek($fh, $offset + $obj->_root->{file_offset}, SEEK_SET);
237 }
238
251dfd0e 239 print( $fh $sig . pack($self->{data_pack}, $size) . $content );
20f7b20c 240
f37c15ab 241 return unless defined $offset;
242
20f7b20c 243 return {
244 signature => $sig,
245 size => $size,
8db25060 246 offset => $offset + SIG_SIZE + $self->{data_size},
20f7b20c 247 content => $content
248 };
d4b1166e 249}
250
251sub load_tag {
20f7b20c 252 ##
253 # Given offset, load single tag and return signature, size and data
254 ##
d4b1166e 255 my $self = shift;
20f7b20c 256 my ($obj, $offset) = @_;
257
e06824f8 258# print join(':',map{$_||''}caller(1)), $/;
259
d4b1166e 260 my $fh = $obj->_fh;
261
20f7b20c 262 seek($fh, $offset + $obj->_root->{file_offset}, SEEK_SET);
e5fc7e69 263
75be6413 264 #XXX I'm not sure this check will work if autoflush isn't enabled ...
e5fc7e69 265 return if eof $fh;
20f7b20c 266
d4b1166e 267 my $b;
8db25060 268 read( $fh, $b, SIG_SIZE + $self->{data_size} );
251dfd0e 269 my ($sig, $size) = unpack( "A $self->{data_pack}", $b );
20f7b20c 270
271 my $buffer;
272 read( $fh, $buffer, $size);
273
274 return {
275 signature => $sig,
276 size => $size,
8db25060 277 offset => $offset + SIG_SIZE + $self->{data_size},
20f7b20c 278 content => $buffer
279 };
d4b1166e 280}
281
29b01632 282sub _length_needed {
283 my $self = shift;
f37c15ab 284 my ($obj, $value, $key) = @_;
29b01632 285
286 my $is_dbm_deep = eval {
287 local $SIG{'__DIE__'};
288 $value->isa( 'DBM::Deep' );
289 };
290
f37c15ab 291 my $len = SIG_SIZE + $self->{data_size}
292 + $self->{data_size} + length( $key );
29b01632 293
f37c15ab 294 if ( $is_dbm_deep && $value->_root eq $obj->_root ) {
295 return $len + $self->{long_size};
29b01632 296 }
297
298 my $r = Scalar::Util::reftype( $value ) || '';
9a187d8c 299 if ( $obj->_root->{autobless} ) {
300 # This is for the bit saying whether or not this thing is blessed.
301 $len += 1;
302 }
303
29b01632 304 unless ( $r eq 'HASH' || $r eq 'ARRAY' ) {
f37c15ab 305 if ( defined $value ) {
306 $len += length( $value );
307 }
308 return $len;
29b01632 309 }
310
f37c15ab 311 $len += $self->{index_size};
29b01632 312
313 # if autobless is enabled, must also take into consideration
f37c15ab 314 # the class name as it is stored after the key.
29b01632 315 if ( $obj->_root->{autobless} ) {
316 my $value_class = Scalar::Util::blessed($value);
f37c15ab 317 if ( defined $value_class && !$is_dbm_deep ) {
318 $len += $self->{data_size} + length($value_class);
29b01632 319 }
320 }
321
f37c15ab 322 return $len;
29b01632 323}
324
20f7b20c 325sub add_bucket {
326 ##
327 # Adds one key/value pair to bucket list, given offset, MD5 digest of key,
328 # plain (undigested) key and value.
329 ##
d4b1166e 330 my $self = shift;
20f7b20c 331 my ($obj, $tag, $md5, $plain_key, $value) = @_;
75be6413 332
eea0d863 333 # This verifies that only supported values will be stored.
334 {
335 my $r = Scalar::Util::reftype( $value );
336 last if !defined $r;
337
338 last if $r eq 'HASH';
339 last if $r eq 'ARRAY';
340
341 $obj->_throw_error(
342 "Storage of variables of type '$r' is not supported."
343 );
344 }
345
20f7b20c 346 my $location = 0;
347 my $result = 2;
348
349 my $root = $obj->_root;
f37c15ab 350 my $fh = $obj->_fh;
20f7b20c 351
f37c15ab 352 my $actual_length = $self->_length_needed( $obj, $value, $plain_key );
20f7b20c 353
9a187d8c 354 my ($subloc, $offset, $size) = $self->_find_in_buckets( $tag, $md5 );
75be6413 355
386bab6c 356 # Updating a known md5
357 if ( $subloc ) {
358 $result = 1;
20f7b20c 359
386bab6c 360 if ($actual_length <= $size) {
361 $location = $subloc;
20f7b20c 362 }
75be6413 363 else {
f37c15ab 364 $location = $self->_request_space( $obj, $actual_length );
386bab6c 365 seek(
366 $fh,
9a187d8c 367 $tag->{offset} + $offset
368 + $self->{hash_size} + $root->{file_offset},
386bab6c 369 SEEK_SET,
370 );
9a187d8c 371 print( $fh pack($self->{long_pack}, $location ) );
372 print( $fh pack($self->{long_pack}, $actual_length ) );
75be6413 373 }
75be6413 374 }
386bab6c 375 # Adding a new md5
376 elsif ( defined $offset ) {
f37c15ab 377 $location = $self->_request_space( $obj, $actual_length );
386bab6c 378
379 seek( $fh, $tag->{offset} + $offset + $root->{file_offset}, SEEK_SET );
9a187d8c 380 print( $fh $md5 . pack($self->{long_pack}, $location ) );
381 print( $fh pack($self->{long_pack}, $actual_length ) );
386bab6c 382 }
383 # If bucket didn't fit into list, split into a new index level
384 else {
9a187d8c 385#XXX This is going to be a problem.
7b1e1aa1 386 $self->split_index( $obj, $md5, $tag );
386bab6c 387
f37c15ab 388 $location = $self->_request_space( $obj, $actual_length );
386bab6c 389 }
20f7b20c 390
d5d7c51d 391 $self->write_value( $obj, $location, $plain_key, $value );
392
393 return $result;
394}
395
396sub write_value {
397 my $self = shift;
398 my ($obj, $location, $key, $value) = @_;
399
400 my $fh = $obj->_fh;
401 my $root = $obj->_root;
402
403 my $is_dbm_deep = eval {
404 local $SIG{'__DIE__'};
405 $value->isa( 'DBM::Deep' );
406 };
407
408 my $internal_ref = $is_dbm_deep && ($value->_root eq $root);
409
410 seek($fh, $location + $root->{file_offset}, SEEK_SET);
411
20f7b20c 412 ##
d5d7c51d 413 # Write signature based on content type, set content length and write
414 # actual value.
20f7b20c 415 ##
d5d7c51d 416 my $r = Scalar::Util::reftype($value) || '';
d5d7c51d 417 if ( $internal_ref ) {
9e4f83a0 418 $self->write_tag( $obj, undef, SIG_INTERNAL,pack($self->{long_pack}, $value->_base_offset) );
f37c15ab 419 }
420 elsif ($r eq 'HASH') {
9e4f83a0 421 $self->write_tag( $obj, undef, SIG_HASH, chr(0)x$self->{index_size} );
f37c15ab 422 }
423 elsif ($r eq 'ARRAY') {
9e4f83a0 424 $self->write_tag( $obj, undef, SIG_ARRAY, chr(0)x$self->{index_size} );
f37c15ab 425 }
426 elsif (!defined($value)) {
9e4f83a0 427 $self->write_tag( $obj, undef, SIG_NULL, '' );
d5d7c51d 428 }
429 else {
9e4f83a0 430 $self->write_tag( $obj, undef, SIG_DATA, $value );
d5d7c51d 431 }
20f7b20c 432
d5d7c51d 433 ##
434 # Plain key is stored AFTER value, as keys are typically fetched less often.
435 ##
436 print( $fh pack($self->{data_pack}, length($key)) . $key );
20f7b20c 437
9a187d8c 438 # Internal references don't care about autobless
439 return 1 if $internal_ref;
440
d5d7c51d 441 ##
442 # If value is blessed, preserve class name
443 ##
444 if ( $root->{autobless} ) {
445 my $value_class = Scalar::Util::blessed($value);
f37c15ab 446 if ( defined $value_class && !$is_dbm_deep ) {
d5d7c51d 447 print( $fh chr(1) );
448 print( $fh pack($self->{data_pack}, length($value_class)) . $value_class );
20f7b20c 449 }
d5d7c51d 450 else {
451 print( $fh chr(0) );
20f7b20c 452 }
d5d7c51d 453 }
20f7b20c 454
d5d7c51d 455 ##
d5d7c51d 456 # If content is a hash or array, create new child DBM::Deep object and
457 # pass each key or element to it.
458 ##
f37c15ab 459 if ( !$internal_ref ) {
d5d7c51d 460 if ($r eq 'HASH') {
461 my $branch = DBM::Deep->new(
462 type => DBM::Deep->TYPE_HASH,
463 base_offset => $location,
464 root => $root,
465 );
466 foreach my $key (keys %{$value}) {
467 $branch->STORE( $key, $value->{$key} );
20f7b20c 468 }
d5d7c51d 469 }
470 elsif ($r eq 'ARRAY') {
471 my $branch = DBM::Deep->new(
472 type => DBM::Deep->TYPE_ARRAY,
473 base_offset => $location,
474 root => $root,
475 );
476 my $index = 0;
477 foreach my $element (@{$value}) {
478 $branch->STORE( $index, $element );
479 $index++;
20f7b20c 480 }
481 }
20f7b20c 482 }
d4b1166e 483
d5d7c51d 484 return 1;
d4b1166e 485}
486
75be6413 487sub split_index {
488 my $self = shift;
489 my ($obj, $md5, $tag) = @_;
490
491 my $fh = $obj->_fh;
492 my $root = $obj->_root;
16d1ad9b 493
494 my $loc = $self->_request_space(
495 $obj, $self->tag_size( $self->{index_size} ),
496 );
497
7b1e1aa1 498 seek($fh, $tag->{ref_loc} + $root->{file_offset}, SEEK_SET);
16d1ad9b 499 print( $fh pack($self->{long_pack}, $loc) );
75be6413 500
9e4f83a0 501 my $index_tag = $self->write_tag(
16d1ad9b 502 $obj, $loc, SIG_INDEX,
f37c15ab 503 chr(0)x$self->{index_size},
75be6413 504 );
505
506 my @offsets = ();
507
7b1e1aa1 508 my $keys = $tag->{content}
509 . $md5 . (pack($self->{long_pack}, 0) x 2);
75be6413 510
511 BUCKET:
512 for (my $i = 0; $i <= $self->{max_buckets}; $i++) {
9a187d8c 513 my ($key, $old_subloc, $size) = $self->_get_key_subloc( $keys, $i );
75be6413 514
515 next BUCKET unless $key;
516
75be6413 517 my $num = ord(substr($key, $tag->{ch} + 1, 1));
518
519 if ($offsets[$num]) {
7b1e1aa1 520 seek($fh, $offsets[$num] + $root->{file_offset}, SEEK_SET);
75be6413 521 my $subkeys;
522 read( $fh, $subkeys, $self->{bucket_list_size});
523
7b1e1aa1 524 my ($subloc, $offset, $size) = $self->_find_in_buckets(
525 { content => $subkeys }, ''
526 );
75be6413 527
7b1e1aa1 528 seek($fh, $offsets[$num] + $offset + $root->{file_offset}, SEEK_SET);
529 print( $fh $key . pack($self->{long_pack}, $old_subloc || $root->{end}) );
530
531 next;
75be6413 532 }
75be6413 533
7b1e1aa1 534 seek($fh, $index_tag->{offset} + ($num * $self->{long_size}) + $root->{file_offset}, SEEK_SET);
2603d86e 535
7b1e1aa1 536 my $loc = $self->_request_space(
537 $obj, $self->tag_size( $self->{bucket_list_size} ),
538 );
2603d86e 539
7b1e1aa1 540 print( $fh pack($self->{long_pack}, $loc) );
75be6413 541
7b1e1aa1 542 my $blist_tag = $self->write_tag(
543 $obj, $loc, SIG_BLIST,
544 chr(0)x$self->{bucket_list_size},
545 );
546
547 seek($fh, $blist_tag->{offset} + $root->{file_offset}, SEEK_SET);
548 print( $fh $key . pack($self->{long_pack}, $old_subloc || $root->{end}) );
549
550 $offsets[$num] = $blist_tag->{offset};
551 }
552
553 $self->_release_space(
554 $obj, $self->tag_size( $self->{index_size} ),
555 $tag->{offset} - SIG_SIZE - $self->{data_size},
556 );
75be6413 557
558 return;
559}
560
8db25060 561sub read_from_loc {
562 my $self = shift;
563 my ($obj, $subloc) = @_;
564
565 my $fh = $obj->_fh;
566
567 ##
568 # Found match -- seek to offset and read signature
569 ##
570 my $signature;
571 seek($fh, $subloc + $obj->_root->{file_offset}, SEEK_SET);
572 read( $fh, $signature, SIG_SIZE);
573
574 ##
575 # If value is a hash or array, return new DBM::Deep object with correct offset
576 ##
577 if (($signature eq SIG_HASH) || ($signature eq SIG_ARRAY)) {
578 my $obj = DBM::Deep->new(
579 type => $signature,
580 base_offset => $subloc,
581 root => $obj->_root,
582 );
583
584 if ($obj->_root->{autobless}) {
585 ##
586 # Skip over value and plain key to see if object needs
587 # to be re-blessed
588 ##
589 seek($fh, $self->{data_size} + $self->{index_size}, SEEK_CUR);
590
591 my $size;
592 read( $fh, $size, $self->{data_size}); $size = unpack($self->{data_pack}, $size);
593 if ($size) { seek($fh, $size, SEEK_CUR); }
594
595 my $bless_bit;
596 read( $fh, $bless_bit, 1);
597 if (ord($bless_bit)) {
598 ##
599 # Yes, object needs to be re-blessed
600 ##
601 my $class_name;
602 read( $fh, $size, $self->{data_size}); $size = unpack($self->{data_pack}, $size);
603 if ($size) { read( $fh, $class_name, $size); }
604 if ($class_name) { $obj = bless( $obj, $class_name ); }
605 }
606 }
607
608 return $obj;
609 }
610 elsif ( $signature eq SIG_INTERNAL ) {
611 my $size;
612 read( $fh, $size, $self->{data_size});
613 $size = unpack($self->{data_pack}, $size);
614
615 if ( $size ) {
616 my $new_loc;
617 read( $fh, $new_loc, $size );
618 $new_loc = unpack( $self->{long_pack}, $new_loc );
619
620 return $self->read_from_loc( $obj, $new_loc );
621 }
622 else {
623 return;
624 }
625 }
626 ##
627 # Otherwise return actual value
628 ##
629 elsif ($signature eq SIG_DATA) {
630 my $size;
631 read( $fh, $size, $self->{data_size});
632 $size = unpack($self->{data_pack}, $size);
633
634 my $value = '';
635 if ($size) { read( $fh, $value, $size); }
636 return $value;
637 }
638
639 ##
640 # Key exists, but content is null
641 ##
642 return;
643}
644
9020ee8c 645sub get_bucket_value {
beac1dff 646 ##
647 # Fetch single value given tag and MD5 digested key.
648 ##
649 my $self = shift;
650 my ($obj, $tag, $md5) = @_;
9020ee8c 651
9a187d8c 652 my ($subloc, $offset, $size) = $self->_find_in_buckets( $tag, $md5 );
386bab6c 653 if ( $subloc ) {
8db25060 654 return $self->read_from_loc( $obj, $subloc );
386bab6c 655 }
beac1dff 656 return;
9020ee8c 657}
ab0e4957 658
659sub delete_bucket {
beac1dff 660 ##
661 # Delete single key/value pair given tag and MD5 digested key.
662 ##
663 my $self = shift;
664 my ($obj, $tag, $md5) = @_;
ab0e4957 665
9a187d8c 666 my ($subloc, $offset, $size) = $self->_find_in_buckets( $tag, $md5 );
386bab6c 667 if ( $subloc ) {
668 my $fh = $obj->_fh;
669 seek($fh, $tag->{offset} + $offset + $obj->_root->{file_offset}, SEEK_SET);
670 print( $fh substr($tag->{content}, $offset + $self->{bucket_size} ) );
251dfd0e 671 print( $fh chr(0) x $self->{bucket_size} );
d0b74c17 672
ab0e4957 673 return 1;
386bab6c 674 }
beac1dff 675 return;
ab0e4957 676}
677
912d50b1 678sub bucket_exists {
beac1dff 679 ##
680 # Check existence of single key given tag and MD5 digested key.
681 ##
682 my $self = shift;
683 my ($obj, $tag, $md5) = @_;
912d50b1 684
9a187d8c 685 my ($subloc, $offset, $size) = $self->_find_in_buckets( $tag, $md5 );
d5d7c51d 686 return $subloc && 1;
912d50b1 687}
688
6736c116 689sub find_bucket_list {
beac1dff 690 ##
691 # Locate offset for bucket list, given digested key
692 ##
693 my $self = shift;
d0b74c17 694 my ($obj, $md5, $args) = @_;
695 $args = {} unless $args;
696
beac1dff 697 ##
698 # Locate offset for bucket list using digest index system
699 ##
e5fc7e69 700 my $tag = $self->load_tag($obj, $obj->_base_offset)
d5d7c51d 701 or $obj->_throw_error( "INTERNAL ERROR - Cannot find tag" );
d0b74c17 702
e5fc7e69 703 my $ch = 0;
8db25060 704 while ($tag->{signature} ne SIG_BLIST) {
d0b74c17 705 my $num = ord substr($md5, $ch, 1);
706
707 my $ref_loc = $tag->{offset} + ($num * $self->{long_size});
708 $tag = $self->index_lookup( $obj, $tag, $num );
709
710 if (!$tag) {
29b01632 711 return if !$args->{create};
d0b74c17 712
16d1ad9b 713 my $loc = $self->_request_space(
714 $obj, $self->tag_size( $self->{bucket_list_size} ),
715 );
716
7b1e1aa1 717 my $fh = $obj->_fh;
718 seek($fh, $ref_loc + $obj->_root->{file_offset}, SEEK_SET);
16d1ad9b 719 print( $fh pack($self->{long_pack}, $loc) );
d0b74c17 720
9e4f83a0 721 $tag = $self->write_tag(
16d1ad9b 722 $obj, $loc, SIG_BLIST,
f37c15ab 723 chr(0)x$self->{bucket_list_size},
d5d7c51d 724 );
725
726 $tag->{ref_loc} = $ref_loc;
727 $tag->{ch} = $ch;
728
729 last;
d0b74c17 730 }
731
16d1ad9b 732 $tag->{ch} = $ch++;
d0b74c17 733 $tag->{ref_loc} = $ref_loc;
beac1dff 734 }
d0b74c17 735
beac1dff 736 return $tag;
6736c116 737}
738
d0b74c17 739sub index_lookup {
740 ##
741 # Given index tag, lookup single entry in index and return .
742 ##
743 my $self = shift;
744 my ($obj, $tag, $index) = @_;
745
746 my $location = unpack(
747 $self->{long_pack},
748 substr(
749 $tag->{content},
750 $index * $self->{long_size},
751 $self->{long_size},
752 ),
753 );
754
755 if (!$location) { return; }
756
757 return $self->load_tag( $obj, $location );
758}
759
6736c116 760sub traverse_index {
beac1dff 761 ##
762 # Scan index and recursively step into deeper levels, looking for next key.
763 ##
6736c116 764 my $self = shift;
765 my ($obj, $offset, $ch, $force_return_next) = @_;
d0b74c17 766
beac1dff 767 my $tag = $self->load_tag($obj, $offset );
6736c116 768
769 my $fh = $obj->_fh;
d0b74c17 770
8db25060 771 if ($tag->{signature} ne SIG_BLIST) {
beac1dff 772 my $content = $tag->{content};
e5fc7e69 773 my $start = $obj->{return_next} ? 0 : ord(substr($obj->{prev_md5}, $ch, 1));
d0b74c17 774
d5d7c51d 775 for (my $idx = $start; $idx < (2**8); $idx++) {
e5fc7e69 776 my $subloc = unpack(
777 $self->{long_pack},
e06824f8 778 substr(
779 $content,
780 $idx * $self->{long_size},
781 $self->{long_size},
782 ),
e5fc7e69 783 );
784
beac1dff 785 if ($subloc) {
e5fc7e69 786 my $result = $self->traverse_index(
787 $obj, $subloc, $ch + 1, $force_return_next,
788 );
789
beac1dff 790 if (defined($result)) { return $result; }
791 }
792 } # index loop
d0b74c17 793
beac1dff 794 $obj->{return_next} = 1;
795 } # tag is an index
d0b74c17 796
e5fc7e69 797 else {
beac1dff 798 my $keys = $tag->{content};
799 if ($force_return_next) { $obj->{return_next} = 1; }
d0b74c17 800
beac1dff 801 ##
802 # Iterate through buckets, looking for a key match
803 ##
8db25060 804 for (my $i = 0; $i < $self->{max_buckets}; $i++) {
9cec1360 805 my ($key, $subloc) = $self->_get_key_subloc( $keys, $i );
d0b74c17 806
8db25060 807 # End of bucket list -- return to outer loop
beac1dff 808 if (!$subloc) {
beac1dff 809 $obj->{return_next} = 1;
810 last;
811 }
8db25060 812 # Located previous key -- return next one found
beac1dff 813 elsif ($key eq $obj->{prev_md5}) {
beac1dff 814 $obj->{return_next} = 1;
815 next;
816 }
8db25060 817 # Seek to bucket location and skip over signature
beac1dff 818 elsif ($obj->{return_next}) {
8db25060 819 seek($fh, $subloc + $obj->_root->{file_offset}, SEEK_SET);
d0b74c17 820
beac1dff 821 # Skip over value to get to plain key
8db25060 822 my $sig;
823 read( $fh, $sig, SIG_SIZE );
824
beac1dff 825 my $size;
e5fc7e69 826 read( $fh, $size, $self->{data_size});
827 $size = unpack($self->{data_pack}, $size);
beac1dff 828 if ($size) { seek($fh, $size, SEEK_CUR); }
d0b74c17 829
beac1dff 830 # Read in plain key and return as scalar
beac1dff 831 my $plain_key;
e5fc7e69 832 read( $fh, $size, $self->{data_size});
833 $size = unpack($self->{data_pack}, $size);
beac1dff 834 if ($size) { read( $fh, $plain_key, $size); }
d0b74c17 835
beac1dff 836 return $plain_key;
837 }
8db25060 838 }
d0b74c17 839
beac1dff 840 $obj->{return_next} = 1;
841 } # tag is a bucket list
d0b74c17 842
beac1dff 843 return;
6736c116 844}
845
846sub get_next_key {
beac1dff 847 ##
848 # Locate next key, given digested previous one
849 ##
6736c116 850 my $self = shift;
851 my ($obj) = @_;
d0b74c17 852
beac1dff 853 $obj->{prev_md5} = $_[1] ? $_[1] : undef;
854 $obj->{return_next} = 0;
d0b74c17 855
beac1dff 856 ##
857 # If the previous key was not specifed, start at the top and
858 # return the first one found.
859 ##
860 if (!$obj->{prev_md5}) {
861 $obj->{prev_md5} = chr(0) x $self->{hash_size};
862 $obj->{return_next} = 1;
863 }
d0b74c17 864
beac1dff 865 return $self->traverse_index( $obj, $obj->_base_offset, 0 );
6736c116 866}
867
75be6413 868# Utilities
869
9cec1360 870sub _get_key_subloc {
75be6413 871 my $self = shift;
872 my ($keys, $idx) = @_;
873
6ed2f3df 874 my ($key, $subloc, $size) = unpack(
875 "a$self->{hash_size} $self->{long_pack} $self->{long_pack}",
75be6413 876 substr(
877 $keys,
9cec1360 878 ($idx * $self->{bucket_size}),
879 $self->{bucket_size},
75be6413 880 ),
881 );
882
6ed2f3df 883 return ($key, $subloc, $size);
75be6413 884}
885
d608b06e 886sub _find_in_buckets {
887 my $self = shift;
888 my ($tag, $md5) = @_;
889
890 BUCKET:
891 for ( my $i = 0; $i < $self->{max_buckets}; $i++ ) {
9a187d8c 892 my ($key, $subloc, $size) = $self->_get_key_subloc(
893 $tag->{content}, $i,
894 );
d608b06e 895
9a187d8c 896 return ($subloc, $i * $self->{bucket_size}, $size) unless $subloc;
d608b06e 897
898 next BUCKET if $key ne $md5;
899
9a187d8c 900 return ($subloc, $i * $self->{bucket_size}, $size);
d608b06e 901 }
902
903 return;
904}
905
7b1e1aa1 906#sub _print_at {
907# my $self = shift;
908# my ($obj, $spot, $data) = @_;
909#
910# my $fh = $obj->_fh;
911# seek( $fh, $spot, SEEK_SET );
912# print( $fh $data );
913#
914# return;
915#}
916
994ccd8e 917sub _request_space {
918 my $self = shift;
919 my ($obj, $size) = @_;
920
921 my $loc = $obj->_root->{end};
c9ec091a 922 $obj->_root->{end} += $size;
994ccd8e 923
924 return $loc;
925}
926
927sub _release_space {
928 my $self = shift;
929 my ($obj, $size, $loc) = @_;
930
7b1e1aa1 931 my $next_loc = 0;
932
933 my $fh = $obj->_fh;
934 seek( $fh, $loc + $obj->_root->{file_offset}, SEEK_SET );
935 print( $fh SIG_FREE
936 . pack($self->{long_pack}, $size )
937 . pack($self->{long_pack}, $next_loc )
938 );
939
994ccd8e 940 return;
941}
942
a20d9a3f 9431;
944__END__
d5d7c51d 945
946# This will be added in later, after more refactoring is done. This is an early
947# attempt at refactoring on the physical level instead of the virtual level.
948sub _read_at {
949 my $self = shift;
950 my ($obj, $spot, $amount, $unpack) = @_;
951
952 my $fh = $obj->_fh;
953 seek( $fh, $spot + $obj->_root->{file_offset}, SEEK_SET );
954
955 my $buffer;
956 my $bytes_read = read( $fh, $buffer, $amount );
957
958 if ( $unpack ) {
959 $buffer = unpack( $unpack, $buffer );
960 }
961
962 if ( wantarray ) {
963 return ($buffer, $bytes_read);
964 }
965 else {
966 return $buffer;
967 }
968}