Moved a lot of open()'s code into setup_fh()
[dbsrgits/DBM-Deep.git] / lib / DBM / Deep / Engine.pm
CommitLineData
a20d9a3f 1package DBM::Deep::Engine;
2
3use strict;
4
5use Fcntl qw( :DEFAULT :flock :seek );
6
8db25060 7##
8# Setup file and tag signatures. These should never change.
9##
10sub SIG_FILE () { 'DPDB' }
11sub SIG_INTERNAL () { 'i' }
12sub SIG_HASH () { 'H' }
13sub SIG_ARRAY () { 'A' }
8db25060 14sub SIG_NULL () { 'N' }
15sub SIG_DATA () { 'D' }
16sub SIG_INDEX () { 'I' }
17sub SIG_BLIST () { 'B' }
18sub SIG_SIZE () { 1 }
19
612969fb 20sub precalc_sizes {
beac1dff 21 ##
22 # Precalculate index, bucket and bucket list sizes
23 ##
251dfd0e 24 my $self = shift;
1bf65be7 25
251dfd0e 26 $self->{index_size} = (2**8) * $self->{long_size};
27 $self->{bucket_size} = $self->{hash_size} + $self->{long_size};
28 $self->{bucket_list_size} = $self->{max_buckets} * $self->{bucket_size};
1bf65be7 29
251dfd0e 30 return 1;
1bf65be7 31}
32
33sub set_pack {
beac1dff 34 ##
35 # Set pack/unpack modes (see file header for more)
36 ##
251dfd0e 37 my $self = shift;
1bf65be7 38 my ($long_s, $long_p, $data_s, $data_p) = @_;
39
81d16922 40 ##
75be6413 41 # Set to 4 and 'N' for 32-bit offset tags (default). Theoretical limit of 4
42 # GB per file.
beac1dff 43 # (Perl must be compiled with largefile support for files > 2 GB)
81d16922 44 #
45 # Set to 8 and 'Q' for 64-bit offsets. Theoretical limit of 16 XB per file.
beac1dff 46 # (Perl must be compiled with largefile and 64-bit long support)
81d16922 47 ##
251dfd0e 48 $self->{long_size} = $long_s ? $long_s : 4;
49 $self->{long_pack} = $long_p ? $long_p : 'N';
1bf65be7 50
81d16922 51 ##
75be6413 52 # Set to 4 and 'N' for 32-bit data length prefixes. Limit of 4 GB for each
d5d7c51d 53 # key/value. Upgrading this is possible (see above) but probably not
54 # necessary. If you need more than 4 GB for a single key or value, this
55 # module is really not for you :-)
81d16922 56 ##
251dfd0e 57 $self->{data_size} = $data_s ? $data_s : 4;
58 $self->{data_pack} = $data_p ? $data_p : 'N';
1bf65be7 59
beac1dff 60 return $self->precalc_sizes();
1bf65be7 61}
62
63sub set_digest {
beac1dff 64 ##
65 # Set key digest function (default is MD5)
66 ##
251dfd0e 67 my $self = shift;
1bf65be7 68 my ($digest_func, $hash_size) = @_;
69
d0b74c17 70 $self->{digest} = $digest_func ? $digest_func : \&Digest::MD5::md5;
251dfd0e 71 $self->{hash_size} = $hash_size ? $hash_size : 16;
612969fb 72
beac1dff 73 return $self->precalc_sizes();
612969fb 74}
75
76sub new {
77 my $class = shift;
78 my ($args) = @_;
79
80 my $self = bless {
81 long_size => 4,
82 long_pack => 'N',
83 data_size => 4,
84 data_pack => 'N',
251dfd0e 85
612969fb 86 digest => \&Digest::MD5::md5,
87 hash_size => 16,
251dfd0e 88
81d16922 89 ##
d5d7c51d 90 # Maximum number of buckets per list before another level of indexing is
91 # done.
92 # Increase this value for slightly greater speed, but larger database
93 # files. DO NOT decrease this value below 16, due to risk of recursive
94 # reindex overrun.
81d16922 95 ##
612969fb 96 max_buckets => 16,
97 }, $class;
98
251dfd0e 99 $self->precalc_sizes;
612969fb 100
101 return $self;
1bf65be7 102}
103
70b55428 104sub setup_fh {
105 my $self = shift;
106 my ($obj) = @_;
107
108 $self->open( $obj ) if !defined $obj->_fh;
109
118ba343 110 unless ( $obj->{base_offset} ) {
111 my $fh = $obj->_fh;
112
113 flock $fh, LOCK_EX;
114
115 seek($fh, 0 + $obj->_root->{file_offset}, SEEK_SET);
116 my $signature;
117 my $bytes_read = read( $fh, $signature, length(SIG_FILE));
118
119 ##
120 # File is empty -- write signature and master index
121 ##
122 if (!$bytes_read) {
123 seek($fh, 0 + $obj->_root->{file_offset}, SEEK_SET);
124 print( $fh SIG_FILE);
125
126 $obj->_root->{end} = length( SIG_FILE );
127
128 $obj->{base_offset} = $self->_request_space($obj, $self->{index_size});
129
130 $self->create_tag(
131 $obj, $obj->_base_offset, $obj->_type, chr(0) x $self->{index_size},
132 );
133
134 # Flush the filehandle
135 my $old_fh = select $fh;
136 my $old_af = $|; $| = 1; $| = $old_af;
137 select $old_fh;
138 }
139 else {
140 $obj->{base_offset} = $bytes_read;
141
142 ##
143 # Check signature was valid
144 ##
145 unless ($signature eq SIG_FILE) {
146 $self->close_fh( $obj );
147 $obj->_throw_error("Signature not found -- file is not a Deep DB");
148 }
149
150 ##
151 # Get our type from master index signature
152 ##
153 my $tag = $self->load_tag($obj, $obj->_base_offset)
154 or $obj->_throw_error("Corrupted file, no master index record");
155
156 unless ($obj->{type} eq $tag->{signature}) {
157 $obj->_throw_error("File type mismatch");
158 }
159 }
160
161 flock $fh, LOCK_UN;
162 }
e06824f8 163
673464d9 164 #XXX We have to make sure we don't mess up when autoflush isn't turned on
70b55428 165 unless ( $obj->_root->{inode} ) {
166 my @stats = stat($obj->_fh);
167 $obj->_root->{inode} = $stats[1];
168 $obj->_root->{end} = $stats[7];
169 }
170
171 return 1;
172}
173
a20d9a3f 174sub open {
20f7b20c 175 ##
176 # Open a fh to the database, create if nonexistent.
177 # Make sure file signature matches DBM::Deep spec.
178 ##
a20d9a3f 179 my $self = shift;
70b55428 180 my ($obj) = @_;
a20d9a3f 181
3d1b8be9 182 if (defined($obj->_fh)) { $self->close_fh( $obj ); }
20f7b20c 183
673464d9 184 # Theoretically, adding O_BINARY should remove the need for the binmode
185 # Of course, testing it is going to be ... interesting.
186 my $flags = O_RDWR | O_CREAT | O_BINARY;
a20d9a3f 187
673464d9 188 my $fh;
d5d7c51d 189 my $filename = $obj->_root->{file};
190 sysopen( $fh, $filename, $flags )
191 or $obj->_throw_error("Cannot sysopen file '$filename': $!");
673464d9 192 $obj->_root->{fh} = $fh;
a20d9a3f 193
194 #XXX Can we remove this by using the right sysopen() flags?
195 # Maybe ... q.v. above
196 binmode $fh; # for win32
197
cd59cad8 198 if ($obj->_root->{autoflush}) {
a20d9a3f 199 my $old = select $fh;
200 $|=1;
201 select $old;
202 }
20f7b20c 203
a20d9a3f 204 return 1;
205}
206
3d1b8be9 207sub close_fh {
cd59cad8 208 my $self = shift;
a21f2d90 209 my ($obj) = @_;
cd59cad8 210
211 if ( my $fh = $obj->_root->{fh} ) {
212 close $fh;
213 }
214 $obj->_root->{fh} = undef;
215
216 return 1;
217}
218
d4b1166e 219sub create_tag {
20f7b20c 220 ##
221 # Given offset, signature and content, create tag and write to disk
222 ##
d4b1166e 223 my $self = shift;
20f7b20c 224 my ($obj, $offset, $sig, $content) = @_;
225 my $size = length($content);
226
d4b1166e 227 my $fh = $obj->_fh;
228
20f7b20c 229 seek($fh, $offset + $obj->_root->{file_offset}, SEEK_SET);
251dfd0e 230 print( $fh $sig . pack($self->{data_pack}, $size) . $content );
20f7b20c 231
232 if ($offset == $obj->_root->{end}) {
8db25060 233 $obj->_root->{end} += SIG_SIZE + $self->{data_size} + $size;
20f7b20c 234 }
235
236 return {
237 signature => $sig,
238 size => $size,
8db25060 239 offset => $offset + SIG_SIZE + $self->{data_size},
20f7b20c 240 content => $content
241 };
d4b1166e 242}
243
244sub load_tag {
20f7b20c 245 ##
246 # Given offset, load single tag and return signature, size and data
247 ##
d4b1166e 248 my $self = shift;
20f7b20c 249 my ($obj, $offset) = @_;
250
e06824f8 251# print join(':',map{$_||''}caller(1)), $/;
252
d4b1166e 253 my $fh = $obj->_fh;
254
20f7b20c 255 seek($fh, $offset + $obj->_root->{file_offset}, SEEK_SET);
e5fc7e69 256
75be6413 257 #XXX I'm not sure this check will work if autoflush isn't enabled ...
e5fc7e69 258 return if eof $fh;
20f7b20c 259
d4b1166e 260 my $b;
8db25060 261 read( $fh, $b, SIG_SIZE + $self->{data_size} );
251dfd0e 262 my ($sig, $size) = unpack( "A $self->{data_pack}", $b );
20f7b20c 263
264 my $buffer;
265 read( $fh, $buffer, $size);
266
267 return {
268 signature => $sig,
269 size => $size,
8db25060 270 offset => $offset + SIG_SIZE + $self->{data_size},
20f7b20c 271 content => $buffer
272 };
d4b1166e 273}
274
20f7b20c 275sub add_bucket {
276 ##
277 # Adds one key/value pair to bucket list, given offset, MD5 digest of key,
278 # plain (undigested) key and value.
279 ##
d4b1166e 280 my $self = shift;
20f7b20c 281 my ($obj, $tag, $md5, $plain_key, $value) = @_;
75be6413 282
eea0d863 283 # This verifies that only supported values will be stored.
284 {
285 my $r = Scalar::Util::reftype( $value );
286 last if !defined $r;
287
288 last if $r eq 'HASH';
289 last if $r eq 'ARRAY';
290
291 $obj->_throw_error(
292 "Storage of variables of type '$r' is not supported."
293 );
294 }
295
20f7b20c 296 my $location = 0;
297 my $result = 2;
298
299 my $root = $obj->_root;
300
d5d7c51d 301 my $is_dbm_deep = eval {
302 local $SIG{'__DIE__'};
303 $value->isa( 'DBM::Deep' );
304 };
305
20f7b20c 306 my $internal_ref = $is_dbm_deep && ($value->_root eq $root);
307
308 my $fh = $obj->_fh;
309
386bab6c 310 my ($subloc, $offset) = $self->_find_in_buckets( $tag, $md5 );
75be6413 311
386bab6c 312 # Updating a known md5
313 if ( $subloc ) {
314 $result = 1;
20f7b20c 315
386bab6c 316 ##
d5d7c51d 317 # If value is a hash, array, or raw value with equal or less size, we
318 # can reuse the same content area of the database. Otherwise, we have
319 # to create a new content area at the EOF.
386bab6c 320 ##
321 my $actual_length;
322 if ( $internal_ref ) {
323 $actual_length = $self->{long_size};
324 }
325 else {
326 my $r = Scalar::Util::reftype( $value ) || '';
327 if ( $r eq 'HASH' || $r eq 'ARRAY' ) {
328 $actual_length = $self->{index_size};
329
330 # if autobless is enabled, must also take into consideration
331 # the class name, as it is stored along with key/value.
332 if ( $root->{autobless} ) {
333 my $value_class = Scalar::Util::blessed($value);
334 if ( defined $value_class && !$value->isa('DBM::Deep') ) {
335 $actual_length += length($value_class);
8db25060 336 }
75be6413 337 }
20f7b20c 338 }
386bab6c 339 else { $actual_length = length($value); }
75be6413 340 }
20f7b20c 341
d5d7c51d 342 seek($fh, $subloc + SIG_SIZE + $root->{file_offset}, SEEK_SET);
343 my $size;
344 read( $fh, $size, $self->{data_size});
345 $size = unpack($self->{data_pack}, $size);
346
386bab6c 347 if ($actual_length <= $size) {
348 $location = $subloc;
20f7b20c 349 }
75be6413 350 else {
351 $location = $root->{end};
386bab6c 352 seek(
353 $fh,
354 $tag->{offset} + $offset + $self->{hash_size} + $root->{file_offset},
355 SEEK_SET,
356 );
357 print( $fh pack($self->{long_pack}, $location) );
75be6413 358 }
75be6413 359 }
386bab6c 360 # Adding a new md5
361 elsif ( defined $offset ) {
386bab6c 362 $location = $root->{end};
363
364 seek( $fh, $tag->{offset} + $offset + $root->{file_offset}, SEEK_SET );
365 print( $fh $md5 . pack($self->{long_pack}, $location) );
366 }
367 # If bucket didn't fit into list, split into a new index level
368 else {
369 $self->split_index( $obj, $md5, $tag );
370
371 $location = $root->{end};
372 }
20f7b20c 373
d5d7c51d 374 $self->write_value( $obj, $location, $plain_key, $value );
375
376 return $result;
377}
378
379sub write_value {
380 my $self = shift;
381 my ($obj, $location, $key, $value) = @_;
382
383 my $fh = $obj->_fh;
384 my $root = $obj->_root;
385
386 my $is_dbm_deep = eval {
387 local $SIG{'__DIE__'};
388 $value->isa( 'DBM::Deep' );
389 };
390
391 my $internal_ref = $is_dbm_deep && ($value->_root eq $root);
392
393 seek($fh, $location + $root->{file_offset}, SEEK_SET);
394
20f7b20c 395 ##
d5d7c51d 396 # Write signature based on content type, set content length and write
397 # actual value.
20f7b20c 398 ##
d5d7c51d 399 my $r = Scalar::Util::reftype($value) || '';
400 my $content_length;
401 if ( $internal_ref ) {
402 print( $fh SIG_INTERNAL );
403 print( $fh pack($self->{data_pack}, $self->{long_size}) );
404 print( $fh pack($self->{long_pack}, $value->_base_offset) );
405 $content_length = $self->{long_size};
406 }
407 else {
408 if ($r eq 'HASH') {
409 print( $fh SIG_HASH );
410 print( $fh pack($self->{data_pack}, $self->{index_size}) . chr(0) x $self->{index_size} );
411 $content_length = $self->{index_size};
412 }
413 elsif ($r eq 'ARRAY') {
414 print( $fh SIG_ARRAY );
415 print( $fh pack($self->{data_pack}, $self->{index_size}) . chr(0) x $self->{index_size} );
416 $content_length = $self->{index_size};
417 }
418 elsif (!defined($value)) {
419 print( $fh SIG_NULL );
420 print( $fh pack($self->{data_pack}, 0) );
421 $content_length = 0;
20f7b20c 422 }
423 else {
d5d7c51d 424 print( $fh SIG_DATA );
425 print( $fh pack($self->{data_pack}, length($value)) . $value );
426 $content_length = length($value);
20f7b20c 427 }
d5d7c51d 428 }
20f7b20c 429
d5d7c51d 430 ##
431 # Plain key is stored AFTER value, as keys are typically fetched less often.
432 ##
433 print( $fh pack($self->{data_pack}, length($key)) . $key );
20f7b20c 434
d5d7c51d 435 ##
436 # If value is blessed, preserve class name
437 ##
438 if ( $root->{autobless} ) {
439 my $value_class = Scalar::Util::blessed($value);
440 if ( defined $value_class && !$value->isa( 'DBM::Deep' ) ) {
441 ##
442 # Blessed ref -- will restore later
443 ##
444 print( $fh chr(1) );
445 print( $fh pack($self->{data_pack}, length($value_class)) . $value_class );
446 $content_length += 1;
447 $content_length += $self->{data_size} + length($value_class);
20f7b20c 448 }
d5d7c51d 449 else {
450 print( $fh chr(0) );
451 $content_length += 1;
20f7b20c 452 }
d5d7c51d 453 }
20f7b20c 454
d5d7c51d 455 ##
456 # If this is a new content area, advance EOF counter
457 ##
458 if ($location == $root->{end}) {
459 $root->{end} += SIG_SIZE;
460 $root->{end} += $self->{data_size} + $content_length;
461 $root->{end} += $self->{data_size} + length($key);
462 }
463
464 ##
465 # If content is a hash or array, create new child DBM::Deep object and
466 # pass each key or element to it.
467 ##
468 if ( ! $internal_ref ) {
469 if ($r eq 'HASH') {
470 my $branch = DBM::Deep->new(
471 type => DBM::Deep->TYPE_HASH,
472 base_offset => $location,
473 root => $root,
474 );
475 foreach my $key (keys %{$value}) {
476 $branch->STORE( $key, $value->{$key} );
20f7b20c 477 }
d5d7c51d 478 }
479 elsif ($r eq 'ARRAY') {
480 my $branch = DBM::Deep->new(
481 type => DBM::Deep->TYPE_ARRAY,
482 base_offset => $location,
483 root => $root,
484 );
485 my $index = 0;
486 foreach my $element (@{$value}) {
487 $branch->STORE( $index, $element );
488 $index++;
20f7b20c 489 }
490 }
20f7b20c 491 }
d4b1166e 492
d5d7c51d 493 return 1;
d4b1166e 494}
495
75be6413 496sub split_index {
497 my $self = shift;
498 my ($obj, $md5, $tag) = @_;
499
500 my $fh = $obj->_fh;
501 my $root = $obj->_root;
502 my $keys = $tag->{content};
503
504 seek($fh, $tag->{ref_loc} + $root->{file_offset}, SEEK_SET);
505 print( $fh pack($self->{long_pack}, $root->{end}) );
506
507 my $index_tag = $self->create_tag(
508 $obj,
509 $root->{end},
8db25060 510 SIG_INDEX,
75be6413 511 chr(0) x $self->{index_size},
512 );
513
514 my @offsets = ();
515
516 $keys .= $md5 . pack($self->{long_pack}, 0);
517
518 BUCKET:
519 for (my $i = 0; $i <= $self->{max_buckets}; $i++) {
9cec1360 520 my ($key, $old_subloc) = $self->_get_key_subloc( $keys, $i );
75be6413 521
522 next BUCKET unless $key;
523
75be6413 524 my $num = ord(substr($key, $tag->{ch} + 1, 1));
525
526 if ($offsets[$num]) {
8db25060 527 my $offset = $offsets[$num] + SIG_SIZE + $self->{data_size};
75be6413 528 seek($fh, $offset + $root->{file_offset}, SEEK_SET);
529 my $subkeys;
530 read( $fh, $subkeys, $self->{bucket_list_size});
531
532 for (my $k=0; $k<$self->{max_buckets}; $k++) {
9cec1360 533 my ($temp, $subloc) = $self->_get_key_subloc( $subkeys, $k );
75be6413 534
535 if (!$subloc) {
536 seek($fh, $offset + ($k * $self->{bucket_size}) + $root->{file_offset}, SEEK_SET);
537 print( $fh $key . pack($self->{long_pack}, $old_subloc || $root->{end}) );
538 last;
539 }
540 } # k loop
541 }
542 else {
543 $offsets[$num] = $root->{end};
544 seek($fh, $index_tag->{offset} + ($num * $self->{long_size}) + $root->{file_offset}, SEEK_SET);
545 print( $fh pack($self->{long_pack}, $root->{end}) );
546
8db25060 547 my $blist_tag = $self->create_tag($obj, $root->{end}, SIG_BLIST, chr(0) x $self->{bucket_list_size});
75be6413 548
549 seek($fh, $blist_tag->{offset} + $root->{file_offset}, SEEK_SET);
550 print( $fh $key . pack($self->{long_pack}, $old_subloc || $root->{end}) );
551 }
552 } # i loop
553
554 return;
555}
556
8db25060 557sub read_from_loc {
558 my $self = shift;
559 my ($obj, $subloc) = @_;
560
561 my $fh = $obj->_fh;
562
563 ##
564 # Found match -- seek to offset and read signature
565 ##
566 my $signature;
567 seek($fh, $subloc + $obj->_root->{file_offset}, SEEK_SET);
568 read( $fh, $signature, SIG_SIZE);
569
570 ##
571 # If value is a hash or array, return new DBM::Deep object with correct offset
572 ##
573 if (($signature eq SIG_HASH) || ($signature eq SIG_ARRAY)) {
574 my $obj = DBM::Deep->new(
575 type => $signature,
576 base_offset => $subloc,
577 root => $obj->_root,
578 );
579
580 if ($obj->_root->{autobless}) {
581 ##
582 # Skip over value and plain key to see if object needs
583 # to be re-blessed
584 ##
585 seek($fh, $self->{data_size} + $self->{index_size}, SEEK_CUR);
586
587 my $size;
588 read( $fh, $size, $self->{data_size}); $size = unpack($self->{data_pack}, $size);
589 if ($size) { seek($fh, $size, SEEK_CUR); }
590
591 my $bless_bit;
592 read( $fh, $bless_bit, 1);
593 if (ord($bless_bit)) {
594 ##
595 # Yes, object needs to be re-blessed
596 ##
597 my $class_name;
598 read( $fh, $size, $self->{data_size}); $size = unpack($self->{data_pack}, $size);
599 if ($size) { read( $fh, $class_name, $size); }
600 if ($class_name) { $obj = bless( $obj, $class_name ); }
601 }
602 }
603
604 return $obj;
605 }
606 elsif ( $signature eq SIG_INTERNAL ) {
607 my $size;
608 read( $fh, $size, $self->{data_size});
609 $size = unpack($self->{data_pack}, $size);
610
611 if ( $size ) {
612 my $new_loc;
613 read( $fh, $new_loc, $size );
614 $new_loc = unpack( $self->{long_pack}, $new_loc );
615
616 return $self->read_from_loc( $obj, $new_loc );
617 }
618 else {
619 return;
620 }
621 }
622 ##
623 # Otherwise return actual value
624 ##
625 elsif ($signature eq SIG_DATA) {
626 my $size;
627 read( $fh, $size, $self->{data_size});
628 $size = unpack($self->{data_pack}, $size);
629
630 my $value = '';
631 if ($size) { read( $fh, $value, $size); }
632 return $value;
633 }
634
635 ##
636 # Key exists, but content is null
637 ##
638 return;
639}
640
9020ee8c 641sub get_bucket_value {
beac1dff 642 ##
643 # Fetch single value given tag and MD5 digested key.
644 ##
645 my $self = shift;
646 my ($obj, $tag, $md5) = @_;
9020ee8c 647
386bab6c 648 my ($subloc, $offset) = $self->_find_in_buckets( $tag, $md5 );
649 if ( $subloc ) {
8db25060 650 return $self->read_from_loc( $obj, $subloc );
386bab6c 651 }
beac1dff 652 return;
9020ee8c 653}
ab0e4957 654
655sub delete_bucket {
beac1dff 656 ##
657 # Delete single key/value pair given tag and MD5 digested key.
658 ##
659 my $self = shift;
660 my ($obj, $tag, $md5) = @_;
ab0e4957 661
386bab6c 662 my ($subloc, $offset) = $self->_find_in_buckets( $tag, $md5 );
663 if ( $subloc ) {
664 my $fh = $obj->_fh;
665 seek($fh, $tag->{offset} + $offset + $obj->_root->{file_offset}, SEEK_SET);
666 print( $fh substr($tag->{content}, $offset + $self->{bucket_size} ) );
251dfd0e 667 print( $fh chr(0) x $self->{bucket_size} );
d0b74c17 668
ab0e4957 669 return 1;
386bab6c 670 }
beac1dff 671 return;
ab0e4957 672}
673
912d50b1 674sub bucket_exists {
beac1dff 675 ##
676 # Check existence of single key given tag and MD5 digested key.
677 ##
678 my $self = shift;
679 my ($obj, $tag, $md5) = @_;
912d50b1 680
386bab6c 681 my ($subloc, $offset) = $self->_find_in_buckets( $tag, $md5 );
d5d7c51d 682 return $subloc && 1;
912d50b1 683}
684
6736c116 685sub find_bucket_list {
beac1dff 686 ##
687 # Locate offset for bucket list, given digested key
688 ##
689 my $self = shift;
d0b74c17 690 my ($obj, $md5, $args) = @_;
691 $args = {} unless $args;
692
beac1dff 693 ##
694 # Locate offset for bucket list using digest index system
695 ##
e5fc7e69 696 my $tag = $self->load_tag($obj, $obj->_base_offset)
d5d7c51d 697 or $obj->_throw_error( "INTERNAL ERROR - Cannot find tag" );
d0b74c17 698
e5fc7e69 699 my $ch = 0;
8db25060 700 while ($tag->{signature} ne SIG_BLIST) {
d0b74c17 701 my $num = ord substr($md5, $ch, 1);
702
703 my $ref_loc = $tag->{offset} + ($num * $self->{long_size});
704 $tag = $self->index_lookup( $obj, $tag, $num );
705
706 if (!$tag) {
d5d7c51d 707 return if ! $args->{create};
d0b74c17 708
d5d7c51d 709 my $fh = $obj->_fh;
710 seek($fh, $ref_loc + $obj->_root->{file_offset}, SEEK_SET);
711 print( $fh pack($self->{long_pack}, $obj->_root->{end}) );
d0b74c17 712
d5d7c51d 713 $tag = $self->create_tag(
714 $obj, $obj->_root->{end},
715 SIG_BLIST,
716 chr(0) x $self->{bucket_list_size},
717 );
718
719 $tag->{ref_loc} = $ref_loc;
720 $tag->{ch} = $ch;
721
722 last;
d0b74c17 723 }
724
725 $tag->{ch} = $ch;
726 $tag->{ref_loc} = $ref_loc;
727
beac1dff 728 $ch++;
729 }
d0b74c17 730
beac1dff 731 return $tag;
6736c116 732}
733
d0b74c17 734sub index_lookup {
735 ##
736 # Given index tag, lookup single entry in index and return .
737 ##
738 my $self = shift;
739 my ($obj, $tag, $index) = @_;
740
741 my $location = unpack(
742 $self->{long_pack},
743 substr(
744 $tag->{content},
745 $index * $self->{long_size},
746 $self->{long_size},
747 ),
748 );
749
750 if (!$location) { return; }
751
752 return $self->load_tag( $obj, $location );
753}
754
6736c116 755sub traverse_index {
beac1dff 756 ##
757 # Scan index and recursively step into deeper levels, looking for next key.
758 ##
6736c116 759 my $self = shift;
760 my ($obj, $offset, $ch, $force_return_next) = @_;
d0b74c17 761
beac1dff 762 my $tag = $self->load_tag($obj, $offset );
6736c116 763
764 my $fh = $obj->_fh;
d0b74c17 765
8db25060 766 if ($tag->{signature} ne SIG_BLIST) {
beac1dff 767 my $content = $tag->{content};
e5fc7e69 768 my $start = $obj->{return_next} ? 0 : ord(substr($obj->{prev_md5}, $ch, 1));
d0b74c17 769
d5d7c51d 770 for (my $idx = $start; $idx < (2**8); $idx++) {
e5fc7e69 771 my $subloc = unpack(
772 $self->{long_pack},
e06824f8 773 substr(
774 $content,
775 $idx * $self->{long_size},
776 $self->{long_size},
777 ),
e5fc7e69 778 );
779
beac1dff 780 if ($subloc) {
e5fc7e69 781 my $result = $self->traverse_index(
782 $obj, $subloc, $ch + 1, $force_return_next,
783 );
784
beac1dff 785 if (defined($result)) { return $result; }
786 }
787 } # index loop
d0b74c17 788
beac1dff 789 $obj->{return_next} = 1;
790 } # tag is an index
d0b74c17 791
e5fc7e69 792 else {
beac1dff 793 my $keys = $tag->{content};
794 if ($force_return_next) { $obj->{return_next} = 1; }
d0b74c17 795
beac1dff 796 ##
797 # Iterate through buckets, looking for a key match
798 ##
8db25060 799 for (my $i = 0; $i < $self->{max_buckets}; $i++) {
9cec1360 800 my ($key, $subloc) = $self->_get_key_subloc( $keys, $i );
d0b74c17 801
8db25060 802 # End of bucket list -- return to outer loop
beac1dff 803 if (!$subloc) {
beac1dff 804 $obj->{return_next} = 1;
805 last;
806 }
8db25060 807 # Located previous key -- return next one found
beac1dff 808 elsif ($key eq $obj->{prev_md5}) {
beac1dff 809 $obj->{return_next} = 1;
810 next;
811 }
8db25060 812 # Seek to bucket location and skip over signature
beac1dff 813 elsif ($obj->{return_next}) {
8db25060 814 seek($fh, $subloc + $obj->_root->{file_offset}, SEEK_SET);
d0b74c17 815
beac1dff 816 # Skip over value to get to plain key
8db25060 817 my $sig;
818 read( $fh, $sig, SIG_SIZE );
819
beac1dff 820 my $size;
e5fc7e69 821 read( $fh, $size, $self->{data_size});
822 $size = unpack($self->{data_pack}, $size);
beac1dff 823 if ($size) { seek($fh, $size, SEEK_CUR); }
d0b74c17 824
beac1dff 825 # Read in plain key and return as scalar
beac1dff 826 my $plain_key;
e5fc7e69 827 read( $fh, $size, $self->{data_size});
828 $size = unpack($self->{data_pack}, $size);
beac1dff 829 if ($size) { read( $fh, $plain_key, $size); }
d0b74c17 830
beac1dff 831 return $plain_key;
832 }
8db25060 833 }
d0b74c17 834
beac1dff 835 $obj->{return_next} = 1;
836 } # tag is a bucket list
d0b74c17 837
beac1dff 838 return;
6736c116 839}
840
841sub get_next_key {
beac1dff 842 ##
843 # Locate next key, given digested previous one
844 ##
6736c116 845 my $self = shift;
846 my ($obj) = @_;
d0b74c17 847
beac1dff 848 $obj->{prev_md5} = $_[1] ? $_[1] : undef;
849 $obj->{return_next} = 0;
d0b74c17 850
beac1dff 851 ##
852 # If the previous key was not specifed, start at the top and
853 # return the first one found.
854 ##
855 if (!$obj->{prev_md5}) {
856 $obj->{prev_md5} = chr(0) x $self->{hash_size};
857 $obj->{return_next} = 1;
858 }
d0b74c17 859
beac1dff 860 return $self->traverse_index( $obj, $obj->_base_offset, 0 );
6736c116 861}
862
75be6413 863# Utilities
864
9cec1360 865sub _get_key_subloc {
75be6413 866 my $self = shift;
867 my ($keys, $idx) = @_;
868
9cec1360 869 my ($key, $subloc) = unpack(
870 "a$self->{hash_size} $self->{long_pack}",
75be6413 871 substr(
872 $keys,
9cec1360 873 ($idx * $self->{bucket_size}),
874 $self->{bucket_size},
75be6413 875 ),
876 );
877
9cec1360 878 return ($key, $subloc);
75be6413 879}
880
d608b06e 881sub _find_in_buckets {
882 my $self = shift;
883 my ($tag, $md5) = @_;
884
885 BUCKET:
886 for ( my $i = 0; $i < $self->{max_buckets}; $i++ ) {
887 my ($key, $subloc) = $self->_get_key_subloc( $tag->{content}, $i );
888
889 return ($subloc, $i * $self->{bucket_size}) unless $subloc;
890
891 next BUCKET if $key ne $md5;
892
893 return ($subloc, $i * $self->{bucket_size});
894 }
895
896 return;
897}
898
994ccd8e 899sub _request_space {
900 my $self = shift;
901 my ($obj, $size) = @_;
902
903 my $loc = $obj->_root->{end};
904
905 return $loc;
906}
907
908sub _release_space {
909 my $self = shift;
910 my ($obj, $size, $loc) = @_;
911
912 return;
913}
914
a20d9a3f 9151;
916__END__
d5d7c51d 917
918# This will be added in later, after more refactoring is done. This is an early
919# attempt at refactoring on the physical level instead of the virtual level.
920sub _read_at {
921 my $self = shift;
922 my ($obj, $spot, $amount, $unpack) = @_;
923
924 my $fh = $obj->_fh;
925 seek( $fh, $spot + $obj->_root->{file_offset}, SEEK_SET );
926
927 my $buffer;
928 my $bytes_read = read( $fh, $buffer, $amount );
929
930 if ( $unpack ) {
931 $buffer = unpack( $unpack, $buffer );
932 }
933
934 if ( wantarray ) {
935 return ($buffer, $bytes_read);
936 }
937 else {
938 return $buffer;
939 }
940}