Added unflocks to all tests so that the tests run on OSX
[dbsrgits/DBM-Deep.git] / lib / DBM / Deep / Engine.pm
CommitLineData
a20d9a3f 1package DBM::Deep::Engine;
2
3use strict;
4
5use Fcntl qw( :DEFAULT :flock :seek );
6
8db25060 7##
8# Setup file and tag signatures. These should never change.
9##
10sub SIG_FILE () { 'DPDB' }
11sub SIG_INTERNAL () { 'i' }
12sub SIG_HASH () { 'H' }
13sub SIG_ARRAY () { 'A' }
8db25060 14sub SIG_NULL () { 'N' }
15sub SIG_DATA () { 'D' }
16sub SIG_INDEX () { 'I' }
17sub SIG_BLIST () { 'B' }
7b1e1aa1 18sub SIG_FREE () { 'F' }
8db25060 19sub SIG_SIZE () { 1 }
20
612969fb 21sub precalc_sizes {
beac1dff 22 ##
23 # Precalculate index, bucket and bucket list sizes
24 ##
251dfd0e 25 my $self = shift;
1bf65be7 26
251dfd0e 27 $self->{index_size} = (2**8) * $self->{long_size};
6ed2f3df 28 $self->{bucket_size} = $self->{hash_size} + $self->{long_size} * 2;
251dfd0e 29 $self->{bucket_list_size} = $self->{max_buckets} * $self->{bucket_size};
1bf65be7 30
251dfd0e 31 return 1;
1bf65be7 32}
33
34sub set_pack {
beac1dff 35 ##
36 # Set pack/unpack modes (see file header for more)
37 ##
251dfd0e 38 my $self = shift;
1bf65be7 39 my ($long_s, $long_p, $data_s, $data_p) = @_;
40
81d16922 41 ##
75be6413 42 # Set to 4 and 'N' for 32-bit offset tags (default). Theoretical limit of 4
43 # GB per file.
beac1dff 44 # (Perl must be compiled with largefile support for files > 2 GB)
81d16922 45 #
46 # Set to 8 and 'Q' for 64-bit offsets. Theoretical limit of 16 XB per file.
beac1dff 47 # (Perl must be compiled with largefile and 64-bit long support)
81d16922 48 ##
251dfd0e 49 $self->{long_size} = $long_s ? $long_s : 4;
50 $self->{long_pack} = $long_p ? $long_p : 'N';
1bf65be7 51
81d16922 52 ##
75be6413 53 # Set to 4 and 'N' for 32-bit data length prefixes. Limit of 4 GB for each
d5d7c51d 54 # key/value. Upgrading this is possible (see above) but probably not
55 # necessary. If you need more than 4 GB for a single key or value, this
56 # module is really not for you :-)
81d16922 57 ##
251dfd0e 58 $self->{data_size} = $data_s ? $data_s : 4;
59 $self->{data_pack} = $data_p ? $data_p : 'N';
1bf65be7 60
beac1dff 61 return $self->precalc_sizes();
1bf65be7 62}
63
64sub set_digest {
beac1dff 65 ##
66 # Set key digest function (default is MD5)
67 ##
251dfd0e 68 my $self = shift;
1bf65be7 69 my ($digest_func, $hash_size) = @_;
70
d0b74c17 71 $self->{digest} = $digest_func ? $digest_func : \&Digest::MD5::md5;
251dfd0e 72 $self->{hash_size} = $hash_size ? $hash_size : 16;
612969fb 73
beac1dff 74 return $self->precalc_sizes();
612969fb 75}
76
77sub new {
78 my $class = shift;
79 my ($args) = @_;
80
81 my $self = bless {
82 long_size => 4,
83 long_pack => 'N',
84 data_size => 4,
85 data_pack => 'N',
251dfd0e 86
612969fb 87 digest => \&Digest::MD5::md5,
88 hash_size => 16,
251dfd0e 89
81d16922 90 ##
d5d7c51d 91 # Maximum number of buckets per list before another level of indexing is
92 # done.
93 # Increase this value for slightly greater speed, but larger database
94 # files. DO NOT decrease this value below 16, due to risk of recursive
95 # reindex overrun.
81d16922 96 ##
612969fb 97 max_buckets => 16,
98 }, $class;
99
251dfd0e 100 $self->precalc_sizes;
612969fb 101
102 return $self;
1bf65be7 103}
104
70b55428 105sub setup_fh {
106 my $self = shift;
107 my ($obj) = @_;
108
109 $self->open( $obj ) if !defined $obj->_fh;
110
6fde4ed2 111 my $fh = $obj->_fh;
58910373 112 print "1\n";
6fde4ed2 113 flock $fh, LOCK_EX;
58910373 114 print "2\n";
118ba343 115
6fde4ed2 116 unless ( $obj->{base_offset} ) {
118ba343 117 seek($fh, 0 + $obj->_root->{file_offset}, SEEK_SET);
118 my $signature;
119 my $bytes_read = read( $fh, $signature, length(SIG_FILE));
120
121 ##
122 # File is empty -- write signature and master index
123 ##
124 if (!$bytes_read) {
c9ec091a 125 my $loc = $self->_request_space( $obj, length( SIG_FILE ) );
126 seek($fh, $loc + $obj->_root->{file_offset}, SEEK_SET);
118ba343 127 print( $fh SIG_FILE);
128
c9ec091a 129 $obj->{base_offset} = $self->_request_space(
16d1ad9b 130 $obj, $self->tag_size( $self->{index_size} ),
c9ec091a 131 );
118ba343 132
9e4f83a0 133 $self->write_tag(
c9ec091a 134 $obj, $obj->_base_offset, $obj->_type,
f37c15ab 135 chr(0)x$self->{index_size},
118ba343 136 );
137
138 # Flush the filehandle
139 my $old_fh = select $fh;
140 my $old_af = $|; $| = 1; $| = $old_af;
141 select $old_fh;
142 }
143 else {
144 $obj->{base_offset} = $bytes_read;
145
146 ##
147 # Check signature was valid
148 ##
149 unless ($signature eq SIG_FILE) {
150 $self->close_fh( $obj );
151 $obj->_throw_error("Signature not found -- file is not a Deep DB");
152 }
153
154 ##
155 # Get our type from master index signature
156 ##
157 my $tag = $self->load_tag($obj, $obj->_base_offset)
158 or $obj->_throw_error("Corrupted file, no master index record");
159
160 unless ($obj->{type} eq $tag->{signature}) {
161 $obj->_throw_error("File type mismatch");
162 }
163 }
118ba343 164 }
e06824f8 165
673464d9 166 #XXX We have to make sure we don't mess up when autoflush isn't turned on
70b55428 167 unless ( $obj->_root->{inode} ) {
168 my @stats = stat($obj->_fh);
169 $obj->_root->{inode} = $stats[1];
170 $obj->_root->{end} = $stats[7];
171 }
172
6fde4ed2 173 flock $fh, LOCK_UN;
174
70b55428 175 return 1;
176}
177
a20d9a3f 178sub open {
20f7b20c 179 ##
180 # Open a fh to the database, create if nonexistent.
181 # Make sure file signature matches DBM::Deep spec.
182 ##
a20d9a3f 183 my $self = shift;
70b55428 184 my ($obj) = @_;
a20d9a3f 185
673464d9 186 # Theoretically, adding O_BINARY should remove the need for the binmode
187 # Of course, testing it is going to be ... interesting.
188 my $flags = O_RDWR | O_CREAT | O_BINARY;
a20d9a3f 189
673464d9 190 my $fh;
d5d7c51d 191 my $filename = $obj->_root->{file};
192 sysopen( $fh, $filename, $flags )
193 or $obj->_throw_error("Cannot sysopen file '$filename': $!");
673464d9 194 $obj->_root->{fh} = $fh;
a20d9a3f 195
196 #XXX Can we remove this by using the right sysopen() flags?
197 # Maybe ... q.v. above
198 binmode $fh; # for win32
199
cd59cad8 200 if ($obj->_root->{autoflush}) {
a20d9a3f 201 my $old = select $fh;
202 $|=1;
203 select $old;
204 }
20f7b20c 205
a20d9a3f 206 return 1;
207}
208
3d1b8be9 209sub close_fh {
cd59cad8 210 my $self = shift;
a21f2d90 211 my ($obj) = @_;
cd59cad8 212
213 if ( my $fh = $obj->_root->{fh} ) {
214 close $fh;
215 }
216 $obj->_root->{fh} = undef;
217
218 return 1;
219}
220
16d1ad9b 221sub tag_size {
222 my $self = shift;
223 my ($size) = @_;
224 return SIG_SIZE + $self->{data_size} + $size;
225}
226
9e4f83a0 227sub write_tag {
20f7b20c 228 ##
229 # Given offset, signature and content, create tag and write to disk
230 ##
d4b1166e 231 my $self = shift;
20f7b20c 232 my ($obj, $offset, $sig, $content) = @_;
f37c15ab 233 my $size = length( $content );
20f7b20c 234
d4b1166e 235 my $fh = $obj->_fh;
236
f37c15ab 237 if ( defined $offset ) {
238 seek($fh, $offset + $obj->_root->{file_offset}, SEEK_SET);
239 }
240
251dfd0e 241 print( $fh $sig . pack($self->{data_pack}, $size) . $content );
20f7b20c 242
f37c15ab 243 return unless defined $offset;
244
20f7b20c 245 return {
246 signature => $sig,
247 size => $size,
8db25060 248 offset => $offset + SIG_SIZE + $self->{data_size},
20f7b20c 249 content => $content
250 };
d4b1166e 251}
252
253sub load_tag {
20f7b20c 254 ##
255 # Given offset, load single tag and return signature, size and data
256 ##
d4b1166e 257 my $self = shift;
20f7b20c 258 my ($obj, $offset) = @_;
259
e06824f8 260# print join(':',map{$_||''}caller(1)), $/;
261
d4b1166e 262 my $fh = $obj->_fh;
263
20f7b20c 264 seek($fh, $offset + $obj->_root->{file_offset}, SEEK_SET);
e5fc7e69 265
75be6413 266 #XXX I'm not sure this check will work if autoflush isn't enabled ...
e5fc7e69 267 return if eof $fh;
20f7b20c 268
d4b1166e 269 my $b;
8db25060 270 read( $fh, $b, SIG_SIZE + $self->{data_size} );
251dfd0e 271 my ($sig, $size) = unpack( "A $self->{data_pack}", $b );
20f7b20c 272
273 my $buffer;
274 read( $fh, $buffer, $size);
275
276 return {
277 signature => $sig,
278 size => $size,
8db25060 279 offset => $offset + SIG_SIZE + $self->{data_size},
20f7b20c 280 content => $buffer
281 };
d4b1166e 282}
283
29b01632 284sub _length_needed {
285 my $self = shift;
f37c15ab 286 my ($obj, $value, $key) = @_;
29b01632 287
288 my $is_dbm_deep = eval {
289 local $SIG{'__DIE__'};
290 $value->isa( 'DBM::Deep' );
291 };
292
f37c15ab 293 my $len = SIG_SIZE + $self->{data_size}
294 + $self->{data_size} + length( $key );
29b01632 295
f37c15ab 296 if ( $is_dbm_deep && $value->_root eq $obj->_root ) {
297 return $len + $self->{long_size};
29b01632 298 }
299
300 my $r = Scalar::Util::reftype( $value ) || '';
9a187d8c 301 if ( $obj->_root->{autobless} ) {
302 # This is for the bit saying whether or not this thing is blessed.
303 $len += 1;
304 }
305
29b01632 306 unless ( $r eq 'HASH' || $r eq 'ARRAY' ) {
f37c15ab 307 if ( defined $value ) {
308 $len += length( $value );
309 }
310 return $len;
29b01632 311 }
312
f37c15ab 313 $len += $self->{index_size};
29b01632 314
315 # if autobless is enabled, must also take into consideration
f37c15ab 316 # the class name as it is stored after the key.
29b01632 317 if ( $obj->_root->{autobless} ) {
318 my $value_class = Scalar::Util::blessed($value);
f37c15ab 319 if ( defined $value_class && !$is_dbm_deep ) {
320 $len += $self->{data_size} + length($value_class);
29b01632 321 }
322 }
323
f37c15ab 324 return $len;
29b01632 325}
326
20f7b20c 327sub add_bucket {
328 ##
329 # Adds one key/value pair to bucket list, given offset, MD5 digest of key,
330 # plain (undigested) key and value.
331 ##
d4b1166e 332 my $self = shift;
20f7b20c 333 my ($obj, $tag, $md5, $plain_key, $value) = @_;
75be6413 334
eea0d863 335 # This verifies that only supported values will be stored.
336 {
337 my $r = Scalar::Util::reftype( $value );
338 last if !defined $r;
339
340 last if $r eq 'HASH';
341 last if $r eq 'ARRAY';
342
343 $obj->_throw_error(
344 "Storage of variables of type '$r' is not supported."
345 );
346 }
347
20f7b20c 348 my $location = 0;
349 my $result = 2;
350
351 my $root = $obj->_root;
f37c15ab 352 my $fh = $obj->_fh;
20f7b20c 353
f37c15ab 354 my $actual_length = $self->_length_needed( $obj, $value, $plain_key );
20f7b20c 355
9a187d8c 356 my ($subloc, $offset, $size) = $self->_find_in_buckets( $tag, $md5 );
75be6413 357
f9c33187 358# $self->_release_space( $obj, $size, $subloc );
386bab6c 359 # Updating a known md5
f9c33187 360#XXX This needs updating to use _release_space
386bab6c 361 if ( $subloc ) {
362 $result = 1;
20f7b20c 363
386bab6c 364 if ($actual_length <= $size) {
365 $location = $subloc;
20f7b20c 366 }
75be6413 367 else {
f37c15ab 368 $location = $self->_request_space( $obj, $actual_length );
386bab6c 369 seek(
370 $fh,
9a187d8c 371 $tag->{offset} + $offset
372 + $self->{hash_size} + $root->{file_offset},
386bab6c 373 SEEK_SET,
374 );
9a187d8c 375 print( $fh pack($self->{long_pack}, $location ) );
376 print( $fh pack($self->{long_pack}, $actual_length ) );
75be6413 377 }
75be6413 378 }
386bab6c 379 # Adding a new md5
380 elsif ( defined $offset ) {
f37c15ab 381 $location = $self->_request_space( $obj, $actual_length );
386bab6c 382
383 seek( $fh, $tag->{offset} + $offset + $root->{file_offset}, SEEK_SET );
9a187d8c 384 print( $fh $md5 . pack($self->{long_pack}, $location ) );
385 print( $fh pack($self->{long_pack}, $actual_length ) );
386bab6c 386 }
387 # If bucket didn't fit into list, split into a new index level
f9c33187 388 # split_index() will do the _request_space() call
386bab6c 389 else {
f9c33187 390 $location = $self->split_index( $obj, $md5, $tag );
386bab6c 391 }
20f7b20c 392
d5d7c51d 393 $self->write_value( $obj, $location, $plain_key, $value );
394
395 return $result;
396}
397
398sub write_value {
399 my $self = shift;
400 my ($obj, $location, $key, $value) = @_;
401
402 my $fh = $obj->_fh;
403 my $root = $obj->_root;
404
405 my $is_dbm_deep = eval {
406 local $SIG{'__DIE__'};
407 $value->isa( 'DBM::Deep' );
408 };
409
f9c33187 410 my $is_internal_ref = $is_dbm_deep && ($value->_root eq $root);
d5d7c51d 411
412 seek($fh, $location + $root->{file_offset}, SEEK_SET);
413
20f7b20c 414 ##
d5d7c51d 415 # Write signature based on content type, set content length and write
416 # actual value.
20f7b20c 417 ##
d5d7c51d 418 my $r = Scalar::Util::reftype($value) || '';
f9c33187 419 if ( $is_internal_ref ) {
9e4f83a0 420 $self->write_tag( $obj, undef, SIG_INTERNAL,pack($self->{long_pack}, $value->_base_offset) );
f37c15ab 421 }
422 elsif ($r eq 'HASH') {
9e4f83a0 423 $self->write_tag( $obj, undef, SIG_HASH, chr(0)x$self->{index_size} );
f37c15ab 424 }
425 elsif ($r eq 'ARRAY') {
9e4f83a0 426 $self->write_tag( $obj, undef, SIG_ARRAY, chr(0)x$self->{index_size} );
f37c15ab 427 }
428 elsif (!defined($value)) {
9e4f83a0 429 $self->write_tag( $obj, undef, SIG_NULL, '' );
d5d7c51d 430 }
431 else {
9e4f83a0 432 $self->write_tag( $obj, undef, SIG_DATA, $value );
d5d7c51d 433 }
20f7b20c 434
d5d7c51d 435 ##
436 # Plain key is stored AFTER value, as keys are typically fetched less often.
437 ##
438 print( $fh pack($self->{data_pack}, length($key)) . $key );
20f7b20c 439
9a187d8c 440 # Internal references don't care about autobless
f9c33187 441 return 1 if $is_internal_ref;
9a187d8c 442
d5d7c51d 443 ##
444 # If value is blessed, preserve class name
445 ##
446 if ( $root->{autobless} ) {
447 my $value_class = Scalar::Util::blessed($value);
f37c15ab 448 if ( defined $value_class && !$is_dbm_deep ) {
d5d7c51d 449 print( $fh chr(1) );
450 print( $fh pack($self->{data_pack}, length($value_class)) . $value_class );
20f7b20c 451 }
d5d7c51d 452 else {
453 print( $fh chr(0) );
20f7b20c 454 }
d5d7c51d 455 }
20f7b20c 456
d5d7c51d 457 ##
d5d7c51d 458 # If content is a hash or array, create new child DBM::Deep object and
459 # pass each key or element to it.
460 ##
f9c33187 461 if ( !$is_internal_ref ) {
d5d7c51d 462 if ($r eq 'HASH') {
463 my $branch = DBM::Deep->new(
464 type => DBM::Deep->TYPE_HASH,
465 base_offset => $location,
466 root => $root,
467 );
468 foreach my $key (keys %{$value}) {
469 $branch->STORE( $key, $value->{$key} );
20f7b20c 470 }
d5d7c51d 471 }
472 elsif ($r eq 'ARRAY') {
473 my $branch = DBM::Deep->new(
474 type => DBM::Deep->TYPE_ARRAY,
475 base_offset => $location,
476 root => $root,
477 );
478 my $index = 0;
479 foreach my $element (@{$value}) {
480 $branch->STORE( $index, $element );
481 $index++;
20f7b20c 482 }
483 }
20f7b20c 484 }
d4b1166e 485
d5d7c51d 486 return 1;
d4b1166e 487}
488
75be6413 489sub split_index {
490 my $self = shift;
491 my ($obj, $md5, $tag) = @_;
492
493 my $fh = $obj->_fh;
494 my $root = $obj->_root;
16d1ad9b 495
496 my $loc = $self->_request_space(
497 $obj, $self->tag_size( $self->{index_size} ),
498 );
499
7b1e1aa1 500 seek($fh, $tag->{ref_loc} + $root->{file_offset}, SEEK_SET);
16d1ad9b 501 print( $fh pack($self->{long_pack}, $loc) );
75be6413 502
9e4f83a0 503 my $index_tag = $self->write_tag(
16d1ad9b 504 $obj, $loc, SIG_INDEX,
f37c15ab 505 chr(0)x$self->{index_size},
75be6413 506 );
507
f9c33187 508 my $newtag_loc = $self->_request_space(
509 $obj, $self->tag_size( $self->{bucket_list_size} ),
510 );
75be6413 511
7b1e1aa1 512 my $keys = $tag->{content}
f9c33187 513 . $md5 . pack($self->{long_pack}, $newtag_loc)
514 . pack($self->{long_pack}, 0);
75be6413 515
f9c33187 516 my @newloc = ();
75be6413 517 BUCKET:
518 for (my $i = 0; $i <= $self->{max_buckets}; $i++) {
9a187d8c 519 my ($key, $old_subloc, $size) = $self->_get_key_subloc( $keys, $i );
75be6413 520
f9c33187 521 die "[INTERNAL ERROR]: No key in split_index()\n" unless $key;
522 die "[INTERNAL ERROR]: No subloc in split_index()\n" unless $old_subloc;
75be6413 523
75be6413 524 my $num = ord(substr($key, $tag->{ch} + 1, 1));
525
f9c33187 526 if ($newloc[$num]) {
527 seek($fh, $newloc[$num] + $root->{file_offset}, SEEK_SET);
75be6413 528 my $subkeys;
529 read( $fh, $subkeys, $self->{bucket_list_size});
530
f9c33187 531 # This is looking for the first empty spot
7b1e1aa1 532 my ($subloc, $offset, $size) = $self->_find_in_buckets(
f9c33187 533 { content => $subkeys }, '',
7b1e1aa1 534 );
75be6413 535
f9c33187 536 seek($fh, $newloc[$num] + $offset + $root->{file_offset}, SEEK_SET);
537 print( $fh $key . pack($self->{long_pack}, $old_subloc) );
7b1e1aa1 538
539 next;
75be6413 540 }
75be6413 541
7b1e1aa1 542 seek($fh, $index_tag->{offset} + ($num * $self->{long_size}) + $root->{file_offset}, SEEK_SET);
2603d86e 543
7b1e1aa1 544 my $loc = $self->_request_space(
545 $obj, $self->tag_size( $self->{bucket_list_size} ),
546 );
2603d86e 547
7b1e1aa1 548 print( $fh pack($self->{long_pack}, $loc) );
75be6413 549
7b1e1aa1 550 my $blist_tag = $self->write_tag(
551 $obj, $loc, SIG_BLIST,
552 chr(0)x$self->{bucket_list_size},
553 );
554
555 seek($fh, $blist_tag->{offset} + $root->{file_offset}, SEEK_SET);
f9c33187 556 print( $fh $key . pack($self->{long_pack}, $old_subloc) );
7b1e1aa1 557
f9c33187 558 $newloc[$num] = $blist_tag->{offset};
7b1e1aa1 559 }
560
561 $self->_release_space(
f9c33187 562 $obj, $self->tag_size( $self->{bucket_list_size} ),
7b1e1aa1 563 $tag->{offset} - SIG_SIZE - $self->{data_size},
564 );
75be6413 565
f9c33187 566 return $newtag_loc;
75be6413 567}
568
8db25060 569sub read_from_loc {
570 my $self = shift;
571 my ($obj, $subloc) = @_;
572
573 my $fh = $obj->_fh;
574
575 ##
576 # Found match -- seek to offset and read signature
577 ##
578 my $signature;
579 seek($fh, $subloc + $obj->_root->{file_offset}, SEEK_SET);
580 read( $fh, $signature, SIG_SIZE);
581
582 ##
583 # If value is a hash or array, return new DBM::Deep object with correct offset
584 ##
585 if (($signature eq SIG_HASH) || ($signature eq SIG_ARRAY)) {
586 my $obj = DBM::Deep->new(
587 type => $signature,
588 base_offset => $subloc,
589 root => $obj->_root,
590 );
591
592 if ($obj->_root->{autobless}) {
593 ##
594 # Skip over value and plain key to see if object needs
595 # to be re-blessed
596 ##
597 seek($fh, $self->{data_size} + $self->{index_size}, SEEK_CUR);
598
599 my $size;
600 read( $fh, $size, $self->{data_size}); $size = unpack($self->{data_pack}, $size);
601 if ($size) { seek($fh, $size, SEEK_CUR); }
602
603 my $bless_bit;
604 read( $fh, $bless_bit, 1);
605 if (ord($bless_bit)) {
606 ##
607 # Yes, object needs to be re-blessed
608 ##
609 my $class_name;
610 read( $fh, $size, $self->{data_size}); $size = unpack($self->{data_pack}, $size);
611 if ($size) { read( $fh, $class_name, $size); }
612 if ($class_name) { $obj = bless( $obj, $class_name ); }
613 }
614 }
615
616 return $obj;
617 }
618 elsif ( $signature eq SIG_INTERNAL ) {
619 my $size;
620 read( $fh, $size, $self->{data_size});
621 $size = unpack($self->{data_pack}, $size);
622
623 if ( $size ) {
624 my $new_loc;
625 read( $fh, $new_loc, $size );
626 $new_loc = unpack( $self->{long_pack}, $new_loc );
627
628 return $self->read_from_loc( $obj, $new_loc );
629 }
630 else {
631 return;
632 }
633 }
634 ##
635 # Otherwise return actual value
636 ##
637 elsif ($signature eq SIG_DATA) {
638 my $size;
639 read( $fh, $size, $self->{data_size});
640 $size = unpack($self->{data_pack}, $size);
641
642 my $value = '';
643 if ($size) { read( $fh, $value, $size); }
644 return $value;
645 }
646
647 ##
648 # Key exists, but content is null
649 ##
650 return;
651}
652
9020ee8c 653sub get_bucket_value {
beac1dff 654 ##
655 # Fetch single value given tag and MD5 digested key.
656 ##
657 my $self = shift;
658 my ($obj, $tag, $md5) = @_;
9020ee8c 659
9a187d8c 660 my ($subloc, $offset, $size) = $self->_find_in_buckets( $tag, $md5 );
386bab6c 661 if ( $subloc ) {
8db25060 662 return $self->read_from_loc( $obj, $subloc );
386bab6c 663 }
beac1dff 664 return;
9020ee8c 665}
ab0e4957 666
667sub delete_bucket {
beac1dff 668 ##
669 # Delete single key/value pair given tag and MD5 digested key.
670 ##
671 my $self = shift;
672 my ($obj, $tag, $md5) = @_;
ab0e4957 673
9a187d8c 674 my ($subloc, $offset, $size) = $self->_find_in_buckets( $tag, $md5 );
f9c33187 675#XXX This needs _release_space()
386bab6c 676 if ( $subloc ) {
677 my $fh = $obj->_fh;
678 seek($fh, $tag->{offset} + $offset + $obj->_root->{file_offset}, SEEK_SET);
679 print( $fh substr($tag->{content}, $offset + $self->{bucket_size} ) );
251dfd0e 680 print( $fh chr(0) x $self->{bucket_size} );
d0b74c17 681
ab0e4957 682 return 1;
386bab6c 683 }
beac1dff 684 return;
ab0e4957 685}
686
912d50b1 687sub bucket_exists {
beac1dff 688 ##
689 # Check existence of single key given tag and MD5 digested key.
690 ##
691 my $self = shift;
692 my ($obj, $tag, $md5) = @_;
912d50b1 693
9a187d8c 694 my ($subloc, $offset, $size) = $self->_find_in_buckets( $tag, $md5 );
d5d7c51d 695 return $subloc && 1;
912d50b1 696}
697
6736c116 698sub find_bucket_list {
beac1dff 699 ##
700 # Locate offset for bucket list, given digested key
701 ##
702 my $self = shift;
d0b74c17 703 my ($obj, $md5, $args) = @_;
704 $args = {} unless $args;
705
beac1dff 706 ##
707 # Locate offset for bucket list using digest index system
708 ##
e5fc7e69 709 my $tag = $self->load_tag($obj, $obj->_base_offset)
d5d7c51d 710 or $obj->_throw_error( "INTERNAL ERROR - Cannot find tag" );
d0b74c17 711
e5fc7e69 712 my $ch = 0;
8db25060 713 while ($tag->{signature} ne SIG_BLIST) {
d0b74c17 714 my $num = ord substr($md5, $ch, 1);
715
716 my $ref_loc = $tag->{offset} + ($num * $self->{long_size});
717 $tag = $self->index_lookup( $obj, $tag, $num );
718
719 if (!$tag) {
29b01632 720 return if !$args->{create};
d0b74c17 721
16d1ad9b 722 my $loc = $self->_request_space(
723 $obj, $self->tag_size( $self->{bucket_list_size} ),
724 );
725
7b1e1aa1 726 my $fh = $obj->_fh;
727 seek($fh, $ref_loc + $obj->_root->{file_offset}, SEEK_SET);
16d1ad9b 728 print( $fh pack($self->{long_pack}, $loc) );
d0b74c17 729
9e4f83a0 730 $tag = $self->write_tag(
16d1ad9b 731 $obj, $loc, SIG_BLIST,
f37c15ab 732 chr(0)x$self->{bucket_list_size},
d5d7c51d 733 );
734
735 $tag->{ref_loc} = $ref_loc;
736 $tag->{ch} = $ch;
737
738 last;
d0b74c17 739 }
740
16d1ad9b 741 $tag->{ch} = $ch++;
d0b74c17 742 $tag->{ref_loc} = $ref_loc;
beac1dff 743 }
d0b74c17 744
beac1dff 745 return $tag;
6736c116 746}
747
d0b74c17 748sub index_lookup {
749 ##
750 # Given index tag, lookup single entry in index and return .
751 ##
752 my $self = shift;
753 my ($obj, $tag, $index) = @_;
754
755 my $location = unpack(
756 $self->{long_pack},
757 substr(
758 $tag->{content},
759 $index * $self->{long_size},
760 $self->{long_size},
761 ),
762 );
763
764 if (!$location) { return; }
765
766 return $self->load_tag( $obj, $location );
767}
768
6736c116 769sub traverse_index {
beac1dff 770 ##
771 # Scan index and recursively step into deeper levels, looking for next key.
772 ##
6736c116 773 my $self = shift;
774 my ($obj, $offset, $ch, $force_return_next) = @_;
d0b74c17 775
beac1dff 776 my $tag = $self->load_tag($obj, $offset );
6736c116 777
778 my $fh = $obj->_fh;
d0b74c17 779
8db25060 780 if ($tag->{signature} ne SIG_BLIST) {
beac1dff 781 my $content = $tag->{content};
e5fc7e69 782 my $start = $obj->{return_next} ? 0 : ord(substr($obj->{prev_md5}, $ch, 1));
d0b74c17 783
d5d7c51d 784 for (my $idx = $start; $idx < (2**8); $idx++) {
e5fc7e69 785 my $subloc = unpack(
786 $self->{long_pack},
e06824f8 787 substr(
788 $content,
789 $idx * $self->{long_size},
790 $self->{long_size},
791 ),
e5fc7e69 792 );
793
beac1dff 794 if ($subloc) {
e5fc7e69 795 my $result = $self->traverse_index(
796 $obj, $subloc, $ch + 1, $force_return_next,
797 );
798
beac1dff 799 if (defined($result)) { return $result; }
800 }
801 } # index loop
d0b74c17 802
beac1dff 803 $obj->{return_next} = 1;
804 } # tag is an index
d0b74c17 805
e5fc7e69 806 else {
beac1dff 807 my $keys = $tag->{content};
808 if ($force_return_next) { $obj->{return_next} = 1; }
d0b74c17 809
beac1dff 810 ##
811 # Iterate through buckets, looking for a key match
812 ##
8db25060 813 for (my $i = 0; $i < $self->{max_buckets}; $i++) {
9cec1360 814 my ($key, $subloc) = $self->_get_key_subloc( $keys, $i );
d0b74c17 815
8db25060 816 # End of bucket list -- return to outer loop
beac1dff 817 if (!$subloc) {
beac1dff 818 $obj->{return_next} = 1;
819 last;
820 }
8db25060 821 # Located previous key -- return next one found
beac1dff 822 elsif ($key eq $obj->{prev_md5}) {
beac1dff 823 $obj->{return_next} = 1;
824 next;
825 }
8db25060 826 # Seek to bucket location and skip over signature
beac1dff 827 elsif ($obj->{return_next}) {
8db25060 828 seek($fh, $subloc + $obj->_root->{file_offset}, SEEK_SET);
d0b74c17 829
beac1dff 830 # Skip over value to get to plain key
8db25060 831 my $sig;
832 read( $fh, $sig, SIG_SIZE );
833
beac1dff 834 my $size;
e5fc7e69 835 read( $fh, $size, $self->{data_size});
836 $size = unpack($self->{data_pack}, $size);
beac1dff 837 if ($size) { seek($fh, $size, SEEK_CUR); }
d0b74c17 838
beac1dff 839 # Read in plain key and return as scalar
beac1dff 840 my $plain_key;
e5fc7e69 841 read( $fh, $size, $self->{data_size});
842 $size = unpack($self->{data_pack}, $size);
beac1dff 843 if ($size) { read( $fh, $plain_key, $size); }
d0b74c17 844
beac1dff 845 return $plain_key;
846 }
8db25060 847 }
d0b74c17 848
beac1dff 849 $obj->{return_next} = 1;
850 } # tag is a bucket list
d0b74c17 851
beac1dff 852 return;
6736c116 853}
854
855sub get_next_key {
beac1dff 856 ##
857 # Locate next key, given digested previous one
858 ##
6736c116 859 my $self = shift;
860 my ($obj) = @_;
d0b74c17 861
beac1dff 862 $obj->{prev_md5} = $_[1] ? $_[1] : undef;
863 $obj->{return_next} = 0;
d0b74c17 864
beac1dff 865 ##
866 # If the previous key was not specifed, start at the top and
867 # return the first one found.
868 ##
869 if (!$obj->{prev_md5}) {
870 $obj->{prev_md5} = chr(0) x $self->{hash_size};
871 $obj->{return_next} = 1;
872 }
d0b74c17 873
beac1dff 874 return $self->traverse_index( $obj, $obj->_base_offset, 0 );
6736c116 875}
876
75be6413 877# Utilities
878
9cec1360 879sub _get_key_subloc {
75be6413 880 my $self = shift;
881 my ($keys, $idx) = @_;
882
6ed2f3df 883 my ($key, $subloc, $size) = unpack(
884 "a$self->{hash_size} $self->{long_pack} $self->{long_pack}",
75be6413 885 substr(
886 $keys,
9cec1360 887 ($idx * $self->{bucket_size}),
888 $self->{bucket_size},
75be6413 889 ),
890 );
891
6ed2f3df 892 return ($key, $subloc, $size);
75be6413 893}
894
d608b06e 895sub _find_in_buckets {
896 my $self = shift;
897 my ($tag, $md5) = @_;
898
899 BUCKET:
900 for ( my $i = 0; $i < $self->{max_buckets}; $i++ ) {
9a187d8c 901 my ($key, $subloc, $size) = $self->_get_key_subloc(
902 $tag->{content}, $i,
903 );
d608b06e 904
9a187d8c 905 return ($subloc, $i * $self->{bucket_size}, $size) unless $subloc;
d608b06e 906
907 next BUCKET if $key ne $md5;
908
9a187d8c 909 return ($subloc, $i * $self->{bucket_size}, $size);
d608b06e 910 }
911
912 return;
913}
914
7b1e1aa1 915#sub _print_at {
916# my $self = shift;
917# my ($obj, $spot, $data) = @_;
918#
919# my $fh = $obj->_fh;
920# seek( $fh, $spot, SEEK_SET );
921# print( $fh $data );
922#
923# return;
924#}
925
994ccd8e 926sub _request_space {
927 my $self = shift;
928 my ($obj, $size) = @_;
929
930 my $loc = $obj->_root->{end};
c9ec091a 931 $obj->_root->{end} += $size;
994ccd8e 932
933 return $loc;
934}
935
936sub _release_space {
937 my $self = shift;
938 my ($obj, $size, $loc) = @_;
939
7b1e1aa1 940 my $next_loc = 0;
941
942 my $fh = $obj->_fh;
943 seek( $fh, $loc + $obj->_root->{file_offset}, SEEK_SET );
944 print( $fh SIG_FREE
945 . pack($self->{long_pack}, $size )
946 . pack($self->{long_pack}, $next_loc )
947 );
948
994ccd8e 949 return;
950}
951
a20d9a3f 9521;
953__END__
d5d7c51d 954
955# This will be added in later, after more refactoring is done. This is an early
956# attempt at refactoring on the physical level instead of the virtual level.
957sub _read_at {
958 my $self = shift;
959 my ($obj, $spot, $amount, $unpack) = @_;
960
961 my $fh = $obj->_fh;
962 seek( $fh, $spot + $obj->_root->{file_offset}, SEEK_SET );
963
964 my $buffer;
965 my $bytes_read = read( $fh, $buffer, $amount );
966
967 if ( $unpack ) {
968 $buffer = unpack( $unpack, $buffer );
969 }
970
971 if ( wantarray ) {
972 return ($buffer, $bytes_read);
973 }
974 else {
975 return $buffer;
976 }
977}