Commit | Line | Data |
a20d9a3f |
1 | package DBM::Deep::Engine; |
2 | |
2120a181 |
3 | use 5.006_000; |
460b1067 |
4 | |
a20d9a3f |
5 | use strict; |
065b45be |
6 | use warnings FATAL => 'all'; |
a20d9a3f |
7 | |
75a6a379 |
8 | # Never import symbols into our namespace. We are a class, not a library. |
9 | # -RobK, 2008-05-27 |
10 | use Scalar::Util (); |
d6d8e27e |
11 | |
12 | #use Data::Dumper (); |
75a6a379 |
13 | |
21838116 |
14 | # File-wide notes: |
2120a181 |
15 | # * Every method in here assumes that the storage has been appropriately |
c3aafc14 |
16 | # safeguarded. This can be anything from flock() to some sort of manual |
17 | # mutex. But, it's the caller's responsability to make sure that this has |
18 | # been done. |
21838116 |
19 | |
8db25060 |
20 | # Setup file and tag signatures. These should never change. |
8db25060 |
21 | sub SIG_FILE () { 'DPDB' } |
460b1067 |
22 | sub SIG_HEADER () { 'h' } |
8db25060 |
23 | sub SIG_HASH () { 'H' } |
24 | sub SIG_ARRAY () { 'A' } |
8db25060 |
25 | sub SIG_NULL () { 'N' } |
26 | sub SIG_DATA () { 'D' } |
27 | sub SIG_INDEX () { 'I' } |
28 | sub SIG_BLIST () { 'B' } |
7b1e1aa1 |
29 | sub SIG_FREE () { 'F' } |
8db25060 |
30 | sub SIG_SIZE () { 1 } |
e9b0b5f0 |
31 | |
065b45be |
32 | our $STALE_SIZE = 2; |
8db25060 |
33 | |
2120a181 |
34 | # Please refer to the pack() documentation for further information |
35 | my %StP = ( |
e9b0b5f0 |
36 | 1 => 'C', # Unsigned char value (no order needed as it's just one byte) |
2120a181 |
37 | 2 => 'n', # Unsigned short in "network" (big-endian) order |
38 | 4 => 'N', # Unsigned long in "network" (big-endian) order |
39 | 8 => 'Q', # Usigned quad (no order specified, presumably machine-dependent) |
40 | ); |
065b45be |
41 | sub StP { $StP{$_[1]} } |
83371fe3 |
42 | |
00d9bd0b |
43 | # Import these after the SIG_* definitions because those definitions are used |
44 | # in the headers of these classes. -RobK, 2008-06-20 |
45 | use DBM::Deep::Engine::Sector::BucketList; |
46 | use DBM::Deep::Engine::Sector::FileHeader; |
47 | use DBM::Deep::Engine::Sector::Index; |
48 | use DBM::Deep::Engine::Sector::Null; |
49 | use DBM::Deep::Engine::Sector::Reference; |
50 | use DBM::Deep::Engine::Sector::Scalar; |
51 | use DBM::Deep::Iterator; |
52 | |
c3aafc14 |
53 | ################################################################################ |
c3aafc14 |
54 | |
612969fb |
55 | sub new { |
56 | my $class = shift; |
57 | my ($args) = @_; |
58 | |
f1879fdc |
59 | $args->{storage} = DBM::Deep::File->new( $args ) |
60 | unless exists $args->{storage}; |
61 | |
612969fb |
62 | my $self = bless { |
2120a181 |
63 | byte_size => 4, |
64 | |
65 | digest => undef, |
66 | hash_size => 16, # In bytes |
67 | hash_chars => 256, # Number of chars the algorithm uses per byte |
612969fb |
68 | max_buckets => 16, |
e9b0b5f0 |
69 | num_txns => 1, # The HEAD |
2120a181 |
70 | trans_id => 0, # Default to the HEAD |
460b1067 |
71 | |
e9b0b5f0 |
72 | data_sector_size => 64, # Size in bytes of each data sector |
73 | |
2120a181 |
74 | entries => {}, # This is the list of entries for transactions |
83371fe3 |
75 | storage => undef, |
612969fb |
76 | }, $class; |
77 | |
e9b0b5f0 |
78 | # Never allow byte_size to be set directly. |
79 | delete $args->{byte_size}; |
e0098e7f |
80 | if ( defined $args->{pack_size} ) { |
81 | if ( lc $args->{pack_size} eq 'small' ) { |
2120a181 |
82 | $args->{byte_size} = 2; |
e0098e7f |
83 | } |
84 | elsif ( lc $args->{pack_size} eq 'medium' ) { |
2120a181 |
85 | $args->{byte_size} = 4; |
e0098e7f |
86 | } |
87 | elsif ( lc $args->{pack_size} eq 'large' ) { |
2120a181 |
88 | $args->{byte_size} = 8; |
e0098e7f |
89 | } |
90 | else { |
2120a181 |
91 | DBM::Deep->_throw_error( "Unknown pack_size value: '$args->{pack_size}'" ); |
e0098e7f |
92 | } |
93 | } |
94 | |
fde3db1a |
95 | # Grab the parameters we want to use |
96 | foreach my $param ( keys %$self ) { |
97 | next unless exists $args->{$param}; |
3e9498a1 |
98 | $self->{$param} = $args->{$param}; |
fde3db1a |
99 | } |
100 | |
e9b0b5f0 |
101 | my %validations = ( |
102 | max_buckets => { floor => 16, ceil => 256 }, |
103 | num_txns => { floor => 1, ceil => 255 }, |
104 | data_sector_size => { floor => 32, ceil => 256 }, |
105 | ); |
106 | |
107 | while ( my ($attr, $c) = each %validations ) { |
108 | if ( !defined $self->{$attr} |
109 | || !length $self->{$attr} |
110 | || $self->{$attr} =~ /\D/ |
111 | || $self->{$attr} < $c->{floor} |
112 | ) { |
113 | $self->{$attr} = '(undef)' if !defined $self->{$attr}; |
114 | warn "Floor of $attr is $c->{floor}. Setting it to $c->{floor} from '$self->{$attr}'\n"; |
115 | $self->{$attr} = $c->{floor}; |
116 | } |
117 | elsif ( $self->{$attr} > $c->{ceil} ) { |
118 | warn "Ceiling of $attr is $c->{ceil}. Setting it to $c->{ceil} from '$self->{$attr}'\n"; |
119 | $self->{$attr} = $c->{ceil}; |
120 | } |
e0098e7f |
121 | } |
122 | |
2120a181 |
123 | if ( !$self->{digest} ) { |
124 | require Digest::MD5; |
125 | $self->{digest} = \&Digest::MD5::md5; |
126 | } |
127 | |
260a80b4 |
128 | return $self; |
129 | } |
130 | |
2120a181 |
131 | ################################################################################ |
460b1067 |
132 | |
2120a181 |
133 | sub read_value { |
c3aafc14 |
134 | my $self = shift; |
2120a181 |
135 | my ($obj, $key) = @_; |
136 | |
137 | # This will be a Reference sector |
138 | my $sector = $self->_load_sector( $obj->_base_offset ) |
139 | or return; |
140 | |
141 | if ( $sector->staleness != $obj->_staleness ) { |
142 | return; |
143 | } |
144 | |
145 | my $key_md5 = $self->_apply_digest( $key ); |
146 | |
147 | my $value_sector = $sector->get_data_for({ |
148 | key_md5 => $key_md5, |
149 | allow_head => 1, |
150 | }); |
151 | |
152 | unless ( $value_sector ) { |
153 | $value_sector = DBM::Deep::Engine::Sector::Null->new({ |
154 | engine => $self, |
155 | data => undef, |
156 | }); |
157 | |
158 | $sector->write_data({ |
159 | key_md5 => $key_md5, |
160 | key => $key, |
161 | value => $value_sector, |
162 | }); |
163 | } |
164 | |
165 | return $value_sector->data; |
c3aafc14 |
166 | } |
167 | |
2120a181 |
168 | sub get_classname { |
260a80b4 |
169 | my $self = shift; |
2120a181 |
170 | my ($obj) = @_; |
260a80b4 |
171 | |
2120a181 |
172 | # This will be a Reference sector |
173 | my $sector = $self->_load_sector( $obj->_base_offset ) |
174 | or DBM::Deep->_throw_error( "How did get_classname fail (no sector for '$obj')?!" ); |
612969fb |
175 | |
2120a181 |
176 | if ( $sector->staleness != $obj->_staleness ) { |
177 | return; |
178 | } |
ea2f6d67 |
179 | |
2120a181 |
180 | return $sector->get_classname; |
1bf65be7 |
181 | } |
182 | |
1cff45d7 |
183 | sub make_reference { |
184 | my $self = shift; |
185 | my ($obj, $old_key, $new_key) = @_; |
186 | |
187 | # This will be a Reference sector |
188 | my $sector = $self->_load_sector( $obj->_base_offset ) |
00d9bd0b |
189 | or DBM::Deep->_throw_error( "How did make_reference fail (no sector for '$obj')?!" ); |
1cff45d7 |
190 | |
191 | if ( $sector->staleness != $obj->_staleness ) { |
192 | return; |
193 | } |
194 | |
195 | my $old_md5 = $self->_apply_digest( $old_key ); |
196 | |
197 | my $value_sector = $sector->get_data_for({ |
198 | key_md5 => $old_md5, |
199 | allow_head => 1, |
200 | }); |
201 | |
202 | unless ( $value_sector ) { |
203 | $value_sector = DBM::Deep::Engine::Sector::Null->new({ |
204 | engine => $self, |
205 | data => undef, |
206 | }); |
207 | |
208 | $sector->write_data({ |
209 | key_md5 => $old_md5, |
210 | key => $old_key, |
211 | value => $value_sector, |
212 | }); |
213 | } |
214 | |
215 | if ( $value_sector->isa( 'DBM::Deep::Engine::Sector::Reference' ) ) { |
216 | $sector->write_data({ |
217 | key => $new_key, |
218 | key_md5 => $self->_apply_digest( $new_key ), |
219 | value => $value_sector, |
220 | }); |
221 | $value_sector->increment_refcount; |
222 | } |
223 | else { |
224 | $sector->write_data({ |
225 | key => $new_key, |
226 | key_md5 => $self->_apply_digest( $new_key ), |
227 | value => $value_sector->clone, |
228 | }); |
229 | } |
230 | } |
231 | |
2120a181 |
232 | sub key_exists { |
0d0f3d5d |
233 | my $self = shift; |
2120a181 |
234 | my ($obj, $key) = @_; |
0d0f3d5d |
235 | |
2120a181 |
236 | # This will be a Reference sector |
237 | my $sector = $self->_load_sector( $obj->_base_offset ) |
238 | or return ''; |
0d0f3d5d |
239 | |
2120a181 |
240 | if ( $sector->staleness != $obj->_staleness ) { |
241 | return ''; |
242 | } |
0d0f3d5d |
243 | |
2120a181 |
244 | my $data = $sector->get_data_for({ |
245 | key_md5 => $self->_apply_digest( $key ), |
246 | allow_head => 1, |
247 | }); |
20b7f047 |
248 | |
2120a181 |
249 | # exists() returns 1 or '' for true/false. |
250 | return $data ? 1 : ''; |
0d0f3d5d |
251 | } |
252 | |
2120a181 |
253 | sub delete_key { |
e064ccd1 |
254 | my $self = shift; |
2120a181 |
255 | my ($obj, $key) = @_; |
e064ccd1 |
256 | |
2120a181 |
257 | my $sector = $self->_load_sector( $obj->_base_offset ) |
258 | or return; |
460b1067 |
259 | |
2120a181 |
260 | if ( $sector->staleness != $obj->_staleness ) { |
261 | return; |
262 | } |
263 | |
264 | return $sector->delete_key({ |
265 | key_md5 => $self->_apply_digest( $key ), |
266 | allow_head => 0, |
267 | }); |
268 | } |
269 | |
270 | sub write_value { |
271 | my $self = shift; |
272 | my ($obj, $key, $value) = @_; |
273 | |
274 | my $r = Scalar::Util::reftype( $value ) || ''; |
275 | { |
276 | last if $r eq ''; |
277 | last if $r eq 'HASH'; |
278 | last if $r eq 'ARRAY'; |
e064ccd1 |
279 | |
2120a181 |
280 | DBM::Deep->_throw_error( |
281 | "Storage of references of type '$r' is not supported." |
282 | ); |
460b1067 |
283 | } |
260a80b4 |
284 | |
1cff45d7 |
285 | # This will be a Reference sector |
286 | my $sector = $self->_load_sector( $obj->_base_offset ) |
00d9bd0b |
287 | or DBM::Deep->_throw_error( "1: Cannot write to a deleted spot in DBM::Deep." ); |
1cff45d7 |
288 | |
289 | if ( $sector->staleness != $obj->_staleness ) { |
00d9bd0b |
290 | DBM::Deep->_throw_error( "2: Cannot write to a deleted spot in DBM::Deep." ); |
1cff45d7 |
291 | } |
292 | |
2120a181 |
293 | my ($class, $type); |
294 | if ( !defined $value ) { |
295 | $class = 'DBM::Deep::Engine::Sector::Null'; |
296 | } |
297 | elsif ( $r eq 'ARRAY' || $r eq 'HASH' ) { |
75a6a379 |
298 | my $tmpvar; |
299 | if ( $r eq 'ARRAY' ) { |
300 | $tmpvar = tied @$value; |
301 | } elsif ( $r eq 'HASH' ) { |
302 | $tmpvar = tied %$value; |
303 | } |
5ef7542f |
304 | |
edd45134 |
305 | if ( $tmpvar ) { |
306 | my $is_dbm_deep = eval { local $SIG{'__DIE__'}; $tmpvar->isa( 'DBM::Deep' ); }; |
307 | |
308 | unless ( $is_dbm_deep ) { |
309 | DBM::Deep->_throw_error( "Cannot store something that is tied." ); |
310 | } |
311 | |
d6d8e27e |
312 | unless ( $tmpvar->_engine->storage == $self->storage ) { |
75a6a379 |
313 | DBM::Deep->_throw_error( "Cannot store values across DBM::Deep files. Please use export() instead." ); |
5ef7542f |
314 | } |
5ef7542f |
315 | |
d6d8e27e |
316 | # First, verify if we're storing the same thing to this spot. If we are, then |
317 | # this should be a no-op. -EJS, 2008-05-19 |
318 | my $loc = $sector->get_data_location_for({ |
319 | key_md5 => $self->_apply_digest( $key ), |
320 | allow_head => 1, |
321 | }); |
322 | |
323 | if ( defined($loc) && $loc == $tmpvar->_base_offset ) { |
1cff45d7 |
324 | return 1; |
325 | } |
326 | |
edd45134 |
327 | #XXX Can this use $loc? |
d6d8e27e |
328 | my $value_sector = $self->_load_sector( $tmpvar->_base_offset ); |
329 | $sector->write_data({ |
330 | key => $key, |
331 | key_md5 => $self->_apply_digest( $key ), |
332 | value => $value_sector, |
333 | }); |
334 | $value_sector->increment_refcount; |
335 | |
336 | return 1; |
1cff45d7 |
337 | } |
edd45134 |
338 | |
2120a181 |
339 | $class = 'DBM::Deep::Engine::Sector::Reference'; |
340 | $type = substr( $r, 0, 1 ); |
341 | } |
342 | else { |
1cff45d7 |
343 | if ( tied($value) ) { |
344 | DBM::Deep->_throw_error( "Cannot store something that is tied." ); |
345 | } |
2120a181 |
346 | $class = 'DBM::Deep::Engine::Sector::Scalar'; |
460b1067 |
347 | } |
9b2370e0 |
348 | |
2120a181 |
349 | # Create this after loading the reference sector in case something bad happens. |
350 | # This way, we won't allocate value sector(s) needlessly. |
351 | my $value_sector = $class->new({ |
352 | engine => $self, |
353 | data => $value, |
354 | type => $type, |
355 | }); |
75a6a379 |
356 | |
2120a181 |
357 | $sector->write_data({ |
358 | key => $key, |
359 | key_md5 => $self->_apply_digest( $key ), |
360 | value => $value_sector, |
361 | }); |
362 | |
363 | # This code is to make sure we write all the values in the $value to the disk |
364 | # and to make sure all changes to $value after the assignment are reflected |
365 | # on disk. This may be counter-intuitive at first, but it is correct dwimmery. |
366 | # NOTE - simply tying $value won't perform a STORE on each value. Hence, the |
367 | # copy to a temp value. |
368 | if ( $r eq 'ARRAY' ) { |
369 | my @temp = @$value; |
370 | tie @$value, 'DBM::Deep', { |
371 | base_offset => $value_sector->offset, |
372 | staleness => $value_sector->staleness, |
373 | storage => $self->storage, |
374 | engine => $self, |
375 | }; |
376 | @$value = @temp; |
377 | bless $value, 'DBM::Deep::Array' unless Scalar::Util::blessed( $value ); |
e064ccd1 |
378 | } |
2120a181 |
379 | elsif ( $r eq 'HASH' ) { |
380 | my %temp = %$value; |
381 | tie %$value, 'DBM::Deep', { |
382 | base_offset => $value_sector->offset, |
383 | staleness => $value_sector->staleness, |
384 | storage => $self->storage, |
385 | engine => $self, |
386 | }; |
e064ccd1 |
387 | |
2120a181 |
388 | %$value = %temp; |
389 | bless $value, 'DBM::Deep::Hash' unless Scalar::Util::blessed( $value ); |
390 | } |
460b1067 |
391 | |
2120a181 |
392 | return 1; |
e064ccd1 |
393 | } |
394 | |
2120a181 |
395 | # XXX Add staleness here |
396 | sub get_next_key { |
460b1067 |
397 | my $self = shift; |
2120a181 |
398 | my ($obj, $prev_key) = @_; |
70b55428 |
399 | |
2120a181 |
400 | # XXX Need to add logic about resetting the iterator if any key in the reference has changed |
401 | unless ( $prev_key ) { |
402 | $obj->{iterator} = DBM::Deep::Iterator->new({ |
403 | base_offset => $obj->_base_offset, |
404 | engine => $self, |
405 | }); |
406 | } |
118ba343 |
407 | |
2120a181 |
408 | return $obj->{iterator}->get_next_key( $obj ); |
409 | } |
118ba343 |
410 | |
2120a181 |
411 | ################################################################################ |
260a80b4 |
412 | |
2120a181 |
413 | sub setup_fh { |
414 | my $self = shift; |
415 | my ($obj) = @_; |
359a01ac |
416 | |
00d9bd0b |
417 | return 1 if $obj->_base_offset; |
118ba343 |
418 | |
00d9bd0b |
419 | my $header = DBM::Deep::Engine::Sector::FileHeader->new({ |
420 | engine => $self, |
421 | }); |
118ba343 |
422 | |
00d9bd0b |
423 | # Creating a new file |
424 | if ( $header->is_new ) { |
425 | # 1) Create Array/Hash entry |
426 | my $sector = DBM::Deep::Engine::Sector::Reference->new({ |
427 | engine => $self, |
428 | type => $obj->_type, |
429 | }); |
430 | $obj->{base_offset} = $sector->offset; |
431 | $obj->{staleness} = $sector->staleness; |
118ba343 |
432 | |
00d9bd0b |
433 | $self->flush; |
434 | } |
435 | # Reading from an existing file |
436 | else { |
437 | $obj->{base_offset} = $header->size; |
438 | my $sector = DBM::Deep::Engine::Sector::Reference->new({ |
439 | engine => $self, |
440 | offset => $obj->_base_offset, |
441 | }); |
442 | unless ( $sector ) { |
443 | DBM::Deep->_throw_error("Corrupted file, no master index record"); |
118ba343 |
444 | } |
2120a181 |
445 | |
00d9bd0b |
446 | unless ($obj->_type eq $sector->type) { |
447 | DBM::Deep->_throw_error("File type mismatch"); |
118ba343 |
448 | } |
f1879fdc |
449 | |
00d9bd0b |
450 | $obj->{staleness} = $sector->staleness; |
118ba343 |
451 | } |
2120a181 |
452 | |
00d9bd0b |
453 | $self->storage->set_inode; |
454 | |
2120a181 |
455 | return 1; |
456 | } |
457 | |
458 | sub begin_work { |
459 | my $self = shift; |
460 | my ($obj) = @_; |
461 | |
462 | if ( $self->trans_id ) { |
463 | DBM::Deep->_throw_error( "Cannot begin_work within an active transaction" ); |
464 | } |
465 | |
466 | my @slots = $self->read_txn_slots; |
e9b0b5f0 |
467 | my $found; |
468 | for my $i ( 0 .. $#slots ) { |
2120a181 |
469 | next if $slots[$i]; |
e9b0b5f0 |
470 | |
2120a181 |
471 | $slots[$i] = 1; |
e9b0b5f0 |
472 | $self->set_trans_id( $i + 1 ); |
473 | $found = 1; |
2120a181 |
474 | last; |
475 | } |
e9b0b5f0 |
476 | unless ( $found ) { |
477 | DBM::Deep->_throw_error( "Cannot allocate transaction ID" ); |
478 | } |
2120a181 |
479 | $self->write_txn_slots( @slots ); |
480 | |
481 | if ( !$self->trans_id ) { |
482 | DBM::Deep->_throw_error( "Cannot begin_work - no available transactions" ); |
483 | } |
484 | |
485 | return; |
486 | } |
487 | |
488 | sub rollback { |
489 | my $self = shift; |
490 | my ($obj) = @_; |
491 | |
492 | if ( !$self->trans_id ) { |
493 | DBM::Deep->_throw_error( "Cannot rollback without an active transaction" ); |
494 | } |
495 | |
496 | # Each entry is the file location for a bucket that has a modification for |
497 | # this transaction. The entries need to be expunged. |
498 | foreach my $entry (@{ $self->get_entries } ) { |
499 | # Remove the entry here |
500 | my $read_loc = $entry |
501 | + $self->hash_size |
502 | + $self->byte_size |
e9b0b5f0 |
503 | + $self->byte_size |
504 | + ($self->trans_id - 1) * ( $self->byte_size + $STALE_SIZE ); |
2120a181 |
505 | |
506 | my $data_loc = $self->storage->read_at( $read_loc, $self->byte_size ); |
507 | $data_loc = unpack( $StP{$self->byte_size}, $data_loc ); |
508 | $self->storage->print_at( $read_loc, pack( $StP{$self->byte_size}, 0 ) ); |
509 | |
510 | if ( $data_loc > 1 ) { |
511 | $self->_load_sector( $data_loc )->free; |
512 | } |
260a80b4 |
513 | } |
e06824f8 |
514 | |
2120a181 |
515 | $self->clear_entries; |
70b55428 |
516 | |
2120a181 |
517 | my @slots = $self->read_txn_slots; |
e9b0b5f0 |
518 | $slots[$self->trans_id-1] = 0; |
2120a181 |
519 | $self->write_txn_slots( @slots ); |
520 | $self->inc_txn_staleness_counter( $self->trans_id ); |
521 | $self->set_trans_id( 0 ); |
6fde4ed2 |
522 | |
70b55428 |
523 | return 1; |
524 | } |
525 | |
2120a181 |
526 | sub commit { |
16d1ad9b |
527 | my $self = shift; |
2120a181 |
528 | my ($obj) = @_; |
529 | |
530 | if ( !$self->trans_id ) { |
531 | DBM::Deep->_throw_error( "Cannot commit without an active transaction" ); |
532 | } |
533 | |
534 | foreach my $entry (@{ $self->get_entries } ) { |
535 | # Overwrite the entry in head with the entry in trans_id |
536 | my $base = $entry |
537 | + $self->hash_size |
538 | + $self->byte_size; |
539 | |
540 | my $head_loc = $self->storage->read_at( $base, $self->byte_size ); |
541 | $head_loc = unpack( $StP{$self->byte_size}, $head_loc ); |
e9b0b5f0 |
542 | |
543 | my $spot = $base + $self->byte_size + ($self->trans_id - 1) * ( $self->byte_size + $STALE_SIZE ); |
2120a181 |
544 | my $trans_loc = $self->storage->read_at( |
e9b0b5f0 |
545 | $spot, $self->byte_size, |
2120a181 |
546 | ); |
547 | |
548 | $self->storage->print_at( $base, $trans_loc ); |
549 | $self->storage->print_at( |
e9b0b5f0 |
550 | $spot, |
551 | pack( $StP{$self->byte_size} . ' ' . $StP{$STALE_SIZE}, (0) x 2 ), |
2120a181 |
552 | ); |
553 | |
554 | if ( $head_loc > 1 ) { |
555 | $self->_load_sector( $head_loc )->free; |
556 | } |
557 | } |
558 | |
559 | $self->clear_entries; |
560 | |
561 | my @slots = $self->read_txn_slots; |
e9b0b5f0 |
562 | $slots[$self->trans_id-1] = 0; |
2120a181 |
563 | $self->write_txn_slots( @slots ); |
564 | $self->inc_txn_staleness_counter( $self->trans_id ); |
565 | $self->set_trans_id( 0 ); |
566 | |
567 | return 1; |
16d1ad9b |
568 | } |
569 | |
2120a181 |
570 | sub read_txn_slots { |
d4b1166e |
571 | my $self = shift; |
e9b0b5f0 |
572 | my $bl = $self->txn_bitfield_len; |
573 | my $num_bits = $bl * 8; |
574 | return split '', unpack( 'b'.$num_bits, |
2120a181 |
575 | $self->storage->read_at( |
e9b0b5f0 |
576 | $self->trans_loc, $bl, |
2120a181 |
577 | ) |
578 | ); |
579 | } |
20f7b20c |
580 | |
2120a181 |
581 | sub write_txn_slots { |
582 | my $self = shift; |
e9b0b5f0 |
583 | my $num_bits = $self->txn_bitfield_len * 8; |
2120a181 |
584 | $self->storage->print_at( $self->trans_loc, |
e9b0b5f0 |
585 | pack( 'b'.$num_bits, join('', @_) ), |
7dcefff3 |
586 | ); |
2120a181 |
587 | } |
588 | |
589 | sub get_running_txn_ids { |
590 | my $self = shift; |
591 | my @transactions = $self->read_txn_slots; |
e9b0b5f0 |
592 | my @trans_ids = map { $_+1} grep { $transactions[$_] } 0 .. $#transactions; |
2120a181 |
593 | } |
594 | |
595 | sub get_txn_staleness_counter { |
596 | my $self = shift; |
597 | my ($trans_id) = @_; |
20f7b20c |
598 | |
2120a181 |
599 | # Hardcode staleness of 0 for the HEAD |
600 | return 0 unless $trans_id; |
f37c15ab |
601 | |
e9b0b5f0 |
602 | return unpack( $StP{$STALE_SIZE}, |
2120a181 |
603 | $self->storage->read_at( |
888453b9 |
604 | $self->trans_loc + $self->txn_bitfield_len + $STALE_SIZE * ($trans_id - 1), |
605 | $STALE_SIZE, |
2120a181 |
606 | ) |
607 | ); |
d4b1166e |
608 | } |
609 | |
2120a181 |
610 | sub inc_txn_staleness_counter { |
d4b1166e |
611 | my $self = shift; |
2120a181 |
612 | my ($trans_id) = @_; |
20f7b20c |
613 | |
2120a181 |
614 | # Hardcode staleness of 0 for the HEAD |
888453b9 |
615 | return 0 unless $trans_id; |
20f7b20c |
616 | |
2120a181 |
617 | $self->storage->print_at( |
888453b9 |
618 | $self->trans_loc + $self->txn_bitfield_len + $STALE_SIZE * ($trans_id - 1), |
e9b0b5f0 |
619 | pack( $StP{$STALE_SIZE}, $self->get_txn_staleness_counter( $trans_id ) + 1 ), |
86867f3a |
620 | ); |
2120a181 |
621 | } |
20f7b20c |
622 | |
2120a181 |
623 | sub get_entries { |
624 | my $self = shift; |
625 | return [ keys %{ $self->{entries}{$self->trans_id} ||= {} } ]; |
d4b1166e |
626 | } |
627 | |
2120a181 |
628 | sub add_entry { |
ea2f6d67 |
629 | my $self = shift; |
2120a181 |
630 | my ($trans_id, $loc) = @_; |
ea2f6d67 |
631 | |
2120a181 |
632 | $self->{entries}{$trans_id} ||= {}; |
633 | $self->{entries}{$trans_id}{$loc} = undef; |
634 | } |
ea2f6d67 |
635 | |
2120a181 |
636 | # If the buckets are being relocated because of a reindexing, the entries |
637 | # mechanism needs to be made aware of it. |
638 | sub reindex_entry { |
639 | my $self = shift; |
640 | my ($old_loc, $new_loc) = @_; |
641 | |
642 | TRANS: |
643 | while ( my ($trans_id, $locs) = each %{ $self->{entries} } ) { |
695c88b1 |
644 | if ( exists $locs->{$old_loc} ) { |
645 | delete $locs->{$old_loc}; |
646 | $locs->{$new_loc} = undef; |
647 | next TRANS; |
2120a181 |
648 | } |
ea2f6d67 |
649 | } |
ea2f6d67 |
650 | } |
651 | |
2120a181 |
652 | sub clear_entries { |
d4b1166e |
653 | my $self = shift; |
2120a181 |
654 | delete $self->{entries}{$self->trans_id}; |
655 | } |
eea0d863 |
656 | |
2120a181 |
657 | ################################################################################ |
eea0d863 |
658 | |
00d9bd0b |
659 | sub _load_sector { |
660 | my $self = shift; |
661 | my ($offset) = @_; |
75be6413 |
662 | |
00d9bd0b |
663 | # Add a catch for offset of 0 or 1 |
664 | return if !$offset || $offset <= 1; |
ea2f6d67 |
665 | |
00d9bd0b |
666 | unless ( exists $self->sector_cache->{ $offset } ) { |
667 | my $type = $self->storage->read_at( $offset, $self->SIG_SIZE ); |
019404df |
668 | |
00d9bd0b |
669 | # XXX Don't we want to do something more proactive here? -RobK, 2008-06-19 |
670 | return if $type eq chr(0); |
ea2f6d67 |
671 | |
00d9bd0b |
672 | if ( $type eq $self->SIG_ARRAY || $type eq $self->SIG_HASH ) { |
673 | $self->sector_cache->{$offset} = DBM::Deep::Engine::Sector::Reference->new({ |
674 | engine => $self, |
675 | type => $type, |
676 | offset => $offset, |
677 | }); |
2120a181 |
678 | } |
00d9bd0b |
679 | # XXX Don't we need key_md5 here? |
680 | elsif ( $type eq $self->SIG_BLIST ) { |
681 | $self->sector_cache->{$offset} = DBM::Deep::Engine::Sector::BucketList->new({ |
682 | engine => $self, |
683 | type => $type, |
684 | offset => $offset, |
685 | }); |
e9b0b5f0 |
686 | } |
00d9bd0b |
687 | elsif ( $type eq $self->SIG_INDEX ) { |
688 | $self->sector_cache->{$offset} = DBM::Deep::Engine::Sector::Index->new({ |
689 | engine => $self, |
690 | type => $type, |
691 | offset => $offset, |
692 | }); |
75be6413 |
693 | } |
00d9bd0b |
694 | elsif ( $type eq $self->SIG_NULL ) { |
695 | $self->sector_cache->{$offset} = DBM::Deep::Engine::Sector::Null->new({ |
696 | engine => $self, |
697 | type => $type, |
698 | offset => $offset, |
699 | }); |
ea2f6d67 |
700 | } |
00d9bd0b |
701 | elsif ( $type eq $self->SIG_DATA ) { |
702 | $self->sector_cache->{$offset} = DBM::Deep::Engine::Sector::Scalar->new({ |
703 | engine => $self, |
704 | type => $type, |
705 | offset => $offset, |
706 | }); |
707 | } |
708 | # This was deleted from under us, so just return and let the caller figure it out. |
709 | elsif ( $type eq $self->SIG_FREE ) { |
710 | return; |
711 | } |
712 | else { |
713 | DBM::Deep->_throw_error( "'$offset': Don't know what to do with type '$type'" ); |
c9b6d0d8 |
714 | } |
20f7b20c |
715 | } |
d4b1166e |
716 | |
00d9bd0b |
717 | return $self->sector_cache->{$offset}; |
d4b1166e |
718 | } |
719 | |
2120a181 |
720 | sub _apply_digest { |
75be6413 |
721 | my $self = shift; |
2120a181 |
722 | return $self->{digest}->(@_); |
723 | } |
16d1ad9b |
724 | |
2120a181 |
725 | sub _add_free_blist_sector { shift->_add_free_sector( 0, @_ ) } |
726 | sub _add_free_data_sector { shift->_add_free_sector( 1, @_ ) } |
727 | sub _add_free_index_sector { shift->_add_free_sector( 2, @_ ) } |
75be6413 |
728 | |
2120a181 |
729 | sub _add_free_sector { |
730 | my $self = shift; |
731 | my ($multiple, $offset, $size) = @_; |
75be6413 |
732 | |
2120a181 |
733 | my $chains_offset = $multiple * $self->byte_size; |
75be6413 |
734 | |
2120a181 |
735 | my $storage = $self->storage; |
75be6413 |
736 | |
2120a181 |
737 | # Increment staleness. |
738 | # XXX Can this increment+modulo be done by "&= 0x1" ? |
e9b0b5f0 |
739 | my $staleness = unpack( $StP{$STALE_SIZE}, $storage->read_at( $offset + SIG_SIZE, $STALE_SIZE ) ); |
740 | $staleness = ($staleness + 1 ) % ( 2 ** ( 8 * $STALE_SIZE ) ); |
741 | $storage->print_at( $offset + SIG_SIZE, pack( $StP{$STALE_SIZE}, $staleness ) ); |
75be6413 |
742 | |
2120a181 |
743 | my $old_head = $storage->read_at( $self->chains_loc + $chains_offset, $self->byte_size ); |
75be6413 |
744 | |
2120a181 |
745 | $storage->print_at( $self->chains_loc + $chains_offset, |
746 | pack( $StP{$self->byte_size}, $offset ), |
747 | ); |
75be6413 |
748 | |
2120a181 |
749 | # Record the old head in the new sector after the signature and staleness counter |
e9b0b5f0 |
750 | $storage->print_at( $offset + SIG_SIZE + $STALE_SIZE, $old_head ); |
2120a181 |
751 | } |
75be6413 |
752 | |
2120a181 |
753 | sub _request_blist_sector { shift->_request_sector( 0, @_ ) } |
754 | sub _request_data_sector { shift->_request_sector( 1, @_ ) } |
755 | sub _request_index_sector { shift->_request_sector( 2, @_ ) } |
7b1e1aa1 |
756 | |
2120a181 |
757 | sub _request_sector { |
758 | my $self = shift; |
759 | my ($multiple, $size) = @_; |
75be6413 |
760 | |
2120a181 |
761 | my $chains_offset = $multiple * $self->byte_size; |
2603d86e |
762 | |
2120a181 |
763 | my $old_head = $self->storage->read_at( $self->chains_loc + $chains_offset, $self->byte_size ); |
764 | my $loc = unpack( $StP{$self->byte_size}, $old_head ); |
75be6413 |
765 | |
2120a181 |
766 | # We don't have any free sectors of the right size, so allocate a new one. |
767 | unless ( $loc ) { |
768 | my $offset = $self->storage->request_space( $size ); |
7b1e1aa1 |
769 | |
2120a181 |
770 | # Zero out the new sector. This also guarantees correct increases |
771 | # in the filesize. |
772 | $self->storage->print_at( $offset, chr(0) x $size ); |
7b1e1aa1 |
773 | |
2120a181 |
774 | return $offset; |
7b1e1aa1 |
775 | } |
776 | |
2120a181 |
777 | # Read the new head after the signature and the staleness counter |
e9b0b5f0 |
778 | my $new_head = $self->storage->read_at( $loc + SIG_SIZE + $STALE_SIZE, $self->byte_size ); |
2120a181 |
779 | $self->storage->print_at( $self->chains_loc + $chains_offset, $new_head ); |
780 | $self->storage->print_at( |
e9b0b5f0 |
781 | $loc + SIG_SIZE + $STALE_SIZE, |
2120a181 |
782 | pack( $StP{$self->byte_size}, 0 ), |
7b1e1aa1 |
783 | ); |
75be6413 |
784 | |
2120a181 |
785 | return $loc; |
75be6413 |
786 | } |
787 | |
2120a181 |
788 | ################################################################################ |
8db25060 |
789 | |
00d9bd0b |
790 | sub sector_cache { |
791 | my $self = shift; |
792 | return $self->{sector_cache} ||= {}; |
793 | } |
794 | |
795 | sub clear_sector_cache { |
796 | my $self = shift; |
797 | $self->{sector_cache} = {}; |
798 | } |
799 | |
a8d2331c |
800 | sub dirty_sectors { |
801 | my $self = shift; |
802 | return $self->{dirty_sectors} ||= {}; |
803 | } |
804 | |
00d9bd0b |
805 | sub clear_dirty_sectors { |
806 | my $self = shift; |
807 | $self->{dirty_sectors} = {}; |
808 | } |
809 | |
a8d2331c |
810 | sub add_dirty_sector { |
811 | my $self = shift; |
812 | my ($sector) = @_; |
813 | |
814 | # if ( exists $self->dirty_sectors->{ $sector->offset } ) { |
815 | # DBM::Deep->_throw_error( "We have a duplicate sector!! " . $sector->offset ); |
816 | # } |
817 | |
818 | $self->dirty_sectors->{ $sector->offset } = $sector; |
819 | } |
820 | |
a8d2331c |
821 | sub flush { |
822 | my $self = shift; |
823 | |
00d9bd0b |
824 | my $sectors = $self->dirty_sectors; |
825 | for my $offset (sort { $a <=> $b } keys %{ $sectors }) { |
826 | $sectors->{$offset}->flush; |
a8d2331c |
827 | } |
828 | |
829 | $self->clear_dirty_sectors; |
00d9bd0b |
830 | |
831 | $self->clear_sector_cache; |
a8d2331c |
832 | } |
833 | |
834 | ################################################################################ |
835 | |
f1879fdc |
836 | sub lock_exclusive { |
837 | my $self = shift; |
838 | my ($obj) = @_; |
839 | return $self->storage->lock_exclusive( $obj ); |
840 | } |
841 | |
842 | sub lock_shared { |
843 | my $self = shift; |
844 | my ($obj) = @_; |
845 | return $self->storage->lock_shared( $obj ); |
846 | } |
847 | |
848 | sub unlock { |
849 | my $self = shift; |
850 | my ($obj) = @_; |
a8d2331c |
851 | |
852 | my $rv = $self->storage->unlock( $obj ); |
853 | |
854 | $self->flush if $rv; |
855 | |
856 | return $rv; |
f1879fdc |
857 | } |
858 | |
859 | ################################################################################ |
860 | |
2120a181 |
861 | sub storage { $_[0]{storage} } |
862 | sub byte_size { $_[0]{byte_size} } |
863 | sub hash_size { $_[0]{hash_size} } |
864 | sub hash_chars { $_[0]{hash_chars} } |
865 | sub num_txns { $_[0]{num_txns} } |
866 | sub max_buckets { $_[0]{max_buckets} } |
867 | sub blank_md5 { chr(0) x $_[0]->hash_size } |
e9b0b5f0 |
868 | sub data_sector_size { $_[0]{data_sector_size} } |
869 | |
870 | # This is a calculated value |
871 | sub txn_bitfield_len { |
872 | my $self = shift; |
873 | unless ( exists $self->{txn_bitfield_len} ) { |
874 | my $temp = ($self->num_txns) / 8; |
875 | if ( $temp > int( $temp ) ) { |
876 | $temp = int( $temp ) + 1; |
877 | } |
878 | $self->{txn_bitfield_len} = $temp; |
879 | } |
880 | return $self->{txn_bitfield_len}; |
881 | } |
8db25060 |
882 | |
2120a181 |
883 | sub trans_id { $_[0]{trans_id} } |
884 | sub set_trans_id { $_[0]{trans_id} = $_[1] } |
8db25060 |
885 | |
2120a181 |
886 | sub trans_loc { $_[0]{trans_loc} } |
887 | sub set_trans_loc { $_[0]{trans_loc} = $_[1] } |
888 | |
889 | sub chains_loc { $_[0]{chains_loc} } |
890 | sub set_chains_loc { $_[0]{chains_loc} = $_[1] } |
891 | |
c57b19c6 |
892 | sub cache { $_[0]{cache} ||= {} } |
893 | sub clear_cache { %{$_[0]->cache} = () } |
894 | |
888453b9 |
895 | sub _dump_file { |
896 | my $self = shift; |
00d9bd0b |
897 | $self->flush; |
888453b9 |
898 | |
899 | # Read the header |
00d9bd0b |
900 | my $header_sector = DBM::Deep::Engine::Sector::FileHeader->new({ |
901 | engine => $self, |
902 | }); |
888453b9 |
903 | |
904 | my %types = ( |
905 | 0 => 'B', |
906 | 1 => 'D', |
907 | 2 => 'I', |
908 | ); |
909 | |
910 | my %sizes = ( |
911 | 'D' => $self->data_sector_size, |
912 | 'B' => DBM::Deep::Engine::Sector::BucketList->new({engine=>$self,offset=>1})->size, |
913 | 'I' => DBM::Deep::Engine::Sector::Index->new({engine=>$self,offset=>1})->size, |
914 | ); |
915 | |
916 | my $return = ""; |
c57b19c6 |
917 | |
a8d2331c |
918 | # Filesize |
919 | $return .= "Size: " . (-s $self->storage->{fh}) . $/; |
920 | |
c57b19c6 |
921 | # Header values |
922 | $return .= "NumTxns: " . $self->num_txns . $/; |
923 | |
888453b9 |
924 | # Read the free sector chains |
925 | my %sectors; |
926 | foreach my $multiple ( 0 .. 2 ) { |
927 | $return .= "Chains($types{$multiple}):"; |
928 | my $old_loc = $self->chains_loc + $multiple * $self->byte_size; |
929 | while ( 1 ) { |
930 | my $loc = unpack( |
931 | $StP{$self->byte_size}, |
932 | $self->storage->read_at( $old_loc, $self->byte_size ), |
933 | ); |
934 | |
935 | # We're now out of free sectors of this kind. |
936 | unless ( $loc ) { |
937 | last; |
938 | } |
939 | |
940 | $sectors{ $types{$multiple} }{ $loc } = undef; |
941 | $old_loc = $loc + SIG_SIZE + $STALE_SIZE; |
942 | $return .= " $loc"; |
943 | } |
944 | $return .= $/; |
945 | } |
946 | |
00d9bd0b |
947 | my $spot = $header_sector->size; |
888453b9 |
948 | SECTOR: |
949 | while ( $spot < $self->storage->{end} ) { |
950 | # Read each sector in order. |
951 | my $sector = $self->_load_sector( $spot ); |
952 | if ( !$sector ) { |
953 | # Find it in the free-sectors that were found already |
954 | foreach my $type ( keys %sectors ) { |
955 | if ( exists $sectors{$type}{$spot} ) { |
956 | my $size = $sizes{$type}; |
957 | $return .= sprintf "%08d: %s %04d\n", $spot, 'F' . $type, $size; |
958 | $spot += $size; |
959 | next SECTOR; |
960 | } |
961 | } |
962 | |
963 | die "********\n$return\nDidn't find free sector for $spot in chains\n********\n"; |
964 | } |
965 | else { |
966 | $return .= sprintf "%08d: %s %04d", $spot, $sector->type, $sector->size; |
967 | if ( $sector->type eq 'D' ) { |
968 | $return .= ' ' . $sector->data; |
969 | } |
970 | elsif ( $sector->type eq 'A' || $sector->type eq 'H' ) { |
971 | $return .= ' REF: ' . $sector->get_refcount; |
972 | } |
973 | elsif ( $sector->type eq 'B' ) { |
974 | foreach my $bucket ( $sector->chopped_up ) { |
975 | $return .= "\n "; |
976 | $return .= sprintf "%08d", unpack($StP{$self->byte_size}, |
977 | substr( $bucket->[-1], $self->hash_size, $self->byte_size), |
978 | ); |
c57b19c6 |
979 | my $l = unpack( $StP{$self->byte_size}, |
980 | substr( $bucket->[-1], |
981 | $self->hash_size + $self->byte_size, |
982 | $self->byte_size, |
983 | ), |
984 | ); |
985 | $return .= sprintf " %08d", $l; |
986 | foreach my $txn ( 0 .. $self->num_txns - 2 ) { |
888453b9 |
987 | my $l = unpack( $StP{$self->byte_size}, |
988 | substr( $bucket->[-1], |
c57b19c6 |
989 | $self->hash_size + 2 * $self->byte_size + $txn * ($self->byte_size + $STALE_SIZE), |
888453b9 |
990 | $self->byte_size, |
991 | ), |
992 | ); |
993 | $return .= sprintf " %08d", $l; |
994 | } |
995 | } |
996 | } |
997 | $return .= $/; |
998 | |
999 | $spot += $sector->size; |
1000 | } |
1001 | } |
1002 | |
1003 | return $return; |
1004 | } |
1005 | |
a20d9a3f |
1006 | 1; |
1007 | __END__ |