3 use vars qw($LAST_TIME $DATE_OF_02 $DATE_OF_03 $HAVE_REANIMATED $VERSION);
5 @CPAN::Index::ISA = qw(CPAN::Debug);
8 # use constant PROTOCOL => "2.0"; # outcommented to avoid warning on upgrade from 1.57
11 #-> sub CPAN::Index::force_reload ;
14 $CPAN::Index::LAST_TIME = 0;
18 #-> sub CPAN::Index::reload ;
20 my($self,$force) = @_;
23 # XXX check if a newer one is available. (We currently read it
25 for ($CPAN::Config->{index_expire}) {
26 $_ = 0.001 unless $_ && $_ > 0.001;
28 unless (1 || $CPAN::Have_warned->{readmetadatacache}++) {
29 # debug here when CPAN doesn't seem to read the Metadata
31 Carp::cluck("META-PROTOCOL[$CPAN::META->{PROTOCOL}]");
33 unless ($CPAN::META->{PROTOCOL}) {
34 $self->read_metadata_cache;
35 $CPAN::META->{PROTOCOL} ||= "1.0";
37 if ( $CPAN::META->{PROTOCOL} < PROTOCOL ) {
38 # warn "Setting last_time to 0";
39 $LAST_TIME = 0; # No warning necessary
41 if ($LAST_TIME + $CPAN::Config->{index_expire}*86400 > $time
44 # CPAN->debug("LAST_TIME[$LAST_TIME]index_expire[$CPAN::Config->{index_expire}]time[$time]");
46 # IFF we are developing, it helps to wipe out the memory
47 # between reloads, otherwise it is not what a user expects.
48 undef $CPAN::META; # Neue Gruendlichkeit since v1.52(r1.274)
49 $CPAN::META = CPAN->new;
52 local $LAST_TIME = $time;
53 local $CPAN::META->{PROTOCOL} = PROTOCOL;
55 my $needshort = $^O eq "dos";
57 $self->rd_authindex($self
59 "authors/01mailrc.txt.gz",
61 File::Spec->catfile('authors', '01mailrc.gz') :
62 File::Spec->catfile('authors', '01mailrc.txt.gz'),
65 $debug = "timing reading 01[".($t2 - $time)."]";
67 return if $CPAN::Signal; # this is sometimes lengthy
68 $self->rd_modpacks($self
70 "modules/02packages.details.txt.gz",
72 File::Spec->catfile('modules', '02packag.gz') :
73 File::Spec->catfile('modules', '02packages.details.txt.gz'),
76 $debug .= "02[".($t2 - $time)."]";
78 return if $CPAN::Signal; # this is sometimes lengthy
79 $self->rd_modlist($self
81 "modules/03modlist.data.gz",
83 File::Spec->catfile('modules', '03mlist.gz') :
84 File::Spec->catfile('modules', '03modlist.data.gz'),
86 $self->write_metadata_cache;
88 $debug .= "03[".($t2 - $time)."]";
90 CPAN->debug($debug) if $CPAN::DEBUG;
92 if ($CPAN::Config->{build_dir_reuse}) {
93 $self->reanimate_build_dir;
95 if (CPAN::_sqlite_running()) {
96 $CPAN::SQLite->reload(time => $time, force => $force)
100 $CPAN::META->{PROTOCOL} = PROTOCOL;
103 #-> sub CPAN::Index::reanimate_build_dir ;
104 sub reanimate_build_dir {
106 unless ($CPAN::META->has_inst($CPAN::Config->{yaml_module}||"YAML")) {
109 return if $HAVE_REANIMATED++;
110 my $d = $CPAN::Config->{build_dir};
111 my $dh = DirHandle->new;
112 opendir $dh, $d or return; # does not exist
117 my @candidates = map { $_->[0] }
118 sort { $b->[1] <=> $a->[1] }
119 map { [ $_, -M File::Spec->catfile($d,$_) ] }
120 grep {/\.yml$/} readdir $dh;
121 unless (@candidates) {
122 $CPAN::Frontend->myprint("Build_dir empty, nothing to restore\n");
125 $CPAN::Frontend->myprint
126 (sprintf("Going to read %d yaml file%s from %s/\n",
128 @candidates==1 ? "" : "s",
129 $CPAN::Config->{build_dir}
131 my $start = CPAN::FTP::_mytime();
132 DISTRO: for $i (0..$#candidates) {
133 my $dirent = $candidates[$i];
134 my $y = eval {CPAN->_yaml_loadfile(File::Spec->catfile($d,$dirent))};
136 warn "Error while parsing file '$dirent'; error: '$@'";
140 if ($c && CPAN->_perl_fingerprint($c->{perl})) {
141 my $key = $c->{distribution}{ID};
142 for my $k (keys %{$c->{distribution}}) {
143 if ($c->{distribution}{$k}
144 && ref $c->{distribution}{$k}
145 && UNIVERSAL::isa($c->{distribution}{$k},"CPAN::Distrostatus")) {
146 $c->{distribution}{$k}{COMMANDID} = $i - @candidates;
150 #we tried to restore only if element already
151 #exists; but then we do not work with metadata
154 = $CPAN::META->{readwrite}{'CPAN::Distribution'}{$key}
155 = $c->{distribution};
158 configure_requires_later
159 configure_requires_later_for
169 delete $do->{$skipper};
171 if ($do->tested_ok_but_not_installed) {
172 $CPAN::META->is_tested($do->{build_dir},$do->{make_test}{TIME});
177 while (($painted/76) < ($i/@candidates)) {
178 $CPAN::Frontend->myprint(".");
182 my $took = CPAN::FTP::_mytime() - $start;
183 $CPAN::Frontend->myprint(sprintf(
184 "DONE\nRestored the state of %s (in %.4f secs)\n",
191 #-> sub CPAN::Index::reload_x ;
193 my($cl,$wanted,$localname,$force) = @_;
194 $force |= 2; # means we're dealing with an index here
195 CPAN::HandleConfig->load; # we should guarantee loading wherever
196 # we rely on Config XXX
197 $localname ||= $wanted;
198 my $abs_wanted = File::Spec->catfile($CPAN::Config->{'keep_source_where'},
202 -M $abs_wanted < $CPAN::Config->{'index_expire'} &&
205 my $s = $CPAN::Config->{'index_expire'} == 1 ? "" : "s";
206 $cl->debug(qq{$abs_wanted younger than $CPAN::Config->{'index_expire'} }.
207 qq{day$s. I\'ll use that.});
210 $force |= 1; # means we're quite serious about it.
212 return CPAN::FTP->localize($wanted,$abs_wanted,$force);
215 #-> sub CPAN::Index::rd_authindex ;
217 my($cl, $index_target) = @_;
218 return unless defined $index_target;
219 return if CPAN::_sqlite_running();
221 $CPAN::Frontend->myprint("Going to read '$index_target'\n");
223 tie *FH, 'CPAN::Tarzip', $index_target;
226 push @lines, split /\012/ while <FH>;
230 my($userid,$fullname,$email) =
231 m/alias\s+(\S+)\s+\"([^\"\<]*)\s+\<(.*)\>\"/;
232 $fullname ||= $email;
233 if ($userid && $fullname && $email) {
234 my $userobj = $CPAN::META->instance('CPAN::Author',$userid);
235 $userobj->set('FULLNAME' => $fullname, 'EMAIL' => $email);
237 CPAN->debug(sprintf "line[%s]", $_) if $CPAN::DEBUG;
240 while (($painted/76) < ($i/@lines)) {
241 $CPAN::Frontend->myprint(".");
244 return if $CPAN::Signal;
246 $CPAN::Frontend->myprint("DONE\n");
250 my($self,$dist) = @_;
251 $dist = $self->{'id'} unless defined $dist;
252 my($ret) = $dist =~ m|(?:\w/\w\w/)?([^/]+)/|;
256 #-> sub CPAN::Index::rd_modpacks ;
258 my($self, $index_target) = @_;
259 return unless defined $index_target;
260 return if CPAN::_sqlite_running();
261 $CPAN::Frontend->myprint("Going to read '$index_target'\n");
262 my $fh = CPAN::Tarzip->TIEHANDLE($index_target);
264 CPAN->debug(sprintf "start[%d]", time) if $CPAN::DEBUG;
267 while (my $bytes = $fh->READ(\$chunk,8192)) {
270 my @lines = split /\012/, $slurp;
271 CPAN->debug(sprintf "end[%d]", time) if $CPAN::DEBUG;
274 my($line_count,$last_updated);
276 my $shift = shift(@lines);
277 last if $shift =~ /^\s*$/;
278 $shift =~ /^Line-Count:\s+(\d+)/ and $line_count = $1;
279 $shift =~ /^Last-Updated:\s+(.+)/ and $last_updated = $1;
281 CPAN->debug("line_count[$line_count]last_updated[$last_updated]") if $CPAN::DEBUG;
282 if (not defined $line_count) {
284 $CPAN::Frontend->mywarn(qq{Warning: Your $index_target does not contain a Line-Count header.
285 Please check the validity of the index file by comparing it to more
286 than one CPAN mirror. I'll continue but problems seem likely to
290 $CPAN::Frontend->mysleep(5);
291 } elsif ($line_count != scalar @lines) {
293 $CPAN::Frontend->mywarn(sprintf qq{Warning: Your %s
294 contains a Line-Count header of %d but I see %d lines there. Please
295 check the validity of the index file by comparing it to more than one
296 CPAN mirror. I'll continue but problems seem likely to happen.\a\n},
297 $index_target, $line_count, scalar(@lines));
300 if (not defined $last_updated) {
302 $CPAN::Frontend->mywarn(qq{Warning: Your $index_target does not contain a Last-Updated header.
303 Please check the validity of the index file by comparing it to more
304 than one CPAN mirror. I'll continue but problems seem likely to
308 $CPAN::Frontend->mysleep(5);
312 ->myprint(sprintf qq{ Database was generated on %s\n},
314 $DATE_OF_02 = $last_updated;
317 if ($CPAN::META->has_inst('HTTP::Date')) {
319 $age -= HTTP::Date::str2time($last_updated);
321 $CPAN::Frontend->mywarn(" HTTP::Date not available\n");
323 my(@d) = $last_updated =~ / (\d+) (\w+) (\d+) (\d+):(\d+):(\d+) /;
324 $d[1] = index("Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec", $d[1])/4;
325 $age -= $d[1]>=0 ? Time::Local::timegm(@d[5,4,3,0,1,2]) : 0;
332 qq{Warning: This index file is %d days old.
333 Please check the host you chose as your CPAN mirror for staleness.
334 I'll continue but problems seem likely to happen.\a\n},
337 } elsif ($age < -1) {
341 qq{Warning: Your system date is %d days behind this index file!
343 Timestamp index file: %s
344 Please fix your system time, problems with the make command expected.\n},
354 # A necessity since we have metadata_cache: delete what isn't
356 my $secondtime = $CPAN::META->exists("CPAN::Module","CPAN");
357 CPAN->debug("secondtime[$secondtime]") if $CPAN::DEBUG;
362 # before 1.56 we split into 3 and discarded the rest. From
363 # 1.57 we assign remaining text to $comment thus allowing to
365 my($mod,$version,$dist,$comment) = split " ", $_, 4;
366 unless ($mod && defined $version && $dist) {
367 $CPAN::Frontend->mywarn("Could not split line[$_]\n");
370 my($bundle,$id,$userid);
372 if ($mod eq 'CPAN' &&
374 CPAN::Queue->exists('Bundle::CPAN') ||
375 CPAN::Queue->exists('CPAN')
379 if ($version > $CPAN::VERSION) {
380 $CPAN::Frontend->mywarn(qq{
381 New CPAN.pm version (v$version) available.
382 [Currently running version is v$CPAN::VERSION]
383 You might want to try
386 to both upgrade CPAN.pm and run the new version without leaving
390 $CPAN::Frontend->mysleep(2);
391 $CPAN::Frontend->myprint(qq{\n});
393 last if $CPAN::Signal;
394 } elsif ($mod =~ /^Bundle::(.*)/) {
399 $id = $CPAN::META->instance('CPAN::Bundle',$mod);
400 # Let's make it a module too, because bundles have so much
401 # in common with modules.
403 # Changed in 1.57_63: seems like memory bloat now without
404 # any value, so commented out
406 # $CPAN::META->instance('CPAN::Module',$mod);
410 # instantiate a module object
411 $id = $CPAN::META->instance('CPAN::Module',$mod);
415 # Although CPAN prohibits same name with different version the
416 # indexer may have changed the version for the same distro
417 # since the last time ("Force Reindexing" feature)
418 if ($id->cpan_file ne $dist
420 $id->cpan_version ne $version
422 $userid = $id->userid || $self->userid($dist);
424 'CPAN_USERID' => $userid,
425 'CPAN_VERSION' => $version,
426 'CPAN_FILE' => $dist,
430 # instantiate a distribution object
431 if ($CPAN::META->exists('CPAN::Distribution',$dist)) {
432 # we do not need CONTAINSMODS unless we do something with
433 # this dist, so we better produce it on demand.
435 ## my $obj = $CPAN::META->instance(
436 ## 'CPAN::Distribution' => $dist
438 ## $obj->{CONTAINSMODS}{$mod} = undef; # experimental
440 $CPAN::META->instance(
441 'CPAN::Distribution' => $dist
443 'CPAN_USERID' => $userid,
444 'CPAN_COMMENT' => $comment,
448 for my $name ($mod,$dist) {
449 # $self->debug("exists name[$name]") if $CPAN::DEBUG;
450 $exists{$name} = undef;
454 while (($painted/76) < ($i/@lines)) {
455 $CPAN::Frontend->myprint(".");
458 return if $CPAN::Signal;
460 $CPAN::Frontend->myprint("DONE\n");
462 for my $class (qw(CPAN::Module CPAN::Bundle CPAN::Distribution)) {
463 for my $o ($CPAN::META->all_objects($class)) {
464 next if exists $exists{$o->{ID}};
465 $CPAN::META->delete($class,$o->{ID});
466 # CPAN->debug("deleting ID[$o->{ID}] in class[$class]")
473 #-> sub CPAN::Index::rd_modlist ;
475 my($cl,$index_target) = @_;
476 return unless defined $index_target;
477 return if CPAN::_sqlite_running();
478 $CPAN::Frontend->myprint("Going to read '$index_target'\n");
479 my $fh = CPAN::Tarzip->TIEHANDLE($index_target);
483 while (my $bytes = $fh->READ(\$chunk,8192)) {
486 my @eval2 = split /\012/, $slurp;
489 my $shift = shift(@eval2);
490 if ($shift =~ /^Date:\s+(.*)/) {
491 if ($DATE_OF_03 eq $1) {
492 $CPAN::Frontend->myprint("Unchanged.\n");
497 last if $shift =~ /^\s*$/;
499 push @eval2, q{CPAN::Modulelist->data;};
501 my($compmt) = Safe->new("CPAN::Safe1");
502 my($eval2) = join("\n", @eval2);
503 CPAN->debug(sprintf "length of eval2[%d]", length $eval2) if $CPAN::DEBUG;
504 my $ret = $compmt->reval($eval2);
505 Carp::confess($@) if $@;
506 return if $CPAN::Signal;
508 my $until = keys(%$ret);
510 CPAN->debug(sprintf "until[%d]", $until) if $CPAN::DEBUG;
512 my $obj = $CPAN::META->instance("CPAN::Module",$_);
513 delete $ret->{$_}{modid}; # not needed here, maybe elsewhere
514 $obj->set(%{$ret->{$_}});
516 while (($painted/76) < ($i/$until)) {
517 $CPAN::Frontend->myprint(".");
520 return if $CPAN::Signal;
522 $CPAN::Frontend->myprint("DONE\n");
525 #-> sub CPAN::Index::write_metadata_cache ;
526 sub write_metadata_cache {
528 return unless $CPAN::Config->{'cache_metadata'};
529 return if CPAN::_sqlite_running();
530 return unless $CPAN::META->has_usable("Storable");
532 foreach my $k (qw(CPAN::Bundle CPAN::Author CPAN::Module
533 CPAN::Distribution)) {
534 $cache->{$k} = $CPAN::META->{readonly}{$k}; # unsafe meta access, ok
536 my $metadata_file = File::Spec->catfile($CPAN::Config->{cpan_home},"Metadata");
537 $cache->{last_time} = $LAST_TIME;
538 $cache->{DATE_OF_02} = $DATE_OF_02;
539 $cache->{PROTOCOL} = PROTOCOL;
540 $CPAN::Frontend->myprint("Going to write $metadata_file\n");
541 eval { Storable::nstore($cache, $metadata_file) };
542 $CPAN::Frontend->mywarn($@) if $@; # ?? missing "\n" after $@ in mywarn ??
545 #-> sub CPAN::Index::read_metadata_cache ;
546 sub read_metadata_cache {
548 return unless $CPAN::Config->{'cache_metadata'};
549 return if CPAN::_sqlite_running();
550 return unless $CPAN::META->has_usable("Storable");
551 my $metadata_file = File::Spec->catfile($CPAN::Config->{cpan_home},"Metadata");
552 return unless -r $metadata_file and -f $metadata_file;
553 $CPAN::Frontend->myprint("Going to read '$metadata_file'\n");
555 eval { $cache = Storable::retrieve($metadata_file) };
556 $CPAN::Frontend->mywarn($@) if $@; # ?? missing "\n" after $@ in mywarn ??
557 if (!$cache || !UNIVERSAL::isa($cache, 'HASH')) {
561 if (exists $cache->{PROTOCOL}) {
562 if (PROTOCOL > $cache->{PROTOCOL}) {
563 $CPAN::Frontend->mywarn(sprintf("Ignoring Metadata cache written ".
564 "with protocol v%s, requiring v%s\n",
571 $CPAN::Frontend->mywarn("Ignoring Metadata cache written ".
572 "with protocol v1.0\n");
577 while(my($class,$v) = each %$cache) {
578 next unless $class =~ /^CPAN::/;
579 $CPAN::META->{readonly}{$class} = $v; # unsafe meta access, ok
580 while (my($id,$ro) = each %$v) {
581 $CPAN::META->{readwrite}{$class}{$id} ||=
582 $class->new(ID=>$id, RO=>$ro);
587 unless ($clcnt) { # sanity check
588 $CPAN::Frontend->myprint("Warning: Found no data in $metadata_file\n");
592 $CPAN::Frontend->myprint("Warning: Found only $idcnt objects ".
593 "in $metadata_file\n");
596 $CPAN::META->{PROTOCOL} ||=
597 $cache->{PROTOCOL}; # reading does not up or downgrade, but it
598 # does initialize to some protocol
599 $LAST_TIME = $cache->{last_time};
600 $DATE_OF_02 = $cache->{DATE_OF_02};
601 $CPAN::Frontend->myprint(" Database was generated on $DATE_OF_02\n")
602 if defined $DATE_OF_02; # An old cache may not contain DATE_OF_02