checker
changeset 10 fd5225120ee9
parent 9 70d578b200ba
child 11 675ef249d116
equal deleted inserted replaced
9:70d578b200ba 10:fd5225120ee9
    41     # timestamp we see
    41     # timestamp we see
    42     #tie %idx, "DB_File" => $tmp->filename;
    42     #tie %idx, "DB_File" => $tmp->filename;
    43     my %block = get_block_list($dir);
    43     my %block = get_block_list($dir);
    44 
    44 
    45     verbose("# indexed: "
    45     verbose("# indexed: "
    46 	. scalar(@{$block{".idx"}//[]}) . " images with "
    46 	. scalar(@{$block{""}//[]}) . " images with "
    47 	. (grep /^\.idx/ => keys(%block) - 1)." files");
    47 	. (grep !/^\.idx$/ => keys(%block))." blocks");
    48 
    48 
    49     purge_unused($dir => %block);
    49     purge_unused($dir => %block);
    50     check_images($dir => %block);
    50     check_images($dir => %block);
    51 }
    51 }
    52 
    52 
    67     my $dir = shift;
    67     my $dir = shift;
    68     my %block;
    68     my %block;
    69     find(sub {
    69     find(sub {
    70 	(-f) and (-M > 0) or return;
    70 	(-f) and (-M > 0) or return;
    71 	#verbose("idx: $File::Find::name");
    71 	#verbose("idx: $File::Find::name");
    72 	push @{$block{".idx"}}, abs_path $_;
    72 	push @{$block{""}}, abs_path $_;
    73 	foreach my $f (get_file_list($_)) {
    73 	foreach my $f (get_file_list($_)) {
    74 	    push @{$block{$f}} => $#{$block{".idx"}};
    74 	    push @{$block{$f}} => $#{$block{""}};
    75 	}
    75 	}
    76     }, "$dir/idx");
    76     }, "$dir/idx");
    77     return %block;
    77     return %block;
    78 }
    78 }
    79 
    79 
    80 sub purge_unused {
    80 sub purge_unused {
    81     my ($dir, %block) = @_;
    81     my ($dir, %block) = @_;
    82 
    82 
    83     my ($total, $done);
    83     my ($total, $done);
       
    84     verbose("# pass 1 - purge unused blocks");
    84 
    85 
    85     # calculate the number of files we expect
    86     # calculate the number of files we expect
    86     find(sub {
    87     find(sub {
    87 	-d or return;
    88 	-d or return;
    88 	opendir(my $dh => $_);
    89 	opendir(my $dh => $_);
    93 
    94 
    94     # progress
    95     # progress
    95     local $SIG{ALRM} = sub {
    96     local $SIG{ALRM} = sub {
    96 	return alarm 1 if not $done;
    97 	return alarm 1 if not $done;
    97 	my $speed = $done / (time - $^T + 1);
    98 	my $speed = $done / (time - $^T + 1);
    98 	verbose sprintf "# done %5.1f%% | %25s (%*d of %d files)",
    99 	verbose sprintf "# pass 1 done %5.1f%% | %25s (%*d of %d blocks)",
    99 	    100 * ($done/$total),
   100 	    100 * ($done/$total),
   100 	    scalar(localtime($^T + $speed * ($total - $done))),
   101 	    scalar(localtime($^T + $speed * ($total - $done))),
   101 	    length($total) => $done,
   102 	    length($total) => $done,
   102 	    $total;
   103 	    $total;
   103 	alarm 5;
   104 	alarm 5;
   111 	# cut away the first part of the filename and
   112 	# cut away the first part of the filename and
   112 	# some optional extension
   113 	# some optional extension
   113 	(my $rn = $File::Find::name) =~ s/^$dir\/data\/(.*?)(?:\..+)?$/$1/;
   114 	(my $rn = $File::Find::name) =~ s/^$dir\/data\/(.*?)(?:\..+)?$/$1/;
   114 	exists $block{$rn} and return;
   115 	exists $block{$rn} and return;
   115 
   116 
       
   117 
   116 	if ($o{yes}) {
   118 	if ($o{yes}) {
   117 	    verbose("unlinking abs_path $File::Find::name");
   119 	   verbose("unlinking " . abs_path $File::Find::name);
   118 	    unlink abs_path $File::Find::name;
   120 	   unlink abs_path $File::Find::name;
   119 	    return;
   121 	   return;
   120 	}
   122 	}
   121 
   123 
   122 	verbose("unused abs_path $File::Find::name");
   124 	verbose("unused " . abs_path $File::Find::name);
   123 	return;
   125 	return;
   124 
   126 
   125     }, "$dir/data");
   127     }, "$dir/data");
   126     $SIG{ALRM}->();
   128     $SIG{ALRM}->();
   127     alarm 0;
   129     alarm 0;
   129 }
   131 }
   130 
   132 
   131 sub check_images {
   133 sub check_images {
   132     my ($dir, %block) = @_;
   134     my ($dir, %block) = @_;
   133 
   135 
   134     my $total = keys(%block) - 1; # .idx
   136     my $total = grep { $_ ne "" } keys(%block);
   135     my $done = 0;
   137     my $done = 0;
       
   138 
       
   139     verbose("# pass 2 - check image completness");
   136 
   140 
   137     # progress
   141     # progress
   138     local $SIG{ALRM} = sub {
   142     local $SIG{ALRM} = sub {
   139 	return alarm 1 if not $done;
   143 	return alarm 1 if not $done;
   140 	my $speed = $done / (time - $^T + 1);
   144 	my $speed = $done / (time - $^T + 1);
   141 	say sprintf "# done %5.1f%% | %25s (%*d of %d files)",
   145 	verbose sprintf "# pass 2 done %5.1f%% | %25s (%*d of %d blocks)",
   142 	    100 * $done/$total, 
   146 	    100 * $done/$total, 
   143 	    scalar(localtime($^T + ($total - $done) * $speed)),
   147 	    scalar(localtime($^T + ($total - $done) * $speed)),
   144 	    length($total) => $done,
   148 	    length($total) => $done,
   145 	    $total;
   149 	    $total;
   146 	    alarm 5;
   150 	    alarm 5;
   148     $SIG{ALRM}->();
   152     $SIG{ALRM}->();
   149 
   153 
   150     my %invalid;
   154     my %invalid;
   151     foreach my $k (keys %block) {
   155     foreach my $k (keys %block) {
   152 	my $i = $block{$k};
   156 	my $i = $block{$k};
   153 	next if $k eq ".idx";
   157 	next if $k eq "";
   154 	++$done;
   158 	++$done;
   155 	
   159 	
   156 	next if -f "$dir/data/$k"
   160 	next if -f "$dir/data/$k"
   157 	    or -f "$dir/data/$k.gz";
   161 	    or -f "$dir/data/$k.gz";
   158 	say "missing $k";
   162 	say "missing $k";
   159 	@invalid{@{$block{".idx"}}} = ();
   163 	@invalid{@{$block{""}}} = ();
   160     }
   164     }
   161     $SIG{ALRM}->();
   165     $SIG{ALRM}->();
   162     alarm 0;
   166     alarm 0;
   163 
   167 
   164     return if not %invalid;
   168     return if not %invalid;