equal
deleted
inserted
replaced
41 # timestamp we see |
41 # timestamp we see |
42 #tie %idx, "DB_File" => $tmp->filename; |
42 #tie %idx, "DB_File" => $tmp->filename; |
43 my %block = get_block_list($dir); |
43 my %block = get_block_list($dir); |
44 |
44 |
45 verbose("# indexed: " |
45 verbose("# indexed: " |
46 . scalar(@{$block{".idx"}//[]}) . " images with " |
46 . scalar(@{$block{""}//[]}) . " images with " |
47 . (grep /^\.idx/ => keys(%block) - 1)." files"); |
47 . (grep !/^\.idx$/ => keys(%block))." blocks"); |
48 |
48 |
49 purge_unused($dir => %block); |
49 purge_unused($dir => %block); |
50 check_images($dir => %block); |
50 check_images($dir => %block); |
51 } |
51 } |
52 |
52 |
67 my $dir = shift; |
67 my $dir = shift; |
68 my %block; |
68 my %block; |
69 find(sub { |
69 find(sub { |
70 (-f) and (-M > 0) or return; |
70 (-f) and (-M > 0) or return; |
71 #verbose("idx: $File::Find::name"); |
71 #verbose("idx: $File::Find::name"); |
72 push @{$block{".idx"}}, abs_path $_; |
72 push @{$block{""}}, abs_path $_; |
73 foreach my $f (get_file_list($_)) { |
73 foreach my $f (get_file_list($_)) { |
74 push @{$block{$f}} => $#{$block{".idx"}}; |
74 push @{$block{$f}} => $#{$block{""}}; |
75 } |
75 } |
76 }, "$dir/idx"); |
76 }, "$dir/idx"); |
77 return %block; |
77 return %block; |
78 } |
78 } |
79 |
79 |
80 sub purge_unused { |
80 sub purge_unused { |
81 my ($dir, %block) = @_; |
81 my ($dir, %block) = @_; |
82 |
82 |
83 my ($total, $done); |
83 my ($total, $done); |
|
84 verbose("# pass 1 - purge unused blocks"); |
84 |
85 |
85 # calculate the number of files we expect |
86 # calculate the number of files we expect |
86 find(sub { |
87 find(sub { |
87 -d or return; |
88 -d or return; |
88 opendir(my $dh => $_); |
89 opendir(my $dh => $_); |
93 |
94 |
94 # progress |
95 # progress |
95 local $SIG{ALRM} = sub { |
96 local $SIG{ALRM} = sub { |
96 return alarm 1 if not $done; |
97 return alarm 1 if not $done; |
97 my $speed = $done / (time - $^T + 1); |
98 my $speed = $done / (time - $^T + 1); |
98 verbose sprintf "# done %5.1f%% | %25s (%*d of %d files)", |
99 verbose sprintf "# pass 1 done %5.1f%% | %25s (%*d of %d blocks)", |
99 100 * ($done/$total), |
100 100 * ($done/$total), |
100 scalar(localtime($^T + $speed * ($total - $done))), |
101 scalar(localtime($^T + $speed * ($total - $done))), |
101 length($total) => $done, |
102 length($total) => $done, |
102 $total; |
103 $total; |
103 alarm 5; |
104 alarm 5; |
111 # cut away the first part of the filename and |
112 # cut away the first part of the filename and |
112 # some optional extension |
113 # some optional extension |
113 (my $rn = $File::Find::name) =~ s/^$dir\/data\/(.*?)(?:\..+)?$/$1/; |
114 (my $rn = $File::Find::name) =~ s/^$dir\/data\/(.*?)(?:\..+)?$/$1/; |
114 exists $block{$rn} and return; |
115 exists $block{$rn} and return; |
115 |
116 |
|
117 |
116 if ($o{yes}) { |
118 if ($o{yes}) { |
117 verbose("unlinking abs_path $File::Find::name"); |
119 verbose("unlinking " . abs_path $File::Find::name); |
118 unlink abs_path $File::Find::name; |
120 unlink abs_path $File::Find::name; |
119 return; |
121 return; |
120 } |
122 } |
121 |
123 |
122 verbose("unused abs_path $File::Find::name"); |
124 verbose("unused " . abs_path $File::Find::name); |
123 return; |
125 return; |
124 |
126 |
125 }, "$dir/data"); |
127 }, "$dir/data"); |
126 $SIG{ALRM}->(); |
128 $SIG{ALRM}->(); |
127 alarm 0; |
129 alarm 0; |
129 } |
131 } |
130 |
132 |
131 sub check_images { |
133 sub check_images { |
132 my ($dir, %block) = @_; |
134 my ($dir, %block) = @_; |
133 |
135 |
134 my $total = keys(%block) - 1; # .idx |
136 my $total = grep { $_ ne "" } keys(%block); |
135 my $done = 0; |
137 my $done = 0; |
|
138 |
|
139 verbose("# pass 2 - check image completness"); |
136 |
140 |
137 # progress |
141 # progress |
138 local $SIG{ALRM} = sub { |
142 local $SIG{ALRM} = sub { |
139 return alarm 1 if not $done; |
143 return alarm 1 if not $done; |
140 my $speed = $done / (time - $^T + 1); |
144 my $speed = $done / (time - $^T + 1); |
141 say sprintf "# done %5.1f%% | %25s (%*d of %d files)", |
145 verbose sprintf "# pass 2 done %5.1f%% | %25s (%*d of %d blocks)", |
142 100 * $done/$total, |
146 100 * $done/$total, |
143 scalar(localtime($^T + ($total - $done) * $speed)), |
147 scalar(localtime($^T + ($total - $done) * $speed)), |
144 length($total) => $done, |
148 length($total) => $done, |
145 $total; |
149 $total; |
146 alarm 5; |
150 alarm 5; |
148 $SIG{ALRM}->(); |
152 $SIG{ALRM}->(); |
149 |
153 |
150 my %invalid; |
154 my %invalid; |
151 foreach my $k (keys %block) { |
155 foreach my $k (keys %block) { |
152 my $i = $block{$k}; |
156 my $i = $block{$k}; |
153 next if $k eq ".idx"; |
157 next if $k eq ""; |
154 ++$done; |
158 ++$done; |
155 |
159 |
156 next if -f "$dir/data/$k" |
160 next if -f "$dir/data/$k" |
157 or -f "$dir/data/$k.gz"; |
161 or -f "$dir/data/$k.gz"; |
158 say "missing $k"; |
162 say "missing $k"; |
159 @invalid{@{$block{".idx"}}} = (); |
163 @invalid{@{$block{""}}} = (); |
160 } |
164 } |
161 $SIG{ALRM}->(); |
165 $SIG{ALRM}->(); |
162 alarm 0; |
166 alarm 0; |
163 |
167 |
164 return if not %invalid; |
168 return if not %invalid; |