--- a/bin/amdumpext Wed Dec 11 23:41:09 2013 +0100
+++ b/bin/amdumpext Mon Dec 16 11:13:32 2013 +0100
@@ -1,13 +1,34 @@
#! /usr/bin/perl
+# (c) 2013 Heiko Schlittermann <hs@schlittermann.de>
+# source: hg clone https://ssl.schlittermann.de/amanda-plugin-dumpext
+#
+# This script should be a plugin to use ext2/3/4 dump/restore via the
+# APPLICATION interface of Amanda. The rationale behind is, that I'd like to use
+# dump(8) with different dumpdates files for the different backup types
+# (daily, weekly, …)
+#
+# The commands we need to support are required by the
+# API: http://wiki.zmanda.com/index.php/Application_API/Operations
+#
+# This script tries do be as standalone as possible.
+# Though we need the tools dump/restore as they exists for the ext2/3/4
+# filesystems.
+
use 5.010;
use strict;
use warnings;
use Pod::Usage;
use Getopt::Long;
-use Readonly;
-use DDP;
+use File::Basename;
+use POSIX;
+
+#use Readonly;
our $VERSION = '0.01';
+my $ME = basename $0;
+
+# to avoid stupid "not found"
+$ENV{PATH} .= ':/usr/local/sbin:/usr/sbin:/sbin';
use constant YES => 'YES';
use constant NO => 'NO';
@@ -15,22 +36,23 @@
use constant FD3 => 3;
use constant FD4 => 4;
-Readonly my %SUPPORT => (
- CONFIG => YES, # --config … (ignored?)
- HOST => YES, # --host … (ignored?)
- DISK => YES, # --disk … (ignored?)
- MAX_LEVEL => 9,
+$SIG{__DIE__} = sub { die "$ME: ", @_ };
+
+my %SUPPORT = (
+ CONFIG => YES, # --config … (default)
+ DISK => NO, # --disk …
+ HOST => NO, # --host …
+ MAX_LEVEL => 9, # --level …
+ INDEX_LINE => YES, # --index line
+ MESSAGE_LINE => YES, # --message line
+ #
CLIENT_ESTIMATE => YES, # estimate
MULTI_ESTIMATE => YES, # estimate for multiple levels
- CALCSIZE => YES, # estimate --calcsize
- MESSAGE_LINE => YES, # --message line
- INDEX_LINE => NO, # --index line
+ CALCSIZE => NO, # estimate --calcsize
+ #
RECORD => YES, # --record
);
-# the commands we need to support as required by the
-# API: http://wiki.zmanda.com/index.php/Application_API/Operations
-
sub exec_support;
sub exec_selfcheck;
sub exec_estimate;
@@ -44,15 +66,12 @@
# bad but common style - the global options
-my $opt_config; # $config
-my $opt_host; # $host
-my $opt_disk; # $disk DLE[1]
-my $opt_device; # $device DLE[2]
-my $opt_message; # line / <>
-my $opt_index; # line / <>
-my $opt_record; # true / <>
-my $opt_level; # 0…99
-my $opt_calcsize; # true / <>
+my $opt_config; # $config
+my $opt_device; # $device DLE[2]
+my $opt_message; # line / <>
+my $opt_index; # line / <>
+my $opt_record; # true / <>
+my @opt_level; # 0…99
my $opt_dumpdates;
@@ -60,52 +79,57 @@
my @argv = @ARGV;
my $command = shift // pod2usage;
GetOptions(
-
- 'config=s' => \$opt_config,
- 'host=s' => \$opt_host, # --host $host
- 'disk=s' => \$opt_disk, # --disk $disk
- 'device=s' => \$opt_device, # --device $device
- 'message=s' => \$opt_message, # --message line|xml
- 'index=s' => \$opt_index, # --index line
- 'record!' => \$opt_record, # --record
- 'level=i@' => \$opt_level, # --level n
- 'calcsize!' => \$opt_calcsize,
-
+ 'config=s' => \$opt_config,
+ 'device=s' => \$opt_device, # --device $device
+ 'message=s' => \$opt_message, # --message line|xml
+ 'index=s' => \$opt_index, # --index line
+ 'record!' => \$opt_record, # --record
+ 'level=i@' => \@opt_level, # --level n
'dumpdates=s' => \$opt_dumpdates, # --dumpdates <file>
+ 'host=s' => sub { }, # ignore
+ 'disk=s' => sub { }, # ignore
) or pod2usage;
given ($command) {
when ("support") { exec_support }
when ("selfcheck") {
- pod2usage if not defined $opt_device;
+ pod2usage if undef ~~ $opt_device;
exec_selfcheck
}
when ("estimate") {
- pod2usage
- if not defined $opt_device
- or not defined $opt_level;
+ pod2usage if undef ~~ [$opt_device, $opt_level[0]];
exec_estimate
}
- when ("backup") { exec_backup }
- default { pod2usage }
+ when ("backup") {
+ pod2usage if undef ~~ [$opt_device, $opt_level[0]];
+ exec_backup
+ }
+ default { pod2usage }
}
}
# output a list of supported options
sub exec_support {
- print map { "$_ $SUPPORT{$_}\n" =~ s/_/-/gr } keys %SUPPORT;
+ print map { "$_ $SUPPORT{$_}\n" =~ s/_/-/gr } sort keys %SUPPORT;
exit 0;
}
sub exec_selfcheck {
+
# must: $opt_device
# may: $opt_level
- if ($opt_level and ref $opt_level) { $opt_level = $opt_level->[0] }
+
+ OK "$ME version $VERSION";
+ OK "euid=$> (" . getpwuid($>) . ')';
+ OK "egid=$) (" . join(', ' => map { '' . getgrgid $_ } split ' ' => $)) . ')';
if ($_ = (grep { -x ($_ .= "/dump") } split /:/ => $ENV{PATH})[0]) {
- OK "dump is \"$_\"";
+ chomp(my $version = (`$_ 2>&1`)[0]);
+ OK "dump is $version";
}
- else { say "ERROR dump not found in $ENV{PATH}\n" }
+ else {
+ ERROR "dump not found in $ENV{PATH}";
+ }
# check the device
# the opt_disk is just a label, the device is in opt_device!
@@ -133,40 +157,44 @@
# may: $opt_record, $opt_dumpdates
my (@errors, @results);
- foreach my $level (@$opt_level) {
+ foreach my $level (@opt_level) {
my @cmd = (
dump => "-$level",
- '-S',
+ '-S', # estimate
$opt_record && $opt_dumpdates ? (-D => expand($opt_dumpdates)) : (),
device($opt_device),
);
- chomp(my @output = `@cmd 2>&1`);
-
- if ($?) {
- say "unexpected output:\n",
- join "\n" => @output;
- exit 1;
- }
+ my @output = `@cmd 2>&1`;
- # the last line should be the number of 1K blocks
- my $blocks = do {
- my $_ = pop @output;
- /^(\d+)/ or do {
- say "can't get estimate";
- exit 1;
- };
- $1 / 1024;
- };
+ given ($?) {
+ when (-1) { say "command not found: $cmd[0]" }
+ when ($_ > 0) {
+ my $rc = ($? & 0xffff) >> 8;
+ my $sig = ($? & 0xff);
+ say
+"unexpected return code (exit: $rc, signal: $sig) from `@cmd':\n",
+ join "\n" => @output;
+ exit 1;
+ }
+ }
+ chomp @output;
+
+ # the last line should be the number of 1K blocks
+ my $blocks = do {
+ my $_ = pop @output;
+ /^(\d+)/ or do {
+ say "can't get estimate";
+ exit 1;
+ };
+ $1 / 1024;
+ };
# level blocks blocksize
- # --> the blocksize unit is K
- push @errors, @output, "---" if @output;
- push @results, "$level $blocks 1";
+ say join "\n" => @output if @output;
+ say "$level $blocks 1";
}
- say join "\n", @errors if @errors;
- say join "\n", @results;
exit 0;
}
@@ -177,50 +205,67 @@
# fd4: index channel
my @dump = (
- dump => "-$opt_level",
- -f => "-",
- $opt_record ? "-u" : (),
+ dump => "-$opt_level[0]",
+ #'-v', # verbose
+ -f => '-',
+ $opt_record ? '-u' : (),
$opt_record && $opt_dumpdates ? (-D => expand($opt_dumpdates)) : (),
device($opt_device)
);
- # messages ----------,
- # ,---------> fd2 ----> fd3
- # dump --o----> fd1 (data)
+ # ,---------> fd3 ----> (messages)
+ # dump --o----> fd1 ----> (data)
# `---> restore -t --> fd4 (index)
- open(my $msg, ">&=", FD3) or die "Can't open fd3: $!\n";
- open(my $idx, ">&=", FD4) or die "Can't open fd4: $!\n" if $opt_index;
+ open(my $msg, '>&=', FD3) or die "Can't open fd3: $!\n";
+ open(my $idx, '>&=', FD4) or die "Can't open fd4: $!\n" if $opt_index;
if ($opt_index) {
my $pid = fork // die "Can't fork: $!\n";
if (not $pid) {
- open(STDOUT, "|-") or do {
- open(my $restore, "|-") or do {
- open(STDOUT, "|-") or do {
- select($idx);
- postprocess_toc();
+ $0 = "$ME [about to exec dump]";
+
+ # dump will be execed soon, first we've to establish
+ # the channels - one for STDOUT, and one for STDIN
+ open(STDOUT, '|-') or do {
+ # this is the child that will read
+ # the STDOUT from dump
+ $0 = "$ME [stdout < dump]";
+
+ my $pid = open(my $restore, '|-') or do {
+ $0 = "$ME [toc]";
+ open(STDOUT, '|-') or do {
+ postprocess_toc($idx);
exit 0;
};
- exec "restore", "-tvf" => "-";
+ exec 'restore', -tvf => '-';
die "Can't exec `restore -tvf -`: $!";
};
- local $/ = 2**16;
+
+ local $/ = \(my $x = 64 * 1024);
while (<STDIN>) {
print $_;
print $restore $_;
}
+ close($restore);
+ exit 0;
+ };
+
+ open(STDERR, '|-') or do {
+ $0 = "$ME [stderr < dump]";
+ postprocess_dump_messages($msg);
exit 0;
};
- open(STDERR, "|-") or do {
- select($msg);
- postprocess_dump_messages();
- exit 0;
- };
+ # we need to fork again, otherwise dump sees
+ # the end of the above children and complains
+ my $pid = fork // die "Can't fork: $!\n";
+ if (not $pid) {
+ exec @dump;
+ die "Can't exec `@dump': $!\n";
+ }
- exec @dump;
- die "Can't exec `@dump`: $!\n";
+ waitpid($pid, 0);
}
waitpid($pid, 0);
@@ -228,11 +273,19 @@
}
# no need to send an index
+ # dump [2] --- (postprocess_dump_messages) --> [fd3]
+ # [1] ----------------------------------> [fd1]
+
my $pid = fork // die "Can't fork: $!\n";
+
+ # child does all the work
if (not $pid) {
- open(STDERR, "|-") or do {
- select($msg);
- postprocess_dump_messages();
+
+ # create the subprocess that will read the
+ # stderr output from dump, convert it and send it
+ # to the message channel
+ open(STDERR, '|-') or do {
+ postprocess_dump_messages($msg);
exit 0;
};
exec @dump;
@@ -243,47 +296,79 @@
}
-sub postprocess_dump_messages() {
+sub postprocess_dump_messages {
+
+ select +shift; # send output to the message channel
+
while (<STDIN>) {
print "| $_";
-
if (/^\s+DUMP: (\d+) blocks?/) {
-
# we assume a block size of 1K
say "sendbackup: size $1";
}
elsif (/^\s+DUMP: DUMP IS DONE/) {
- say "sendbackup: end";
+ say 'sendbackup: end';
}
}
}
sub postprocess_toc {
- # dir 4711 ./aaa
- # leaf 4712 ./bbb/xxx
- # leaf 4713 ./bbb/a
+ # the output of restore -tv looks
+ # about like this:
+ #
+ # dir 4711 ./aaa
+ # leaf 4712 ./bbb/xxx
+ # leaf 4713 ./bbb/a
# b
- # leaf 8819 ./bbb/x
+ # leaf 8819 ./bbb/x
+ #
+ # it may break if there is a lf/cr
+ # embedded in the filename
+ #
+ # the more generic solution would be to force
+ # restore to use a \0 separated output format
- my $name;
+ select +shift;
+ local $/ = "\n"; # make sure to have it line separated!
+
+ my $buffer = undef;
+ my $type = undef;
+
+ while (1) {
- while (<STDIN>) {
- chomp;
- if (/^(dir|leaf)\s+\d+\s+(\.\/.*)/) {
- say $name if defined $name;
- $name = $2 . ($1 eq "dir" ? "/" : "");
+ $_ = <STDIN>;
+
+ # skip the header lines
+ if (1 .. defined && /\Adir\s+\d+\s+(.*)\Z/) {
+ $buffer = '';
+ $type = 'dir';
+ die "Unexpected end of input\n" if not defined;
next;
}
- if ($name) {
- $name .= $_;
+ # if we match really good the buffer may be output
+ if (not defined
+ or chomp and /\A(?'type' dir|leaf)\s+\d+\s+\.(?'name' \/.*)\Z/x)
+ {
+
+ # output
+ say $buffer . ($type eq 'dir' ? '/' : '');
+
+ # we're done if this was the last line of output
+ last if not defined;
+
+ # order matters, do not exchange the next two lines! The %+
+ # will break
+ $type = $+{type};
+ $buffer = $+{name} =~ s/\\/\\\\/gr;
+
next;
}
- }
+ $buffer .= "\\n$_";
- say $name if defined $name;
+ }
}
@@ -299,8 +384,8 @@
return $_;
}
-sub OK { say "OK ", @_ }
-sub ERROR { say "ERROR ", @_ }
+sub OK { say "OK @_" }
+sub ERROR { say "ERROR [@_]" }
=head1 NAME
@@ -378,4 +463,15 @@
=back
+=head1 EXAMPLE
+
+ define application "dump" {
+ plugin "amdumpext"
+ # optional - define some additional parameters
+ property "dumpdates" "/tmp/dumpdates.${c}"
+ }
+
+
+=cut
+
# vim:sts=4 sw=4 aw ai sm: