# HG changeset patch # User Heiko Schlittermann (JUMPER) # Date 1387188812 -3600 # Node ID 5488aa9488af8cb63e560b4fed2c0634daeff534 # Parent bdf6e224ffe62858e97e6a8ea3a21de5d0dcf89d [snapshot] diff -r bdf6e224ffe6 -r 5488aa9488af bin/amdumpext --- a/bin/amdumpext Wed Dec 11 23:41:09 2013 +0100 +++ b/bin/amdumpext Mon Dec 16 11:13:32 2013 +0100 @@ -1,13 +1,34 @@ #! /usr/bin/perl +# (c) 2013 Heiko Schlittermann +# source: hg clone https://ssl.schlittermann.de/amanda-plugin-dumpext +# +# This script should be a plugin to use ext2/3/4 dump/restore via the +# APPLICATION interface of Amanda. The rationale behind is, that I'd like to use +# dump(8) with different dumpdates files for the different backup types +# (daily, weekly, …) +# +# The commands we need to support are required by the +# API: http://wiki.zmanda.com/index.php/Application_API/Operations +# +# This script tries do be as standalone as possible. +# Though we need the tools dump/restore as they exists for the ext2/3/4 +# filesystems. + use 5.010; use strict; use warnings; use Pod::Usage; use Getopt::Long; -use Readonly; -use DDP; +use File::Basename; +use POSIX; + +#use Readonly; our $VERSION = '0.01'; +my $ME = basename $0; + +# to avoid stupid "not found" +$ENV{PATH} .= ':/usr/local/sbin:/usr/sbin:/sbin'; use constant YES => 'YES'; use constant NO => 'NO'; @@ -15,22 +36,23 @@ use constant FD3 => 3; use constant FD4 => 4; -Readonly my %SUPPORT => ( - CONFIG => YES, # --config … (ignored?) - HOST => YES, # --host … (ignored?) - DISK => YES, # --disk … (ignored?) - MAX_LEVEL => 9, +$SIG{__DIE__} = sub { die "$ME: ", @_ }; + +my %SUPPORT = ( + CONFIG => YES, # --config … (default) + DISK => NO, # --disk … + HOST => NO, # --host … + MAX_LEVEL => 9, # --level … + INDEX_LINE => YES, # --index line + MESSAGE_LINE => YES, # --message line + # CLIENT_ESTIMATE => YES, # estimate MULTI_ESTIMATE => YES, # estimate for multiple levels - CALCSIZE => YES, # estimate --calcsize - MESSAGE_LINE => YES, # --message line - INDEX_LINE => NO, # --index line + CALCSIZE => NO, # estimate --calcsize + # RECORD => YES, # --record ); -# the commands we need to support as required by the -# API: http://wiki.zmanda.com/index.php/Application_API/Operations - sub exec_support; sub exec_selfcheck; sub exec_estimate; @@ -44,15 +66,12 @@ # bad but common style - the global options -my $opt_config; # $config -my $opt_host; # $host -my $opt_disk; # $disk DLE[1] -my $opt_device; # $device DLE[2] -my $opt_message; # line / <> -my $opt_index; # line / <> -my $opt_record; # true / <> -my $opt_level; # 0…99 -my $opt_calcsize; # true / <> +my $opt_config; # $config +my $opt_device; # $device DLE[2] +my $opt_message; # line / <> +my $opt_index; # line / <> +my $opt_record; # true / <> +my @opt_level; # 0…99 my $opt_dumpdates; @@ -60,52 +79,57 @@ my @argv = @ARGV; my $command = shift // pod2usage; GetOptions( - - 'config=s' => \$opt_config, - 'host=s' => \$opt_host, # --host $host - 'disk=s' => \$opt_disk, # --disk $disk - 'device=s' => \$opt_device, # --device $device - 'message=s' => \$opt_message, # --message line|xml - 'index=s' => \$opt_index, # --index line - 'record!' => \$opt_record, # --record - 'level=i@' => \$opt_level, # --level n - 'calcsize!' => \$opt_calcsize, - + 'config=s' => \$opt_config, + 'device=s' => \$opt_device, # --device $device + 'message=s' => \$opt_message, # --message line|xml + 'index=s' => \$opt_index, # --index line + 'record!' => \$opt_record, # --record + 'level=i@' => \@opt_level, # --level n 'dumpdates=s' => \$opt_dumpdates, # --dumpdates + 'host=s' => sub { }, # ignore + 'disk=s' => sub { }, # ignore ) or pod2usage; given ($command) { when ("support") { exec_support } when ("selfcheck") { - pod2usage if not defined $opt_device; + pod2usage if undef ~~ $opt_device; exec_selfcheck } when ("estimate") { - pod2usage - if not defined $opt_device - or not defined $opt_level; + pod2usage if undef ~~ [$opt_device, $opt_level[0]]; exec_estimate } - when ("backup") { exec_backup } - default { pod2usage } + when ("backup") { + pod2usage if undef ~~ [$opt_device, $opt_level[0]]; + exec_backup + } + default { pod2usage } } } # output a list of supported options sub exec_support { - print map { "$_ $SUPPORT{$_}\n" =~ s/_/-/gr } keys %SUPPORT; + print map { "$_ $SUPPORT{$_}\n" =~ s/_/-/gr } sort keys %SUPPORT; exit 0; } sub exec_selfcheck { + # must: $opt_device # may: $opt_level - if ($opt_level and ref $opt_level) { $opt_level = $opt_level->[0] } + + OK "$ME version $VERSION"; + OK "euid=$> (" . getpwuid($>) . ')'; + OK "egid=$) (" . join(', ' => map { '' . getgrgid $_ } split ' ' => $)) . ')'; if ($_ = (grep { -x ($_ .= "/dump") } split /:/ => $ENV{PATH})[0]) { - OK "dump is \"$_\""; + chomp(my $version = (`$_ 2>&1`)[0]); + OK "dump is $version"; } - else { say "ERROR dump not found in $ENV{PATH}\n" } + else { + ERROR "dump not found in $ENV{PATH}"; + } # check the device # the opt_disk is just a label, the device is in opt_device! @@ -133,40 +157,44 @@ # may: $opt_record, $opt_dumpdates my (@errors, @results); - foreach my $level (@$opt_level) { + foreach my $level (@opt_level) { my @cmd = ( dump => "-$level", - '-S', + '-S', # estimate $opt_record && $opt_dumpdates ? (-D => expand($opt_dumpdates)) : (), device($opt_device), ); - chomp(my @output = `@cmd 2>&1`); - - if ($?) { - say "unexpected output:\n", - join "\n" => @output; - exit 1; - } + my @output = `@cmd 2>&1`; - # the last line should be the number of 1K blocks - my $blocks = do { - my $_ = pop @output; - /^(\d+)/ or do { - say "can't get estimate"; - exit 1; - }; - $1 / 1024; - }; + given ($?) { + when (-1) { say "command not found: $cmd[0]" } + when ($_ > 0) { + my $rc = ($? & 0xffff) >> 8; + my $sig = ($? & 0xff); + say +"unexpected return code (exit: $rc, signal: $sig) from `@cmd':\n", + join "\n" => @output; + exit 1; + } + } + chomp @output; + + # the last line should be the number of 1K blocks + my $blocks = do { + my $_ = pop @output; + /^(\d+)/ or do { + say "can't get estimate"; + exit 1; + }; + $1 / 1024; + }; # level blocks blocksize - # --> the blocksize unit is K - push @errors, @output, "---" if @output; - push @results, "$level $blocks 1"; + say join "\n" => @output if @output; + say "$level $blocks 1"; } - say join "\n", @errors if @errors; - say join "\n", @results; exit 0; } @@ -177,50 +205,67 @@ # fd4: index channel my @dump = ( - dump => "-$opt_level", - -f => "-", - $opt_record ? "-u" : (), + dump => "-$opt_level[0]", + #'-v', # verbose + -f => '-', + $opt_record ? '-u' : (), $opt_record && $opt_dumpdates ? (-D => expand($opt_dumpdates)) : (), device($opt_device) ); - # messages ----------, - # ,---------> fd2 ----> fd3 - # dump --o----> fd1 (data) + # ,---------> fd3 ----> (messages) + # dump --o----> fd1 ----> (data) # `---> restore -t --> fd4 (index) - open(my $msg, ">&=", FD3) or die "Can't open fd3: $!\n"; - open(my $idx, ">&=", FD4) or die "Can't open fd4: $!\n" if $opt_index; + open(my $msg, '>&=', FD3) or die "Can't open fd3: $!\n"; + open(my $idx, '>&=', FD4) or die "Can't open fd4: $!\n" if $opt_index; if ($opt_index) { my $pid = fork // die "Can't fork: $!\n"; if (not $pid) { - open(STDOUT, "|-") or do { - open(my $restore, "|-") or do { - open(STDOUT, "|-") or do { - select($idx); - postprocess_toc(); + $0 = "$ME [about to exec dump]"; + + # dump will be execed soon, first we've to establish + # the channels - one for STDOUT, and one for STDIN + open(STDOUT, '|-') or do { + # this is the child that will read + # the STDOUT from dump + $0 = "$ME [stdout < dump]"; + + my $pid = open(my $restore, '|-') or do { + $0 = "$ME [toc]"; + open(STDOUT, '|-') or do { + postprocess_toc($idx); exit 0; }; - exec "restore", "-tvf" => "-"; + exec 'restore', -tvf => '-'; die "Can't exec `restore -tvf -`: $!"; }; - local $/ = 2**16; + + local $/ = \(my $x = 64 * 1024); while () { print $_; print $restore $_; } + close($restore); + exit 0; + }; + + open(STDERR, '|-') or do { + $0 = "$ME [stderr < dump]"; + postprocess_dump_messages($msg); exit 0; }; - open(STDERR, "|-") or do { - select($msg); - postprocess_dump_messages(); - exit 0; - }; + # we need to fork again, otherwise dump sees + # the end of the above children and complains + my $pid = fork // die "Can't fork: $!\n"; + if (not $pid) { + exec @dump; + die "Can't exec `@dump': $!\n"; + } - exec @dump; - die "Can't exec `@dump`: $!\n"; + waitpid($pid, 0); } waitpid($pid, 0); @@ -228,11 +273,19 @@ } # no need to send an index + # dump [2] --- (postprocess_dump_messages) --> [fd3] + # [1] ----------------------------------> [fd1] + my $pid = fork // die "Can't fork: $!\n"; + + # child does all the work if (not $pid) { - open(STDERR, "|-") or do { - select($msg); - postprocess_dump_messages(); + + # create the subprocess that will read the + # stderr output from dump, convert it and send it + # to the message channel + open(STDERR, '|-') or do { + postprocess_dump_messages($msg); exit 0; }; exec @dump; @@ -243,47 +296,79 @@ } -sub postprocess_dump_messages() { +sub postprocess_dump_messages { + + select +shift; # send output to the message channel + while () { print "| $_"; - if (/^\s+DUMP: (\d+) blocks?/) { - # we assume a block size of 1K say "sendbackup: size $1"; } elsif (/^\s+DUMP: DUMP IS DONE/) { - say "sendbackup: end"; + say 'sendbackup: end'; } } } sub postprocess_toc { - # dir 4711 ./aaa - # leaf 4712 ./bbb/xxx - # leaf 4713 ./bbb/a + # the output of restore -tv looks + # about like this: + # + # dir 4711 ./aaa + # leaf 4712 ./bbb/xxx + # leaf 4713 ./bbb/a # b - # leaf 8819 ./bbb/x + # leaf 8819 ./bbb/x + # + # it may break if there is a lf/cr + # embedded in the filename + # + # the more generic solution would be to force + # restore to use a \0 separated output format - my $name; + select +shift; + local $/ = "\n"; # make sure to have it line separated! + + my $buffer = undef; + my $type = undef; + + while (1) { - while () { - chomp; - if (/^(dir|leaf)\s+\d+\s+(\.\/.*)/) { - say $name if defined $name; - $name = $2 . ($1 eq "dir" ? "/" : ""); + $_ = ; + + # skip the header lines + if (1 .. defined && /\Adir\s+\d+\s+(.*)\Z/) { + $buffer = ''; + $type = 'dir'; + die "Unexpected end of input\n" if not defined; next; } - if ($name) { - $name .= $_; + # if we match really good the buffer may be output + if (not defined + or chomp and /\A(?'type' dir|leaf)\s+\d+\s+\.(?'name' \/.*)\Z/x) + { + + # output + say $buffer . ($type eq 'dir' ? '/' : ''); + + # we're done if this was the last line of output + last if not defined; + + # order matters, do not exchange the next two lines! The %+ + # will break + $type = $+{type}; + $buffer = $+{name} =~ s/\\/\\\\/gr; + next; } - } + $buffer .= "\\n$_"; - say $name if defined $name; + } } @@ -299,8 +384,8 @@ return $_; } -sub OK { say "OK ", @_ } -sub ERROR { say "ERROR ", @_ } +sub OK { say "OK @_" } +sub ERROR { say "ERROR [@_]" } =head1 NAME @@ -378,4 +463,15 @@ =back +=head1 EXAMPLE + + define application "dump" { + plugin "amdumpext" + # optional - define some additional parameters + property "dumpdates" "/tmp/dumpdates.${c}" + } + + +=cut + # vim:sts=4 sw=4 aw ai sm: