Description: Use a single database (BM_UPLOADED_ARCHIVES) to track the
 uploaded archives, in order to avoid multiple uploads within a day.
Author: Georgios M. Zarkadas <gz@member.fsf.org>
Forwarded: https://github.com/sukria/Backup-Manager/pull/14
Accepted: https://github.com/sukria/Backup-Manager/commit/72b93f0f6223e4cf08bad4c1cb4d0ba4b4cfe38d
Last-Update: 2011-10-13

--- a/backup-manager-upload
+++ b/backup-manager-upload
@@ -108,6 +108,61 @@
 	}
 }
 
+# The idea behind BM_UPLOADED_ARCHIVES is to have a database of what archives
+# have been uploaded so far. This allows multiple execution of upload actions
+# within a day without resending all archives of the day from the beginning.
+
+# Add one file,host pair to $BM_UPLOADED_ARCHIVES database.
+# Called immediately *after* successful uploading of an archive.
+sub appendto_uploaded_archives($$)
+{
+    my $file = shift;
+    my $host = shift;
+    unless ( defined $file and defined $host ) {
+        print_error "required args needed";
+        return FALSE;
+    }
+
+    my $upload_fname = $ENV{BM_UPLOADED_ARCHIVES};
+    unless ( defined $upload_fname ) {
+        # Uncomment next line if you want the mandatory use
+        # of BM_UPLOADED_ARCHIVES (ie always have it around).
+        #print_error "BM_UPLOADED_ARCHIVES is not defined";
+        return FALSE;
+    }
+
+    # if $file already in database, append host to that line;
+    # else append a lines "$file $host" to the end.
+
+    my $io_error = 0;
+    if ( ! system( "grep -q \"^$file \" $upload_fname" ) ) {
+        my $cmd = "sed -i \"s:^$file .*\$:\& $host:\" $upload_fname";
+        $io_error = system("$cmd");
+    }
+    elsif ( open(my $fh, ">>", $upload_fname) ) {
+        print($fh "$file $host\n") or $io_error = 1;
+        close $fh;
+    }
+    else {
+        $io_error = 2;
+    }
+    if ( $io_error ) {
+        print_error "IO error: did not update $upload_fname with '$file $host'";
+        return FALSE;
+    }
+
+    return TRUE;
+}
+
+# Get all files of the specified date; filter the list through 
+# BM_UPLOADED_ARCHIVES if it is set in the environment.
+# NOTE:  Doing the filtering here implies that the archive is considered
+# uploaded if a single upload to a host succeeds; that is even when there
+# are failures to other hosts (in case of multiple host uploading).
+# To consider it uploaded when all hosts succeed, the filtering must be
+# transfered to the individual upload subroutines (and check for existence
+# of file,host pair in the database).
+#
 sub get_files_list_from_date($)
 {
 	my $date = shift;
@@ -132,8 +187,21 @@
         exit E_INVALID;
     }
 
-	while (<$g_root_dir/*$date*>) {
-        push @{$ra_files}, $_;
+    my $upload_fname = $ENV{BM_UPLOADED_ARCHIVES};
+    if ( defined $upload_fname ) {
+        # filter file list through the BM_UPLOADED_ARCHIVES database
+    	while (<$g_root_dir/*$date*>) {
+            my $file = $_;
+            my $cmd = "grep -q '$file' $upload_fname";
+            if ( system ("$cmd") ) {
+                push @{$ra_files}, $file;
+            }
+        }
+    }
+    else {
+    	while (<$g_root_dir/*$date*>) {
+            push @{$ra_files}, $_;
+        }
 	}
 
 	return $ra_files;
@@ -270,7 +338,12 @@
 		print_error ("Unable to upload \"$file\". ".($! || $@ || $ret));
         return 0;
 	}
-	return 1;
+    else {
+        # use same name in both cases (gpg encryption is done on the fly);
+        # continue if writing to uploaded archives file fails.
+        appendto_uploaded_archives($file, $host);
+    }
+    return 1;
 }
 
 # How to upload files with scp.
@@ -644,22 +717,25 @@
         # Put all the files over the connexion
         foreach my $file (@{$ra_files}) {
             chomp $file;
+            # continue if writing to uploaded archives file fails.
             if ($BM_UPLOAD_FTP_SECURE) {
-                if (!ftptls_put_file ($ftp, $file)) {
-		    print_error "Unable to transfer $file";
-		    return FALSE;
+                if (ftptls_put_file ($ftp, $file)) {
+                    appendto_uploaded_archives($file, $host);
+                    print_info "File $file transfered\n";
                 }
                 else {
-			print_info "File $file transfered\n";
+                    print_error "Unable to transfer $file";
+                    return FALSE;
                 }
             }
             else {
-                if (!ftp_put_file ($ftp, $file)) {
-		    print_error "Unable to transfer $file: " . $ftp->message;
-		    return FALSE;
+                if (ftp_put_file ($ftp, $file)) {
+                    appendto_uploaded_archives($file, $host);
+                    print_info "File $file transfered\n";
                 }
                 else {
-            print_info "File $file transfered\n";
+                    print_error "Unable to transfer $file: " . $ftp->message;
+                    return FALSE;
                 }
             }
         }
@@ -847,6 +923,8 @@
 				);
 				$uploaded{$filename} = $file_length;
 			}
+            # For the S3 method, we assume success in any case.
+            appendto_uploaded_archives($file, $host);
 		}
 
 		# get a list of files and confirm uploads
--- a/backup-manager.conf.tpl
+++ b/backup-manager.conf.tpl
@@ -332,6 +332,20 @@
 # Where to put archives on the remote hosts (global)
 export BM_UPLOAD_DESTINATION=""
 
+# Uncomment the 'export ...' line below to activate the uploaded archives
+# database.
+# Using the database will avoid extraneous uploads to remote hosts in the
+# case of running more than one backup-manager jobs per day (such as when
+# you are using different configuration files for different parts of your
+# filesystem).
+# Note that when you upload to multiple hosts, a single succesfull upload
+# will mark the archive as uploaded. Thus upload errors to specific hosts
+# will have to be resolved manually.
+# You can specify any filename, but it is recommended to keep the database
+# inside the archive repository. The variable's value has been preset to
+# that.
+#export BM_UPLOADED_ARCHIVES=${BM_REPOSITORY_ROOT}/${BM_ARCHIVE_PREFIX}-uploaded.list
+
 ##############################################################
 # The SSH method
 #############################################################