Support separate compress/decompress commands

Change kup-server so that separate compress/decompress commands can be
specified for each compressor. This allows us to use parallelizing
compressors such as pigz and pixz without impacting our ability to
decompress incoming tarballs.

Signed-off-by: Konstantin Ryabitsev <konstantin@linuxfoundation.org>

4 files changed, 64 insertions, 38 deletions

diff --git a/ChangeLog b/ChangeLog
index db969aa..8be93b9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -3,6 +3,11 @@
 	  kup is used with another authZ system relying on ssh (specifically,
 	  gitolite).
 	* Small typo fixes
+	* Change kup-server so that separate compress/decompress commands can be
+	  specified for each compressor. This allows us to use parallelizing
+	  compressors such as pigz and pixz without impacting our ability to
+	  decompress incoming tarballs.
+
 2012-12-10 Konstantin Ryabitsev <mricon@kernel.org> - 0.3.4
 	* Calculate and log sha256sums of all uploaded files for forensic
 	  record-keeping.
diff --git a/kup-server b/kup-server
index 02dd502..5c7f40d 100755
--- a/kup-server
+++ b/kup-server
@@ -134,20 +134,41 @@ my %zformats;
 
 if (defined($cfg->param('compressors.use'))) {
 
-    foreach my $zformat ($cfg->param('compressors.use')) {
-        # Do we have a path defined?
-        if (!defined($cfg->param("compressors.${zformat}"))) {
-            fatal("Compressor ${zformat} requested, but path not specified.");
-        }
-        my $ext = '.' . $zformat;
-        $zformats{$ext} = $cfg->param("compressors.${zformat}");
-    }
+	foreach my $zformat ($cfg->param('compressors.use')) {
+		my $ext = '.' . $zformat;
+		# Do we have a path defined?
+		if (!defined($cfg->param("compressors.${zformat}"))) {
+			# Do we have a section with compress/decompress commands?
+			if (!defined($cfg->param("${zformat}.compress_cmd"))) {
+				fatal("Compressor ${zformat} requested, but commands not specified.");
+			}
+			$zformats{$ext} = {
+				'compress'   => $cfg->param("${zformat}.compress_cmd"),
+				'decompress' => $cfg->param("${zformat}.decompress_cmd"),
+			}
+        } else {
+			# We have old-style definition, so just add -9 and -cd to it
+			$zformats{$ext} = {
+				'compress' => $cfg->param("compressors.${zformat}") . " -9",
+				'decompress' => $cfg->param("compressors.${zformat}") . " -cd",
+			}
+		}
+	}
 
 } else {
 	%zformats = (
-		'.gz'  => '/bin/gzip',
-		'.bz2' => '/usr/bin/bzip2',
-		'.xz'  => '/usr/bin/xz'
+		'.gz'  => {
+			'compress'   => '/bin/gzip -9',
+			'decompress' => '/bin/gzip -cd',
+		},
+		'.bz2' => {
+			'compress'   => '/usr/bin/bzip2 -9',
+			'decompress' => '/usr/bin/bzip2 -cd',
+		},
+		'.xz'  => {
+			'compress' => '/usr/bin/xz -9',
+			'decompress' => '/usr/bin/xz -cd',
+		},
 	);
 }
 
@@ -370,7 +391,7 @@ sub get_blob($$@)
 
 	if ($format eq '') {
 		undef $zcmd;
-	} elsif (!defined($zcmd = $zformats{'.'.$format})) {
+	} elsif (!defined($zcmd = $zformats{'.'.$format}{'decompress'})) {
 		fatal("Unsupported compression format");
 	}
 
@@ -387,12 +408,13 @@ sub get_blob($$@)
 	binmode($outfd);
 
 	if (defined($zcmd)) {
+		my @c = split(/\s+/, $zcmd);
 		open($oldstdout, '>&', \*STDOUT) or die;
 		open(STDOUT, '>&', $outfd) or die;
 		close($outfd);
 		undef $outfd;
 
-		open($outfd, '|-', $zcmd, '-cd') or die;
+		open($outfd, '|-', @c) or die;
 		binmode($outfd);
 
 		open(STDOUT, '>&', $oldstdout) or die;
@@ -615,11 +637,7 @@ sub make_compressed_data()
 	my $tarsize = -s $tmpdir.'/data';
 
 	foreach my $e (keys(%zformats)) {
-		my @c = ($zformats{$e}, '-9');
-		if ($zformats{$e} =~ /\s+/) {
-			@c = split(/\s+/, $zformats{$e});
-			push (@c, '-9');
-		}
+		my @c = split(/\s+/, $zformats{$e}{'compress'});
 
 		sysopen($infds{$e}, $tmpdir.'/data', O_RDONLY) or
 		    fatal("Failed to open uncompressed data file");
diff --git a/kup-server.1 b/kup-server.1
index 8888aee..2143090 100644
--- a/kup-server.1
+++ b/kup-server.1
@@ -98,23 +98,23 @@ Each compression command must take at most this long in CPU time.
 This section allows specifying the compressors to use when creating
 compressed versions of uploaded content.
 .TP
-\fBuse\fP = \fIgz, bz2, xz\fP
-A comma-separated list of file extensions to create (minus the leading dot). 
-For each extension specified, you will need to add an extra entry to this
-section with the path to the matching gzip-compatible utility (i.e. it 
-must accept \fI-9\fP and \fI-cd\fP command-line arguments). E.g., if you 
-specified "\fIgz, bz2, xz\fP" as values in \fBuse\fP, you must add the
-following entries as well:
+\fBuse\fP = \fIgz, xz\fP
+A comma-separated list of file extensions to create (minus the leading dot).
+For each extension specified, you will need to add a matching section specifying
+which command and flags to use for decompression and which for compression. Make
+sure to configure the decompress command to output to stdout. E.g.:
 .PP
 .RS
-.RS
 .nf
-gz = /bin/gzip
-bz2 = /usr/bin/bzip2
-xz = /usr/bin/xz
+[gz]
+compress_command = /bin/pigz -9
+decompress_command = /bin/gzip -cd
+
+[xz]
+compress_command = /bin/xz -9 -T0
+decompress_command = /bin/xz -cd
 .fi
 .RE
-.RE
 .SH AUTHOR
 Written by H. Peter Anvin <hpa@zytor.com>.
 .SH COPYRIGHT
diff --git a/kup-server.cfg b/kup-server.cfg
index 17b00e2..1ece762 100644
--- a/kup-server.cfg
+++ b/kup-server.cfg
@@ -55,11 +55,14 @@ timeout_compress_cpu = 900
 [compressors]
 ; Specify which compressors to use, separated by comma. These must match the
 ; file extensions that will be added to the compressed file (after the dot).
-use = gz, bz2, xz
-;
-; Specify paths to each compressor listed above. Each of these must accept
-; "-9" as commandline parameter for compression and "-cd" for decompression
-; to stdout.
-gz  = /bin/gzip
-bz2 = /usr/bin/bzip2
-xz  = /usr/bin/xz
+use = gz, xz
+
+; Specify which commands should be used for compression and decompression of
+; each archival format. Make sure the decompression command outputs to stdout.
+[gz]
+compress_command = /bin/pigz -9
+decompress_command = /bin/gzip -cd
+
+[xz]
+compress_command = /bin/xz -9 -T0
+decompress_command = /bin/xz -cd