e_tpm-rsa.c: add additional padding types

Up to now we've only handled PKCS1 padding, but since this is being
deprecated by NIST, we need to handle all the others.  It turns out
that the RSA layer of openssl can only really handle OAEP with
mgf1(sha1) or unpadded, so add both of those to the engine.  All other
sophisticated padding in openssl is handled at the pmeth layer and
thus only requires an unpadded key operation at the RSA layer, which
we've now added, so the tpm2 engine should now work for any padding
type supported by openssl.

Signed-off-by: James Bottomley <James.Bottomley@HansenPartnership.com>

3 files changed, 58 insertions, 17 deletions

diff --git a/e_tpm2-rsa.c b/e_tpm2-rsa.c
index 3be6302..fe82bc6 100644
--- a/e_tpm2-rsa.c
+++ b/e_tpm2-rsa.c
@@ -180,12 +180,19 @@ static int tpm2_rsa_priv_dec(int flen,
 	}
 
 	rv = -1;
-	if (padding != RSA_PKCS1_PADDING) {
-		fprintf(stderr, "Non PKCS1 padding asked for\n");
+	if (padding == RSA_PKCS1_PADDING) {
+		in.inScheme.scheme = TPM_ALG_RSAES;
+	} else if (padding == RSA_NO_PADDING) {
+		in.inScheme.scheme = TPM_ALG_NULL;
+	} else if (padding == RSA_PKCS1_OAEP_PADDING) {
+		in.inScheme.scheme = TPM_ALG_OAEP;
+		/* for openssl RSA, the padding is hard coded */
+		in.inScheme.details.oaep.hashAlg = TPM_ALG_SHA1;
+	} else {
+		fprintf(stderr, "Can't process padding type: %d\n", padding);
 		goto out;
 	}
 
-	in.inScheme.scheme = TPM_ALG_RSAES;
 	in.cipherText.t.size = flen;
 	memcpy(in.cipherText.t.buffer, from, flen);
 	in.label.t.size = 0;
@@ -243,8 +250,28 @@ static int tpm2_rsa_priv_enc(int flen,
 	struct policy_command *commands;
 	TPM_ALG_ID nameAlg;
 
-	if (padding != RSA_PKCS1_PADDING) {
-		fprintf(stderr, "Non PKCS1 padding asked for\n");
+	/* this is slightly paradoxical that we're doing a Decrypt
+	 * operation: the only material difference between decrypt and
+	 * encrypt is where the padding is applied or checked, so if
+	 * you apply your own padding up to the RSA block size and use
+	 * TPM_ALG_NULL, which means no padding check, a decrypt
+	 * operation effectively becomes an encrypt */
+	size = RSA_size(rsa);
+	in.inScheme.scheme = TPM_ALG_NULL;
+	in.cipherText.t.size = size;
+	in.label.t.size = 0;
+
+	/* note: currently openssl doesn't do OAEP signatures and all
+	 * PSS signatures are padded and handled in the RSA layer
+	 * as a no-padding private encryption */
+	if (padding == RSA_PKCS1_PADDING) {
+		RSA_padding_add_PKCS1_type_1(in.cipherText.t.buffer, size,
+					     from, flen);
+	} else if (padding == RSA_NO_PADDING) {
+		/* do nothing, we're already doing a no padding encrypt */
+		memcpy(in.cipherText.t.buffer, from, size);
+	} else {
+		fprintf(stderr, "Can't process padding type: %d\n", padding);
 		return -1;
 	}
 
@@ -271,18 +298,6 @@ static int tpm2_rsa_priv_enc(int flen,
 			goto out;
 	}
 
-	/* this is slightly paradoxical that we're doing a Decrypt
-	 * operation: the only material difference between decrypt and
-	 * encrypt is where the padding is applied or checked, so if
-	 * you apply your own padding up to the RSA block size and use
-	 * TPM_ALG_NULL, which means no padding check, a decrypt
-	 * operation effectively becomes an encrypt */
-	size = RSA_size(rsa);
-	in.inScheme.scheme = TPM_ALG_NULL;
-	in.cipherText.t.size = size;
-	RSA_padding_add_PKCS1_type_1(in.cipherText.t.buffer, size, from, flen);
-	in.label.t.size = 0;
-
 	rc = TSS_Execute(tssContext,
 			 (RESPONSE_PARAMETERS *)&out,
 			 (COMMAND_PARAMETERS *)&in,
diff --git a/tests/Makefile.am b/tests/Makefile.am
index 0294dd0..d9cb3b8 100644
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -18,6 +18,7 @@ TESTS = fail_connect.sh \
 	check_enhanced_auth.sh \
 	check_counter_timer.sh \
 	check_importable.sh \
+	check_rsa_oaep_pss.sh \
 	stop_sw_tpm.sh
 
 AM_TESTS_ENVIRONMENT = TPM_INTERFACE_TYPE=socsim; \
diff --git a/tests/check_rsa_oaep_pss.sh b/tests/check_rsa_oaep_pss.sh
new file mode 100755
index 0000000..76793a6
--- /dev/null
+++ b/tests/check_rsa_oaep_pss.sh
@@ -0,0 +1,25 @@
+#!/bin/bash
+
+bindir=${srcdir}/..
+
+openssl genrsa 2048 > key.priv || exit 1
+openssl rsa -in key.priv -out key.pub -pubout || exit 1
+${bindir}/create_tpm2_key --wrap key.priv -a -k passw0rd key.tpm || exit 1
+echo "Checked encryption of OAEP PSS" > tmp.txt
+openssl rsautl -encrypt -oaep -in tmp.txt -out tmp.msg -inkey key.pub -pubin || exit 1
+openssl rsautl -decrypt -oaep -in tmp.msg -engine tpm2 -keyform engine -inkey key.tpm -passin pass:passw0rd || exit 1
+##
+# this PSS signature will be padded manually and done as an unpadded encrypt
+# by the TPM
+##
+openssl sha256 -out tmp.md -binary tmp.txt || exit 1
+openssl pkeyutl -sign -engine tpm2 -keyform engine -inkey key.tpm -pkeyopt rsa_padding_mode:pss -pkeyopt digest:sha256 -pkeyopt rsa_mgf1_md:sha256 -in tmp.md -out tmp.msg -passin pass:passw0rd || exit 1
+# OpenSSL bug in some versions returns false for correct signature
+openssl pkeyutl -verify -inkey key.pub -pubin -pkeyopt rsa_padding_mode:pss -pkeyopt digest:sha256 -pkeyopt rsa_mgf1_md:sha256 -in tmp.md -sigfile tmp.msg|grep 'Signature Verified Successfully'|| exit 1
+##
+# finally an OAEP encrypt which triggers an unpadded decrypt
+##
+openssl pkeyutl -encrypt -inkey key.pub -pubin -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 -pkeyopt rsa_mgf1_md:sha256 -in tmp.txt -out tmp.msg || exit 1
+openssl pkeyutl -decrypt -engine tpm2 -keyform engine -inkey key.tpm -pkeyopt rsa_padding_mode:oaep -pkeyopt rsa_oaep_md:sha256 -pkeyopt rsa_mgf1_md:sha256 -in tmp.msg -out recover.txt -passin pass:passw0rd || exit 1
+diff -q tmp.txt recover.txt || exit 1
+