Handle memory allocation errors too

Albeit unlikely to occur in practice, according to OpenSSL
documentation, NULL or -1 can be returned in case of allocation
errors, so these errors must be handled for correctness.

Signed-off-by: Jonathan Wernberg <jonathaw@axis.com>
Signed-off-by: James Bottomley <James.Bottomley@HansenPartnership.com>

3 files changed, 59 insertions, 12 deletions

diff --git a/e_tpm2-ecc.c b/e_tpm2-ecc.c
index 2172feb..0c9525d 100644
--- a/e_tpm2-ecc.c
+++ b/e_tpm2-ecc.c
@@ -45,13 +45,13 @@ struct ecdh_method {
     int flags;
     char *app_data;
 };
-static ECDSA_METHOD *tpm2_ecdsa;
+static ECDSA_METHOD *tpm2_ecdsa = NULL;
 static ECDH_METHOD tpm2_ecdh = {
 	.name = "tpm2 ecc",
 	.compute_key = tpm2_ecdh_compute_key,
 };
 #else
-static EC_KEY_METHOD *tpm2_eck;
+static EC_KEY_METHOD *tpm2_eck = NULL;
 #endif
 
 /* varibles used to get/set CRYPTO_EX_DATA values */
@@ -307,12 +307,15 @@ int tpm2_setup_ecc_methods(void)
 	tpm2_ecdsa = ECDSA_METHOD_new(NULL);
 
 	if (!tpm2_ecdsa)
-		return 0;
+		goto err;
 
 	ECDSA_METHOD_set_name(tpm2_ecdsa, "tpm2 ecc");
 	ECDSA_METHOD_set_sign(tpm2_ecdsa, tpm2_ecdsa_sign);
 
-	ec_app_data =  ECDSA_get_ex_new_index(0, NULL, NULL, NULL, tpm2_ecc_free);
+	ec_app_data = ECDSA_get_ex_new_index(0, NULL, NULL, NULL, tpm2_ecc_free);
+
+	if (ec_app_data < 0)
+		goto err;
 #else
 	int (*psign)(int type, const unsigned char *dgst,
 		     int dlen, unsigned char *sig,
@@ -322,24 +325,48 @@ int tpm2_setup_ecc_methods(void)
 
 	tpm2_eck = EC_KEY_METHOD_new(EC_KEY_OpenSSL());
 
+	if (!tpm2_eck)
+		goto err;
+
 	EC_KEY_METHOD_get_sign(tpm2_eck, &psign, NULL, NULL);
 	EC_KEY_METHOD_set_sign(tpm2_eck, psign, NULL, tpm2_ecdsa_sign);
 	EC_KEY_METHOD_set_compute_key(tpm2_eck, tpm2_ecc_compute_key);
 
 	ec_app_data = EC_KEY_get_ex_new_index(0, NULL, NULL, NULL, tpm2_ecc_free);
-#endif
 
+	if (ec_app_data < 0)
+		goto err;
+#endif
 
 	return 1;
+
+err:
+	tpm2_teardown_ecc_methods();
+
+	return 0;
 }
 
 void tpm2_teardown_ecc_methods(void)
 {
 #if OPENSSL_VERSION_NUMBER < 0x10100000
-	ECDSA_METHOD_free(tpm2_ecdsa);
-	CRYPTO_free_ex_index(CRYPTO_EX_INDEX_ECDSA, ec_app_data);
+	if (tpm2_ecdsa) {
+		ECDSA_METHOD_free(tpm2_ecdsa);
+		tpm2_ecdsa = NULL;
+	}
+
+	if (ec_app_data >= 0) {
+		CRYPTO_free_ex_index(CRYPTO_EX_INDEX_ECDSA, ec_app_data);
+		ec_app_data = TPM2_ENGINE_EX_DATA_UNINIT;
+	}
 #else
-	EC_KEY_METHOD_free(tpm2_eck);
-	CRYPTO_free_ex_index(CRYPTO_EX_INDEX_EC_KEY, ec_app_data);
+	if (tpm2_eck) {
+		EC_KEY_METHOD_free(tpm2_eck);
+		tpm2_eck = NULL;
+	}
+
+	if (ec_app_data >= 0) {
+		CRYPTO_free_ex_index(CRYPTO_EX_INDEX_EC_KEY, ec_app_data);
+		ec_app_data = TPM2_ENGINE_EX_DATA_UNINIT;
+	}
 #endif
 }
diff --git a/e_tpm2-rsa.c b/e_tpm2-rsa.c
index afd88a3..49dffd1 100644
--- a/e_tpm2-rsa.c
+++ b/e_tpm2-rsa.c
@@ -26,7 +26,7 @@
 /* varibles used to get/set CRYPTO_EX_DATA values */
 static int ex_app_data = TPM2_ENGINE_EX_DATA_UNINIT;
 
-RSA_METHOD *tpm2_rsa;
+RSA_METHOD *tpm2_rsa = NULL;
 
 #if OPENSSL_VERSION_NUMBER < 0x10100000
 /* rsa functions */
@@ -315,6 +315,10 @@ int tpm2_setup_rsa_methods(void)
 	tpm2_rsa = &tpm2_rsa_meths;
 #else
 	tpm2_rsa = RSA_meth_dup(RSA_PKCS1_OpenSSL());
+
+	if (!tpm2_rsa)
+		goto err;
+
 	RSA_meth_set1_name(tpm2_rsa, "tpm2 rsa");
 	RSA_meth_set_priv_enc(tpm2_rsa, tpm2_rsa_priv_enc);
 	RSA_meth_set_priv_dec(tpm2_rsa, tpm2_rsa_priv_dec);
@@ -322,14 +326,28 @@ int tpm2_setup_rsa_methods(void)
 
 	ex_app_data = RSA_get_ex_new_index(0, NULL, NULL, NULL, tpm2_rsa_free);
 
+	if (ex_app_data < 0)
+		goto err;
+
 	return 1;
+
+err:
+	tpm2_teardown_rsa_methods();
+
+	return 0;
 }
 
 void tpm2_teardown_rsa_methods(void)
 {
 #if OPENSSL_VERSION_NUMBER >= 0x10100000
-	RSA_meth_free(tpm2_rsa);
+	if (tpm2_rsa) {
+		RSA_meth_free(tpm2_rsa);
+		tpm2_rsa = NULL;
+	}
 #endif
 
-	CRYPTO_free_ex_index(CRYPTO_EX_INDEX_RSA, ex_app_data);
+	if (ex_app_data >= 0) {
+		CRYPTO_free_ex_index(CRYPTO_EX_INDEX_RSA, ex_app_data);
+		ex_app_data = TPM2_ENGINE_EX_DATA_UNINIT;
+	}
 }
diff --git a/e_tpm2.c b/e_tpm2.c
index 389ba41..4d1b2fa 100644
--- a/e_tpm2.c
+++ b/e_tpm2.c
@@ -30,6 +30,8 @@ static int tpm2_set_nvkey_prefix(char *prefix)
 		OPENSSL_free(nvprefix);
 	len = strlen(prefix);
 	nvprefix = OPENSSL_malloc(len+1);
+	if (!nvprefix)
+		return 0;
 	strcpy(nvprefix, prefix);
 
 	return 1;