[TESTING, DO NOT MERGE] crypto - add benchmark and testing moduleold/cryptobench

Add a module which exposes a file /proc/cryptobench.  Writing commands
to this file triggers benchmarking and testing of crypto algorithms
(symmetric ciphers and hashes).

Examples:
    # echo 'algname=adiantum(xchacha12,aes) keysize=32' > /proc/cryptobench
    # cat /proc/cryptobench
    SUCCESS algname=adiantum(xchacha12,aes) driver_name=adiantum(xchacha12-software,aes-aesni,nhpoly1305-generic) measurement=0x30a5abd5776e0af enc_time=44831104 dec_time=38303077

    # echo 'algname=sha256 algtype=hash' > /proc/cryptobench
    # cat /proc/cryptobench
    SUCCESS algname=sha256 driver_name=sha256-avx2 measurement=0x2ed36e096cdb833c time=50339190

Signed-off-by: Eric Biggers <ebiggers@google.com>

5 files changed, 983 insertions, 0 deletions

diff --git a/crypto/Kconfig b/crypto/Kconfig
index 41068811fd0e1..e7f89cb2aeec0 100644
--- a/crypto/Kconfig
+++ b/crypto/Kconfig
@@ -1951,6 +1951,15 @@ config CRYPTO_STATS
 config CRYPTO_HASH_INFO
 	bool
 
+config CRYPTO_BENCHMARK
+	tristate "Crypto algorithm benchmark and testing module"
+	select CRYPTO_MANAGER
+	select CRYPTO_LIB_BLAKE2S_GENERIC
+	help
+	  This module exposes a file /proc/cryptobench.  Writing
+	  commands to this file triggers benchmarking and testing of
+	  crypto algorithms (symmetric ciphers and hashes).
+
 source "drivers/crypto/Kconfig"
 source "crypto/asymmetric_keys/Kconfig"
 source "certs/Kconfig"
diff --git a/crypto/Makefile b/crypto/Makefile
index f754c4d17d6bd..0aad79f35ba7b 100644
--- a/crypto/Makefile
+++ b/crypto/Makefile
@@ -168,6 +168,7 @@ KASAN_SANITIZE_jitterentropy.o = n
 UBSAN_SANITIZE_jitterentropy.o = n
 jitterentropy_rng-y := jitterentropy.o jitterentropy-kcapi.o
 obj-$(CONFIG_CRYPTO_TEST) += tcrypt.o
+obj-$(CONFIG_CRYPTO_BENCHMARK) += benchmark.o
 obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o
 obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o
 obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o
diff --git a/crypto/benchmark.c b/crypto/benchmark.c
new file mode 100644
index 0000000000000..968ef62c0a35d
--- /dev/null
+++ b/crypto/benchmark.c
@@ -0,0 +1,665 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Crypto algorithm benchmark and testing module
+ *
+ * Copyright 2018 Google LLC
+ */
+
+#include <crypto/blake2s.h>
+#include <crypto/hash.h>
+#include <crypto/skcipher.h>
+#include <linux/delay.h>
+#include <linux/module.h>
+#include <linux/parser.h>
+#include <linux/proc_fs.h>
+#include <linux/random.h>
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+#include <linux/timekeeping.h>
+#include <linux/version.h>
+
+#undef pr_fmt
+#define pr_fmt(fmt) "cryptobench: " fmt
+
+static char cryptobench_results[4096];
+static DEFINE_MUTEX(cryptobench_mutex);
+
+#define MAX_KEYSIZE	64
+
+enum cryptobench_result {
+	SUCCESS = 0,
+	ALG_NOT_FOUND,
+	ALG_ALLOCATION_ERROR,
+	KEYSIZE_NOT_SUPPORTED,
+	CRYPTO_ERROR,
+	CRYPTO_INCORRECT,
+	BAD_PARAMETERS,
+	OUT_OF_MEMORY,
+};
+
+static const char * const result_strings[] = {
+	[SUCCESS]		= "SUCCESS",
+	[ALG_NOT_FOUND]		= "ALG_NOT_FOUND",
+	[ALG_ALLOCATION_ERROR]	= "ALG_ALLOCATION_ERROR",
+	[KEYSIZE_NOT_SUPPORTED]	= "KEYSIZE_NOT_SUPPORTED",
+	[CRYPTO_ERROR]		= "CRYPTO_ERROR",
+	[CRYPTO_INCORRECT]	= "CRYPTO_INCORRECT",
+	[BAD_PARAMETERS]	= "BAD_PARAMETERS",
+	[OUT_OF_MEMORY]		= "OUT_OF_MEMORY",
+};
+
+struct cryptobench_params {
+	char *algtype;
+	char *algname;
+	int keysize;
+	unsigned long niter;
+	unsigned long bufsize;
+	bool inplace;
+	bool sgl_fuzz;
+	unsigned long long random_seed;
+};
+
+static u32 rand32(struct cryptobench_params *params)
+{
+	params->random_seed *= 25214903917;
+	params->random_seed += 11;
+	params->random_seed &= ((u64)1 << 48) - 1;
+	return params->random_seed >> 16;
+}
+
+static void rand_bytes(struct cryptobench_params *params,
+		       void *buf, size_t size)
+{
+	u8 *p = buf;
+
+	while (size--)
+		*p++ = rand32(params);
+}
+
+static u64 measure_buf(const void *buf, size_t size)
+{
+	__le64 hash;
+
+	blake2s((u8 *)&hash, buf, NULL, sizeof(hash), size, 0);
+	return le64_to_cpu(hash);
+}
+
+static ssize_t cryptobench_read(struct file *f, char __user *ubuf,
+				size_t size, loff_t *off)
+{
+	size_t len;
+	ssize_t ret = 0;
+
+	mutex_lock(&cryptobench_mutex);
+	len = strnlen(cryptobench_results, sizeof(cryptobench_results));
+
+	if (*off >= len)
+		goto out;
+
+	len = min_t(size_t, size, len - *off);
+	ret = -EFAULT;
+	if (copy_to_user(ubuf, &cryptobench_results[*off], len))
+		goto out;
+
+	ret = len;
+	*off += len;
+out:
+	mutex_unlock(&cryptobench_mutex);
+	return ret;
+}
+
+#define MAX_NUM_SGS		10
+#define MAX_SG_GAP		32
+
+static bool setup_buffer_and_sglist(void **buf_ret, size_t bufsize,
+				    struct scatterlist sg[MAX_NUM_SGS],
+				    bool sgl_fuzz)
+{
+	int i;
+	void *p;
+	int num_sgs;
+
+	if (!sgl_fuzz) {
+		*buf_ret = kzalloc(bufsize, GFP_KERNEL);
+		if (!*buf_ret)
+			return false;
+		sg_init_one(&sg[0], *buf_ret, bufsize);
+		return true;
+	}
+
+	*buf_ret = kzalloc(bufsize + (MAX_NUM_SGS * MAX_SG_GAP), GFP_KERNEL);
+	if (!*buf_ret)
+		return false;
+	p = *buf_ret;
+
+	if (bufsize == 0) {
+		sg_init_one(&sg[0], p, 0);
+		return true;
+	}
+
+	num_sgs = 1 + (get_random_int() % MAX_NUM_SGS);
+	sg_init_table(sg, num_sgs);
+	i = 0;
+	do {
+		int r = get_random_int() % 3;
+		size_t n;
+
+		if (i == num_sgs - 1 || r == 0)
+			n = bufsize;
+		else if (r == 1)
+			n = 1 + (get_random_long() % 64);
+		else
+			n = 1 + (get_random_long() % bufsize);
+		n = min(n, bufsize);
+		p += get_random_int() % (MAX_SG_GAP + 1);
+		sg_set_buf(&sg[i], p, n);
+		p += n;
+		bufsize -= n;
+	} while (++i < num_sgs && bufsize != 0);
+	sg_mark_end(&sg[i - 1]);
+	return true;
+}
+
+static void memcpy_to_sgl(struct scatterlist *sg, const void *buf, size_t size)
+{
+	size_t copied;
+
+	copied = sg_pcopy_from_buffer(sg, sg_nents(sg), buf, size, 0);
+
+	WARN_ON(copied != size);
+}
+
+static void memcpy_from_sgl(void *buf, struct scatterlist *sg, size_t size)
+{
+	size_t copied;
+
+	copied = sg_pcopy_to_buffer(sg, sg_nents(sg), buf, size, 0);
+
+	WARN_ON(copied != size);
+}
+
+static enum cryptobench_result
+benchmark_skcipher(struct cryptobench_params *params)
+{
+	struct crypto_skcipher *tfm = NULL;
+	struct skcipher_request *req = NULL;
+	const char *driver_name = NULL;
+	u8 orig_iv[32];
+	u8 iv[32] __aligned(8);
+	void *inbuf = NULL, *outbuf = NULL, *tmpbuf = NULL, *orig_data = NULL;
+	unsigned long i;
+	unsigned long parity = 0;
+	struct scatterlist _src[MAX_NUM_SGS];
+	struct scatterlist _dst[MAX_NUM_SGS];
+	struct scatterlist *src = _src, *dst = _dst;
+	u64 t1, t2, t3;
+	u64 measurement;
+	int err;
+	DECLARE_CRYPTO_WAIT(wait);
+	enum cryptobench_result result;
+	u8 key[MAX_KEYSIZE];
+
+	if (params->keysize < 1 || params->keysize > MAX_KEYSIZE) {
+		pr_err("bad value for 'keysize' option");
+		return BAD_PARAMETERS;
+	}
+
+	rand_bytes(params, orig_iv, sizeof(orig_iv));
+	rand_bytes(params, key, params->keysize);
+
+	tmpbuf = kmalloc(params->bufsize, GFP_KERNEL);
+	orig_data = kmalloc(params->bufsize, GFP_KERNEL);
+	if (!tmpbuf || !orig_data) {
+		result = OUT_OF_MEMORY;
+		goto out;
+	}
+
+	if (!setup_buffer_and_sglist(&inbuf, params->bufsize, src,
+				     params->sgl_fuzz)) {
+		result = OUT_OF_MEMORY;
+		goto out;
+	}
+	rand_bytes(params, orig_data, params->bufsize);
+	memcpy_to_sgl(src, orig_data, params->bufsize);
+
+	if (params->inplace) {
+		outbuf = inbuf;
+		dst = src;
+	} else {
+		if (!setup_buffer_and_sglist(&outbuf, params->bufsize, dst,
+					     params->sgl_fuzz)) {
+			result = OUT_OF_MEMORY;
+			goto out;
+		}
+	}
+
+	tfm = crypto_alloc_skcipher(params->algname, 0, 0);
+	if (IS_ERR(tfm)) {
+		if (PTR_ERR(tfm) == -ENOENT) {
+			result = ALG_NOT_FOUND;
+		} else {
+			pr_err("error allocating %s: %ld\n",
+			       params->algname, PTR_ERR(tfm));
+			result = ALG_ALLOCATION_ERROR;
+		}
+		tfm = NULL;
+		goto out;
+	}
+	driver_name = crypto_skcipher_alg(tfm)->base.cra_driver_name;
+	req = skcipher_request_alloc(tfm, GFP_KERNEL);
+	if (!req) {
+		result = OUT_OF_MEMORY;
+		goto out;
+	}
+
+	err = crypto_skcipher_setkey(tfm, key, params->keysize);
+	if (err) {
+		result = KEYSIZE_NOT_SUPPORTED;
+		goto out;
+	}
+
+	skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
+				      CRYPTO_TFM_REQ_MAY_BACKLOG,
+				      crypto_req_done, &wait);
+
+	cond_resched();
+	t1 = ktime_get_ns();
+	for (i = 0; i < params->niter; i++) {
+		memcpy(iv, orig_iv, sizeof(iv));
+		*(u64 *)iv += i;
+		if (parity++ & 1) {
+			skcipher_request_set_crypt(req, dst, src,
+						   params->bufsize, iv);
+		} else {
+			skcipher_request_set_crypt(req, src, dst,
+						   params->bufsize, iv);
+		}
+		err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait);
+		if (err) {
+			pr_err("encryption error w/ alg %s: %d\n",
+			       driver_name, err);
+			result = CRYPTO_ERROR;
+			goto out;
+		}
+	}
+	t2 = ktime_get_ns();
+	memcpy_from_sgl(tmpbuf, (params->niter & 1) ? dst : src,
+			params->bufsize);
+	if (params->bufsize && !memcmp(tmpbuf, orig_data, params->bufsize)) {
+		pr_err("encryption w/ alg %s didn't do anything!\n",
+		       driver_name);
+		result = CRYPTO_INCORRECT;
+		goto out;
+	}
+	measurement = measure_buf(tmpbuf, params->bufsize);
+	cond_resched();
+	t2 = ktime_get_ns();
+	for (i = 0; i < params->niter; i++) {
+		memcpy(iv, orig_iv, sizeof(iv));
+		*(u64 *)iv += params->niter - 1 - i;
+		if (parity++ & 1) {
+			skcipher_request_set_crypt(req, dst, src,
+						   params->bufsize, iv);
+		} else {
+			skcipher_request_set_crypt(req, src, dst,
+						   params->bufsize, iv);
+		}
+		err = crypto_wait_req(crypto_skcipher_decrypt(req), &wait);
+		if (err) {
+			pr_err("decryption error w/ alg %s: %d\n",
+			       driver_name, err);
+			result = CRYPTO_ERROR;
+			goto out;
+		}
+	}
+	t3 = ktime_get_ns();
+	cond_resched();
+
+	memcpy_from_sgl(tmpbuf, src, params->bufsize);
+	if (memcmp(tmpbuf, orig_data, params->bufsize)) {
+		pr_err("%s decryption didn't invert encryption!\n",
+		       driver_name);
+		result = CRYPTO_INCORRECT;
+		goto out;
+	}
+
+	sprintf(cryptobench_results,
+		"SUCCESS algname=%s driver_name=%s measurement=%#016llx enc_time=%llu dec_time=%llu\n",
+		params->algname, driver_name, measurement, t2 - t1, t3 - t2);
+	result = SUCCESS;
+out:
+	skcipher_request_free(req);
+	crypto_free_skcipher(tfm);
+	kfree(inbuf);
+	if (inbuf != outbuf)
+		kfree(outbuf);
+	kfree(tmpbuf);
+	kfree(orig_data);
+	return result;
+}
+
+static enum cryptobench_result
+benchmark_shash(struct cryptobench_params *params,
+		const void *key, const void *inbuf)
+{
+	struct crypto_shash *tfm;
+	SHASH_DESC_ON_STACK(desc, unused);
+	const char *driver_name;
+	u8 digest[HASH_MAX_DIGESTSIZE];
+	unsigned long i;
+	u64 t1, t2;
+	u64 measurement;
+	int err;
+	enum cryptobench_result result;
+
+	tfm = crypto_alloc_shash(params->algname, 0, 0);
+	if (IS_ERR(tfm)) {
+		if (PTR_ERR(tfm) == -ENOENT)
+			return ALG_NOT_FOUND;
+		pr_err("error allocating %s: %ld\n", params->algname,
+		       PTR_ERR(tfm));
+		return ALG_ALLOCATION_ERROR;
+	}
+	desc->tfm = tfm;
+	driver_name = crypto_shash_driver_name(tfm);
+
+	if (params->keysize) {
+		err = crypto_shash_setkey(tfm, key, params->keysize);
+		if (err) {
+			result = KEYSIZE_NOT_SUPPORTED;
+			goto out;
+		}
+	}
+
+	cond_resched();
+	t1 = ktime_get_ns();
+	for (i = 0; i < params->niter; i++) {
+		err = crypto_shash_digest(desc, inbuf, params->bufsize, digest);
+		if (err) {
+			pr_err("hash error w/ alg %s: %d\n", driver_name, err);
+			result = CRYPTO_ERROR;
+			goto out;
+		}
+	}
+	t2 = ktime_get_ns();
+	measurement = measure_buf(digest, crypto_shash_digestsize(tfm));
+
+	sprintf(cryptobench_results,
+		"SUCCESS algname=%s driver_name=%s measurement=%#016llx time=%llu\n",
+		params->algname, driver_name, measurement, t2 - t1);
+	result = SUCCESS;
+out:
+	crypto_free_shash(tfm);
+	return result;
+}
+
+static enum cryptobench_result
+benchmark_ahash(struct cryptobench_params *params,
+		const void *key, const void *inbuf)
+{
+	struct crypto_ahash *tfm;
+	struct ahash_request *req;
+	const char *driver_name;
+	u8 digest[HASH_MAX_DIGESTSIZE];
+	unsigned long i;
+	struct scatterlist sg;
+	u64 t1, t2;
+	u64 measurement;
+	int err;
+	DECLARE_CRYPTO_WAIT(wait);
+	enum cryptobench_result result;
+
+	tfm = crypto_alloc_ahash(params->algname, 0, 0);
+	if (IS_ERR(tfm)) {
+		if (PTR_ERR(tfm) == -ENOENT)
+			return ALG_NOT_FOUND;
+		pr_err("error allocating %s: %ld\n", params->algname,
+		       PTR_ERR(tfm));
+		return ALG_ALLOCATION_ERROR;
+	}
+	req = ahash_request_alloc(tfm, GFP_KERNEL);
+	if (!req) {
+		result = OUT_OF_MEMORY;
+		goto out;
+	}
+	driver_name = crypto_ahash_driver_name(tfm);
+
+	if (params->keysize) {
+		err = crypto_ahash_setkey(tfm, key, params->keysize);
+		if (err) {
+			result = KEYSIZE_NOT_SUPPORTED;
+			goto out;
+		}
+	}
+
+	sg_init_one(&sg, inbuf, params->bufsize);
+	ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP |
+				   CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done,
+				   &wait);
+	ahash_request_set_crypt(req, &sg, digest, params->bufsize);
+
+	cond_resched();
+	t1 = ktime_get_ns();
+	for (i = 0; i < params->niter; i++) {
+		err = crypto_wait_req(crypto_ahash_digest(req), &wait);
+		if (err) {
+			pr_err("hash error w/ alg %s: %d\n", driver_name, err);
+			result = CRYPTO_ERROR;
+			goto out;
+		}
+	}
+	t2 = ktime_get_ns();
+	measurement = measure_buf(digest, crypto_ahash_digestsize(tfm));
+
+	sprintf(cryptobench_results,
+		"SUCCESS algname=%s driver_name=%s measurement=%#016llx time=%llu\n",
+		params->algname, driver_name, measurement, t2 - t1);
+	result = SUCCESS;
+out:
+	ahash_request_free(req);
+	crypto_free_ahash(tfm);
+	return result;
+}
+
+static enum cryptobench_result
+benchmark_hash(struct cryptobench_params *params)
+{
+	u8 key[MAX_KEYSIZE];
+	void *inbuf;
+	enum cryptobench_result result;
+
+	if (params->keysize) {
+		if (params->keysize > MAX_KEYSIZE) {
+			pr_err("bad value for 'keysize' option");
+			return BAD_PARAMETERS;
+		}
+		rand_bytes(params, key, params->keysize);
+	}
+
+	inbuf = kzalloc(params->bufsize, GFP_KERNEL);
+	if (!inbuf)
+		return OUT_OF_MEMORY;
+	rand_bytes(params, inbuf, params->bufsize);
+
+	result = benchmark_shash(params, key, inbuf);
+	if (result != ALG_NOT_FOUND)
+		result = benchmark_ahash(params, key, inbuf);
+	kfree(inbuf);
+	return result;
+}
+
+enum {
+	Opt_algtype,
+	Opt_algname,
+	Opt_keysize,
+	Opt_niter,
+	Opt_bufsize,
+	Opt_inplace,
+	Opt_sgl_fuzz,
+	Opt_random_seed,
+	Opt_err,
+};
+
+static const match_table_t tokens = {
+	{Opt_algtype, "algtype=%s"},
+	{Opt_algname, "algname=%s"},
+	{Opt_keysize, "keysize=%s"},
+	{Opt_niter, "niter=%s"},
+	{Opt_bufsize, "bufsize=%s"},
+	{Opt_inplace, "inplace"},
+	{Opt_sgl_fuzz, "sgl_fuzz"},
+	{Opt_random_seed, "random_seed=%s"},
+	{Opt_err, NULL},
+};
+
+static const struct {
+	const char *algtype;
+	enum cryptobench_result (*f)(struct cryptobench_params *params);
+} benchmark_funcs[] = {
+	{ "skcipher", benchmark_skcipher },
+	{ "hash",     benchmark_hash },
+};
+
+static ssize_t cryptobench_write(struct file *f, const char __user *ubuf,
+				 size_t size, loff_t *off)
+{
+	char *optstr = NULL;
+	char *opt, *optp;
+	struct cryptobench_params params = {
+		.algtype = "skcipher",
+		.niter = 4096,
+		.bufsize = 4096,
+	};
+	ssize_t ret;
+	int i;
+	enum cryptobench_result result;
+
+	mutex_lock(&cryptobench_mutex);
+
+	if (size >= 4096 || *off != 0)
+		goto bad_params;
+
+	optstr = kmalloc(size + 1, GFP_KERNEL);
+	if (!optstr)
+		goto bad_params;
+	if (copy_from_user(optstr, ubuf, size))
+		goto bad_params;
+	optstr[size] = '\0';
+
+	optp = optstr;
+	while ((opt = strsep(&optp, "\n ")) != NULL) {
+		substring_t args[MAX_OPT_ARGS];
+		int token;
+
+		if (!*opt)
+			continue;
+
+		token = match_token(opt, tokens, args);
+		switch (token) {
+		case Opt_algtype:
+			params.algtype = args[0].from;
+			break;
+		case Opt_algname:
+			params.algname = args[0].from;
+			break;
+		case Opt_keysize:
+			if (kstrtoint(args[0].from, 10, &params.keysize)) {
+				pr_err("bad value for 'keysize' option");
+				goto bad_params;
+			}
+			break;
+		case Opt_niter:
+			if (kstrtoul(args[0].from, 10, &params.niter)) {
+				pr_err("bad value for 'niter' option");
+				goto bad_params;
+			}
+			break;
+		case Opt_bufsize:
+			if (kstrtoul(args[0].from, 10, &params.bufsize)) {
+				pr_err("bad value for 'bufsize' option");
+				goto bad_params;
+			}
+			break;
+		case Opt_inplace:
+			params.inplace = true;
+			break;
+		case Opt_sgl_fuzz:
+			params.sgl_fuzz = true;
+			break;
+		case Opt_random_seed:
+			if (kstrtoull(args[0].from, 10, &params.random_seed)) {
+				pr_err("bad value for 'random_seed' option");
+				goto bad_params;
+			}
+			break;
+		default:
+			pr_err("unrecognized option '%s'\n", opt);
+			goto bad_params;
+		}
+	}
+
+	if (!params.algtype) {
+		pr_err("'algtype' option is missing");
+		goto bad_params;
+	}
+
+	if (!params.algname) {
+		pr_err("'algname' option is missing");
+		goto bad_params;
+	}
+
+	for (i = 0; i < ARRAY_SIZE(benchmark_funcs); i++) {
+		if (!strcmp(benchmark_funcs[i].algtype, params.algtype)) {
+			result = benchmark_funcs[i].f(&params);
+			ret = size;
+			if (result != SUCCESS)
+				goto save_error;
+			goto out;
+		}
+	}
+
+	pr_err("bad value for 'algtype' option");
+bad_params:
+	ret = -EINVAL;
+	result = BAD_PARAMETERS;
+save_error:
+	sprintf(cryptobench_results, "ERROR %s\n", result_strings[result]);
+out:
+	mutex_unlock(&cryptobench_mutex);
+	kfree(optstr);
+	return ret;
+}
+
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 6, 0)
+static const struct proc_ops cryptobench_fops = {
+	.proc_write = cryptobench_write,
+	.proc_read = cryptobench_read,
+	.proc_lseek = noop_llseek,
+};
+#else
+static const struct file_operations cryptobench_fops = {
+	.write = cryptobench_write,
+	.read = cryptobench_read,
+};
+#endif
+
+static struct proc_dir_entry *proc_cryptobench;
+
+static int __init cryptobench_init(void)
+{
+	proc_cryptobench = proc_create("cryptobench", 0600, NULL,
+				       &cryptobench_fops);
+	return PTR_ERR_OR_ZERO(proc_cryptobench);
+}
+
+static void __exit cryptobench_exit(void)
+{
+	proc_remove(proc_cryptobench);
+}
+
+module_init(cryptobench_init);
+module_exit(cryptobench_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_DESCRIPTION("Crypto algorithm benchmark and testing module");
+MODULE_AUTHOR("Eric Biggers <ebiggers@google.com>");
diff --git a/tools/crypto/benchmark-hctr2.sh b/tools/crypto/benchmark-hctr2.sh
new file mode 100755
index 0000000000000..97d07bf5e9de8
--- /dev/null
+++ b/tools/crypto/benchmark-hctr2.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+
+CRYPTOBENCH='cryptobench.py --ntries=10'
+
+$CRYPTOBENCH --keysizes=32 --ciphers='hctr2(aes)'
+$CRYPTOBENCH --keysizes=32 --ciphers='adiantum(xchacha12,aes)'
+$CRYPTOBENCH --keysizes=64 --ciphers='xts(aes)'
+$CRYPTOBENCH --keysizes=32 --ciphers='cts(cbc(aes))'
+echo
+$CRYPTOBENCH --keysizes=32 \
+        --ciphers='ctr(aes),xctr(aes),ctr(aes-generic),xctr(aes-generic)'
+echo
+$CRYPTOBENCH --keysizes=16 --hashes='polyval,ghash,polyval-generic'
+echo
+$CRYPTOBENCH --keysizes=32 --bufsize=32 --niter=20000 --ciphers='hctr2(aes),cts(cbc(aes)),adiantum(xchacha12,aes)'
diff --git a/tools/crypto/cryptobench.py b/tools/crypto/cryptobench.py
new file mode 100755
index 0000000000000..1f60d376b85da
--- /dev/null
+++ b/tools/crypto/cryptobench.py
@@ -0,0 +1,293 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: GPL-2.0-only
+#
+# Crypto algorithm benchmark and testing script
+#
+# Userspace utility for /proc/cryptobench (CONFIG_CRYPTO_BENCHMARK).
+#
+# Copyright 2018 Google LLC
+#
+
+import argparse
+import subprocess
+import sys
+
+KNOWN_CIPHERS = [
+    {
+        'name': 'aes',
+        'blocksize': 16,
+        'keysizes': [16, 24, 32],
+        'impls': ['generic', 'aesni', 'ce', 'neonbs', 'neon'],
+    }, {
+        'name': 'blowfish',
+        'blocksize': 8,
+        'keysizes': [4, 16, 24, 32, 56],
+        'impls': ['generic', 'asm'],
+    }, {
+        'name': 'camellia',
+        'blocksize': 16,
+        'keysizes': [16, 24, 32],
+        'impls': ['generic', 'asm', 'aesni', 'aesni-avx'],
+    }, {
+        'name': 'cast5',
+        'blocksize': 8,
+        'keysizes': [5, 12, 16],
+        'impls': ['generic', 'avx'],
+    }, {
+        'name': 'cast6',
+        'blocksize': 16,
+        'keysizes': [16, 24, 32],
+        'impls': ['generic', 'avx'],
+    }, {
+        'name': 'chacha20',
+        'keysizes': [32],
+        'impls': ['generic', 'simd', 'neon', 'arm'],
+    }, {
+        'name': 'des3_ede',
+        'blocksize': 8,
+        'keysizes': [24],
+        'impls': ['generic', 'asm']
+    }, {
+        'name': 'lea',
+        'blocksize': 16,
+        'keysizes': [16, 24, 32],
+        'impls': ['generic', 'neon'],
+    }, {
+        'name': 'salsa20',
+        'keysizes': [32],
+        'impls': ['generic'],
+    }, {
+        'name': 'serpent',
+        'blocksize': 16,
+        'keysizes': [16, 24, 32],
+        'impls': ['generic', 'sse2', 'avx', 'avx2'],
+    }, {
+        'name': 'twofish',
+        'blocksize': 16,
+        'keysizes': [16, 24, 32],
+        'impls': ['generic', '3way', 'avx'],
+    }
+]
+
+def find_cipher(name):
+    for cipher in KNOWN_CIPHERS:
+        if cipher['name'] == name:
+            return cipher
+    return None
+
+def error(msg):
+    sys.stderr.write(msg + '\n')
+    sys.exit(1)
+
+def MB_per_s(nbytes, ns_elapsed):
+    return (nbytes * 1000) / ns_elapsed
+
+class BenchmarkError(Exception):
+    def __init__(self, message, error_type):
+        super(BenchmarkError, self).__init__(message)
+        self.error_type = error_type
+
+def proc_cryptobench(cmd, args):
+    if args.adb:
+        sh_cmd = ''
+        if args.cpu_mask:
+            sh_cmd += ' taskset ' + args.cpu_mask
+        sh_cmd += ' sh -c "echo -e \'{}\' > /proc/cryptobench; cat /proc/cryptobench"'.format(cmd)
+        return str(subprocess.check_output(['adb', 'shell', sh_cmd]), 'utf-8')
+    else:
+        if args.cpu_mask:
+            raise ValueError('TODO: support --cpu-mask without --adb')
+        with open('/proc/cryptobench', 'rt+') as f:
+            f.write(cmd)
+            f.seek(0)
+            return f.readline()
+
+def do_kernel_benchmark(algtype, algname, keysize, args):
+    cmd = ''
+    cmd += ' algtype=' + algtype
+    cmd += ' algname=' + algname
+    cmd += ' keysize=' + str(keysize)
+    cmd += ' niter=' + str(args.niter)
+    cmd += ' bufsize=' + str(args.bufsize)
+    if args.sgl_fuzz:
+        cmd += ' sgl_fuzz'
+    if args.inplace:
+        cmd += ' inplace'
+    cmd += '\n'
+
+    fields = proc_cryptobench(cmd, args).split()
+    if fields[0] == 'ERROR':
+        raise BenchmarkError('error with algorithm ' + algname, fields[1])
+
+    results = {}
+    for item in fields[1:]:
+        (key, value) = item.split('=')
+        results[key] = value
+    return results
+
+def check_measurement(prev_measurement, results, algname):
+    measurement = results['measurement']
+    if prev_measurement is not None and measurement != prev_measurement:
+        error('Algorithm {} (driver: {}) gave inconsistent results!'.format(
+              algname, results['driver_name']))
+    return measurement
+
+def benchmark_skcipher(algname, friendly_name, keysize, args):
+    enc_time = 2**64
+    dec_time = 2**64
+    measurement = None
+    for _try in range(args.ntries):
+        try:
+            results = do_kernel_benchmark('skcipher', algname, keysize, args)
+        except BenchmarkError as e:
+            if e.error_type != 'ALG_NOT_FOUND':
+                print('{} {}'.format(algname, e.error_type))
+            return
+        measurement = check_measurement(measurement, results, algname)
+        enc_time = min(enc_time, int(results['enc_time']))
+        dec_time = min(dec_time, int(results['dec_time']))
+    print('{:17} {:30} {:30} {:4.2f} {:4.2f} {:17}'.format(
+          friendly_name,
+          algname,
+          results['driver_name'],
+          MB_per_s(int(args.niter) * int(args.bufsize), enc_time),
+          MB_per_s(int(args.niter) * int(args.bufsize), dec_time),
+          measurement))
+
+def benchmark_hash(algname, keysize, args):
+    time = 2**64
+    measurement = None
+    for _try in range(args.ntries):
+        try:
+            results = do_kernel_benchmark('hash', algname, keysize, args)
+        except BenchmarkError as e:
+            if e.error_type != 'ALG_NOT_FOUND':
+                print('{} {}'.format(algname, e.error_type))
+            return
+        measurement = check_measurement(measurement, results, algname)
+        time = min(time, int(results['time']))
+    print('{:17} {:30} {:6.1f} {:17}'.format(
+          algname,
+          results['driver_name'],
+          MB_per_s(int(args.niter) * int(args.bufsize), time),
+          measurement))
+
+def keysize_for_mode(mode, keysize, blocksize):
+    if mode == 'xts':
+        return keysize * 2
+    if mode == 'lrw':
+        return keysize + blocksize
+    return keysize
+
+def benchmark_cipher_spec(cipher, keysize, args):
+    blocksize = cipher.get('blocksize')
+    name = cipher['name']
+    if not blocksize or blocksize == 1:
+        # stream cipher
+        benchmark_skcipher(name, name, keysize, args)
+        if args.all_impls:
+            for impl in cipher['impls']:
+                benchmark_skcipher('{}-{}'.format(name, impl),
+                                   name, keysize, args)
+        return
+
+    # block cipher
+    available_modes = ['ecb', 'cbc', 'ctr', 'xctr']
+    if blocksize == 16:
+        available_modes.extend(['lrw', 'xts'])
+
+    for mode in available_modes if args.modes is None else args.modes:
+        algname = '{}({})'.format(mode, name)
+        friendly_name = '{}-{}-{}'.format(name.upper(), keysize*8, mode.upper())
+        actual_keysize = keysize_for_mode(mode, keysize, blocksize)
+        benchmark_skcipher(algname, friendly_name, actual_keysize, args)
+
+    if args.all_impls:
+        for impl in cipher['impls']:
+            for mode in available_modes if args.modes is None else args.modes:
+                if impl == 'generic':
+                    if mode == 'xts' or mode == 'lrw':
+                        algname = '{}(ecb({}-generic))'.format(mode, name)
+                    else:
+                        algname = '{}({}-generic)'.format(mode, name)
+                else:
+                    algname = '{}-{}-{}'.format(mode, name, impl)
+                friendly_name = '{}-{}-{}'.format(name.upper(), keysize*8,
+                                                  mode.upper())
+                actual_keysize = keysize_for_mode(mode, keysize, blocksize)
+                benchmark_skcipher(algname, friendly_name, actual_keysize, args)
+
+def parse_algnames(optarg):
+    cur_name = ''
+    nesting_level = 0
+    names = set()
+    for c in optarg + ',':
+        if c == '(':
+            nesting_level += 1
+        elif c == ')':
+            nesting_level -= 1
+            if nesting_level < 0:
+                raise ValueError('Malformed argument: ' + optarg)
+        elif c == ',' and nesting_level == 0 and cur_name != '':
+            names.add(cur_name)
+            cur_name = ''
+            continue
+        cur_name += c
+    return sorted(names)
+
+parser = argparse.ArgumentParser(description='Run cryptographic benchmarks.')
+
+parser.add_argument('--ciphers', action='store', help='ciphers to enable')
+parser.add_argument('--hashes', action='store', help='hashes to enable')
+parser.add_argument('--modes', action='store', help='cipher modes to enable')
+parser.add_argument('--keysizes', action='store', help='keysizes to enable')
+parser.add_argument('--all-impls', action='store_true', help='test all impls')
+parser.add_argument('--ntries', action='store', type=int, default=1, help='num tries per benchmark')
+parser.add_argument('--bufsize', action='store', default=4096, help='buffer size')
+parser.add_argument('--niter', action='store', default=4096, help='num iterations per benchmark')
+parser.add_argument('--inplace', action='store_true', default=False, help='crypt in place?')
+parser.add_argument('--sgl-fuzz', action='store_true', default=False, help='use random sglists')
+parser.add_argument('--adb', action='store_true', default=False, help='use connected Android device')
+parser.add_argument('--cpu-mask', action='store', help='CPUs to allow (default: all)')
+
+args = parser.parse_args()
+
+if args.ciphers:
+    args.ciphers = parse_algnames(args.ciphers)
+
+if args.hashes:
+    args.hashes = parse_algnames(args.hashes)
+
+if args.ntries <= 0:
+    error('Must have ntries >= 1')
+
+if not (args.ciphers or args.hashes):
+    error('Must specify at least one of --ciphers or --hashes')
+
+if args.modes:
+    args.modes = sorted(set(x for x in args.modes.split(',')))
+
+if args.keysizes:
+    args.keysizes = sorted(set(int(x) for x in args.keysizes.split(',')))
+
+if args.ciphers:
+    for cipher_name in args.ciphers:
+        cipher = find_cipher(cipher_name)
+        if not cipher:
+            if args.keysizes:
+                cipher = {
+                    'name': cipher_name,
+                    'keysizes': [args.keysizes],
+                    'impls': [],
+                }
+            else:
+                error('Cipher "{}" not found and --keysizes not specified!'.format(
+                      cipher_name))
+        for keysize in args.keysizes if args.keysizes else cipher['keysizes']:
+            benchmark_cipher_spec(cipher, keysize, args)
+
+if args.hashes:
+    args.hashes = sorted(args.hashes)
+    for hash_ in args.hashes:
+        for keysize in args.keysizes if args.keysizes else [0]:
+            benchmark_hash(hash_, keysize, args)