path: root/main.c

#include <assert.h>
#include <pthread.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <time.h>
#include <unistd.h>

#include "memcpy.h"

#define KB	1024
#define MB	(1024 * KB)
#define GB	(1024 * MB)

enum mode {
	LDP_STR		= 0,
	LDP_STTR,
	LDR_STP,
	LDTR_STP,
	LDP_STP,
	PRFM_2K_LDP_STR,
	LDP_STP_DC_CIVAC,

	NR_MODES
};

static size_t test_size = 2UL * GB;
static size_t chunk_size = 2UL * GB;
static size_t test_iterations = 100;
static size_t dcache_worker_threads = 1;
static enum mode test_mode;
static bool use_hugetlb;

#define PR_MODE(mode)				\
	printf("    %d: "#mode"\n", mode);

static void pr_modes(void)
{
	PR_MODE(LDP_STR);
	PR_MODE(LDP_STTR);
	PR_MODE(LDR_STP);
	PR_MODE(LDTR_STP);
	PR_MODE(LDP_STP);
	PR_MODE(PRFM_2K_LDP_STR);
	PR_MODE(LDP_STP_DC_CIVAC);
}

static void pr_help(const char *progname)
{
	printf("%s [OPTIONS]\n", progname);
	printf("  -s SIZE		(default: %lu)\n", test_size);
	printf("  -c CHUNK_SIZE		(default: %lu)\n", chunk_size);
	printf("  -i ITERATIONS		(default: %lu)\n", test_iterations);
	printf("  -m MODE		(default: %d)\n", test_mode);
	pr_modes();
	printf("  -t WORKER_THREADS	(default: %lu)\n", dcache_worker_threads);
	printf("  -H use hugetlb for test buffers\n");
	printf("  -h prints this message\n");
}

static void *setup_test_buffer(void)
{
	int flags = MAP_PRIVATE | MAP_ANONYMOUS;

	if (use_hugetlb)
		flags |= MAP_HUGETLB;

	void *buf = mmap(0, test_size, PROT_READ|PROT_WRITE,
			 flags, 0, 0);

	assert(buf != MAP_FAILED);
	memset(buf, 0xf, test_size);
	return buf;
}

static void destroy_test_buffer(void *buf)
{
	munmap(buf, test_size);
}

static void dcache_clean_invalidate(const void *buf, size_t size)
{
	const void *addr;

	for (addr = buf; addr < buf + size; addr += 64)
		asm volatile("dc civac, %0" : : "r"(addr)
			     : "memory");

	asm volatile("dsb ish");
}

static void *dcache_maintenance_thread(void *data)
{
	dcache_clean_invalidate(data, test_size / dcache_worker_threads);
	return NULL;
}

static void do_dcache_maintenance(const void *buf)
{
	pthread_t *threads = calloc(dcache_worker_threads, sizeof(pthread_t));
	size_t i;

	for (i = 0; i < dcache_worker_threads; i++)
		pthread_create(&threads[i], NULL, dcache_maintenance_thread,
			       (void *)(buf + (i * (test_size / dcache_worker_threads))));

	for (i = 0; i < dcache_worker_threads; i++)
		pthread_join(threads[i], NULL);

	free(threads);
}

static void do_memcpy(void *dst, const void *src, size_t chunk_size)
{
	switch (test_mode) {
	case LDP_STR:
		memcpy_ldp_str(dst, src, chunk_size);
		break;
	case LDP_STTR:
		memcpy_ldp_sttr(dst, src, chunk_size);
		break;
	case LDR_STP:
		memcpy_ldr_stp(dst, src, chunk_size);
		break;
	case LDTR_STP:
		memcpy_ldtr_stp(dst, src, chunk_size);
		break;
	case LDP_STP:
	case LDP_STP_DC_CIVAC:
		memcpy_ldp_stp(dst, src, chunk_size);
		break;
	case PRFM_2K_LDP_STR:
		memcpy_prfm_2k_ldp_str(dst, src, chunk_size);
		break;
	default:
		assert(0);
	}
}

#define NSEC_PER_SEC	1000000000UL
#define NSEC_PER_MSEC	1000000UL
#define MSEC_PER_SEC	1000UL

static void report_bandwidth(struct timespec *start, struct timespec *end)
{
	double nsec = ((end->tv_sec - start->tv_sec) * NSEC_PER_SEC) +
			(end->tv_nsec - start->tv_nsec);
	double data = test_size * test_iterations;
	double rate = (data / (nsec / NSEC_PER_SEC)) / GB;

	printf("Rate:	%.03f GiB/sec\n", rate);
}

static void report_dcache_time(struct timespec *start, struct timespec *end)
{
	unsigned long msecs;

	msecs = (end->tv_sec - start->tv_sec) * MSEC_PER_SEC;
	msecs += (end->tv_nsec - start->tv_nsec) / NSEC_PER_MSEC;

	printf("cache maintenance took %lu milliseconds\n", msecs);
}

static void run_test(void)
{
	void *dst = setup_test_buffer();
	void *src = setup_test_buffer();
	struct timespec start, end, dc_start, dc_end;
	size_t offset, i;

	printf("Iters:	%lu\n", test_iterations);
	printf("Mode:	%d\n", test_mode);
	printf("Source:	[%p, %p)\n", src, src + test_size);
	printf("Dest:	[%p, %p)\n", dst, dst + test_size);

	assert(!clock_gettime(CLOCK_MONOTONIC, &start));

	for (i = 0; i < test_iterations; i++) {
		if (test_mode == LDP_STP_DC_CIVAC) {
			assert(!clock_gettime(CLOCK_MONOTONIC, &dc_start));
			do_dcache_maintenance(src);
			assert(!clock_gettime(CLOCK_MONOTONIC, &dc_end));
		}

		for (offset = 0; offset < test_size; offset += chunk_size) {
			do_memcpy(dst + offset, src + offset, chunk_size);
			assert(!memchr(dst + offset, 0, chunk_size));
		}
	}

	assert(!clock_gettime(CLOCK_MONOTONIC, &end));

	report_bandwidth(&start, &end);

	if (test_mode == LDP_STP_DC_CIVAC)
		report_dcache_time(&dc_start, &dc_end);

	destroy_test_buffer(dst);
	destroy_test_buffer(src);
}

int main(int argc, char **argv)
{
	int c;

	while ((c = getopt(argc, argv, "Hhc:s:i:m:t:")) != -1) {
		switch (c) {
		case 'c':
			chunk_size = strtoul(optarg, NULL, 0);
			if ((chunk_size % 64 != 0) || (chunk_size < 128)) {
				pr_help(argv[0]);
				return 1;
			}
			break;
		case 's':
			test_size = strtoul(optarg, NULL, 0);
			if ((test_size % 64 != 0) || (test_size < 128)) {
				pr_help(argv[0]);
				return 1;
			}
			break;
		case 'i':
			test_iterations = strtoul(optarg, NULL, 0);
			break;
		case 'm':
			test_mode = strtoul(optarg, NULL, 0);
			if (test_mode >= NR_MODES) {
				pr_help(argv[0]);
				return 1;
			}
			break;
		case 'H':
			use_hugetlb = true;
			break;
		case 't':
			dcache_worker_threads = strtoul(optarg, NULL, 0);
			break;
		case 'h':
			pr_help(argv[0]);
			return 0;
		}
	}

	if (chunk_size > test_size || test_size % chunk_size != 0 ||
            test_size % dcache_worker_threads != 0) {
		pr_help(argv[0]);
		return 1;
	}

	run_test();
	return 0;
}