Add some new error testing toys:

cmcistorm - inject a bunch of corrected errors, then trigger them all quickly

hornet - inject a UC memory error into some other process

einj_mem_uc - inject a UC error and then trigger it in one of a variety of ways.

10 files changed, 1052 insertions, 0 deletions

diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..d7c83c6
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,21 @@
+CFLAGS	= -O
+
+all: mca-recover vtop cmcistorm hornet einj_mem_uc
+
+clean:
+	rm -f *.o mca-recover vtop cmcistorm hornet einj_mem_uc
+
+mca-recover: mca-recover.c
+	cc -o mca-recover $(CFLAGS) mca-recover.c
+
+vtop: vtop.c
+	cc -o vtop $(CFLAGS) vtop.c
+
+cmcistorm: cmcistorm.o proc_pagemap.o
+	cc -o cmcistorm $(CFLAGS) cmcistorm.o proc_pagemap.o
+
+hornet: hornet.c
+	cc -o hornet $(CFLAGS) hornet.c
+
+einj_mem_uc: einj_mem_uc.o proc_cpuinfo.o proc_interrupt.o proc_pagemap.o do_memcpy.o
+	cc -o einj_mem_uc einj_mem_uc.o proc_cpuinfo.o proc_interrupt.o proc_pagemap.o do_memcpy.o
diff --git a/cmcistorm.c b/cmcistorm.c
new file mode 100644
index 0000000..4f2d38a
--- /dev/null
+++ b/cmcistorm.c
@@ -0,0 +1,101 @@
+/*
+ * Copyright (C) 2015 Intel Corporation
+ * Author: Tony Luck
+ *
+ * This software may be redistributed and/or modified under the terms of
+ * the GNU General Public License ("GPL") version 2 only as published by the
+ * Free Software Foundation.
+ */
+
+/*
+ * Allocate memory - use EINJ to inject a bunch of soft errors,
+ * then consume them all as fast a possible.
+ */
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <string.h>
+#include <ctype.h>
+
+#define EINJ_ETYPE "/sys/kernel/debug/apei/einj/error_type"
+#define EINJ_ADDR "/sys/kernel/debug/apei/einj/param1"
+#define EINJ_MASK "/sys/kernel/debug/apei/einj/param2"
+#define EINJ_NOTRIGGER "/sys/kernel/debug/apei/einj/notrigger"
+#define EINJ_DOIT "/sys/kernel/debug/apei/einj/error_inject"
+
+volatile int trigger;
+
+#define	BUFSZ	(64 * 1024)
+
+static void wfile(char *file, unsigned long val)
+{
+	FILE *fp;
+	static int total_errors;
+
+tryagain:
+	fp = fopen(file, "w");
+	if (fp == NULL) {
+		perror(file);
+		exit(1);
+	}
+	fprintf(fp, "0x%lx\n", val);
+	if (fclose(fp) == EOF) {
+		perror(file);
+		if (++total_errors == 10)
+			exit(1);
+		sleep(3);
+		goto tryagain;
+	}
+}
+
+static void inject(int nerrors, double interval)
+{
+	char	*b, *buf;
+	long long paddr;
+	extern long long vtop(char *);
+	int	i;
+	unsigned long s, e;
+	int	bufsz = nerrors * 4096;
+
+	buf = malloc(bufsz);
+	if (buf == NULL) {
+		perror("malloc");
+		exit(1);
+	}
+	memset(buf, '*', bufsz);
+
+	for (i = 0; i < nerrors; i++) {
+		b = buf + i * 4096;
+		paddr = vtop(b);
+
+		printf("%d: vaddr = %p paddr = %llx\n", i, b, paddr);
+		wfile(EINJ_ADDR, paddr);
+		wfile(EINJ_DOIT, 1);
+
+		/* wait a bit to make sure SMI is all done on all cpus */
+		usleep((int)(interval * 1.0e6));
+	}
+
+
+	/* Trigger error by reading from target location */
+	for (i = 0; i < bufsz; i++)
+		trigger += *(buf + i);
+
+	/* wait a bit to allow CMCI handlers to complete */
+	usleep((int)(interval * 1.0e6));
+}
+
+int main(int argc, char **argv)
+{
+	int nerrors = (argc > 1) ? atoi(argv[1]) : 20;
+	double interval = (argc > 2) ? atof(argv[2]) : 1.0;
+
+	wfile(EINJ_ETYPE, 0x8);
+	wfile(EINJ_MASK, ~0x0ul);
+	wfile(EINJ_NOTRIGGER, 1);
+
+	inject(nerrors, interval);
+
+	return 0;
+}
diff --git a/do_memcpy.S b/do_memcpy.S
new file mode 100644
index 0000000..7768437
--- /dev/null
+++ b/do_memcpy.S
@@ -0,0 +1,9 @@
+	.globl do_memcpy
+	.type   do_memcpy, @function
+do_memcpy:
+	.cfi_startproc
+	rep movsq %ds:(%rsi),%es:(%rdi)
+	mov    $0x0,%eax
+	retq
+	.cfi_endproc
+	.size   do_memcpy, .-do_memcpy
diff --git a/einj_mem_uc.c b/einj_mem_uc.c
new file mode 100644
index 0000000..f239210
--- /dev/null
+++ b/einj_mem_uc.c
@@ -0,0 +1,422 @@
+/*
+ * Copyright (C) 2015 Intel Corporation
+ * Author: Tony Luck
+ *
+ * This software may be redistributed and/or modified under the terms of
+ * the GNU General Public License ("GPL") version 2 only as published by the
+ * Free Software Foundation.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <string.h>
+#include <time.h>
+#include <sys/mman.h>
+#include <setjmp.h>
+#include <signal.h>
+
+extern long long vtop(long long);
+
+static char *progname;
+static int nsockets, ncpus, lcpus_persocket;
+static int force_flag;
+static int all_flag;
+static long pagesize;
+#define	CACHE_LINE_SIZE	64
+
+#define EINJ_ETYPE "/sys/kernel/debug/apei/einj/error_type"
+#define EINJ_ADDR "/sys/kernel/debug/apei/einj/param1"
+#define EINJ_MASK "/sys/kernel/debug/apei/einj/param2"
+#define EINJ_NOTRIGGER "/sys/kernel/debug/apei/einj/notrigger"
+#define EINJ_DOIT "/sys/kernel/debug/apei/einj/error_inject"
+
+static void wfile(char *file, unsigned long long val)
+{
+	FILE *fp = fopen(file, "w");
+
+	if (fp == NULL) {
+		fprintf(stderr, "%s: cannot open '%s'\n", progname, file);
+		exit(1);
+	}
+	fprintf(fp, "0x%llx\n", val);
+	if (fclose(fp) == EOF) {
+		fprintf(stderr, "%s: write error on '%s'\n", progname, file);
+		exit(1);
+	}
+}
+
+static void inject_uc(unsigned long long addr, int notrigger)
+{
+	wfile(EINJ_ETYPE, 0x10);
+	wfile(EINJ_ADDR, addr);
+	wfile(EINJ_MASK, ~0x0ul);
+	wfile(EINJ_NOTRIGGER, notrigger);
+	wfile(EINJ_DOIT, 1);
+}
+
+static void check_configuration(void)
+{
+	char	model[512];
+
+	pagesize = getpagesize();
+
+	if (getuid() != 0) {
+		fprintf(stderr, "%s: must be root to run error injection tests\n", progname);
+		exit(1);
+	}
+	if (access("/sys/firmware/acpi/tables/EINJ", R_OK) == -1) {
+		fprintf(stderr, "%s: Error injection not supported, check your BIOS settings\n", progname);
+		exit(1);
+	}
+	if (access(EINJ_NOTRIGGER, R_OK|W_OK) == -1) {
+		fprintf(stderr, "%s: Is the einj.ko module loaded?\n", progname);
+		exit(1);
+	}
+	model[0] = '\0';
+	proc_cpuinfo(&nsockets, &ncpus, model);
+	if (nsockets == 0 || ncpus == 0) {
+		fprintf(stderr, "%s: could not find number of sockets/cpus\n", progname);
+		exit(1);
+	}
+	if (ncpus % nsockets) {
+		fprintf(stderr, "%s: strange topology. Are all cpus online?\n", progname);
+		exit(1);
+	}
+	lcpus_persocket = ncpus / nsockets;
+	if (!force_flag && strstr(model, "E7-") == NULL) {
+		fprintf(stderr, "%s: warning: cpu may not support recovery\n", progname);
+		exit(1);
+	}
+}
+
+#define REP9(stmt) stmt;stmt;stmt;stmt;stmt;stmt;stmt;stmt;stmt
+
+volatile int vol;
+
+int dosums(void)
+{
+	vol = 0;
+	REP9(REP9(REP9(vol++)));
+	return vol;
+}
+
+#define MB(n)	((n) * 1024 * 1024)
+
+static void *thp_data_alloc(void)
+{
+	char	*p = malloc(MB(128));
+	int	i;
+
+	if (p == NULL) {
+		fprintf(stderr, "%s: cannot allocate memory\n", progname);
+		exit(1);
+	}
+	srandom(getpid() * time(NULL));
+	for (i = 0; i < MB(128); i++)
+		p[i] = random();
+	return p + MB(64);
+}
+
+static void *data_alloc(void)
+{
+	char	*p = mmap(NULL, pagesize, PROT_READ|PROT_WRITE, MAP_SHARED|MAP_ANON, -1, 0);
+	int	i;
+
+	if (p == NULL) {
+		fprintf(stderr, "%s: cannot allocate memory\n", progname);
+		exit(1);
+	}
+	srandom(getpid() * time(NULL));
+	for (i = 0; i < pagesize; i++)
+		p[i] = random();
+	return p + CACHE_LINE_SIZE;
+}
+
+static void *instr_alloc(void)
+{
+	char	*p = (char *)dosums;
+
+	p += 2 * pagesize;
+
+	return (void *)((long)p & ~(pagesize - 1));
+}
+
+int trigger_single(char *addr)
+{
+	return addr[0];
+}
+
+int trigger_double(char *addr)
+{
+	return addr[0] + addr[1];
+}
+
+int trigger_split(char *addr)
+{
+	long *a = (long *)(addr - 1);
+
+	return a[0];
+}
+
+int trigger_write(char *addr)
+{
+	addr[0] = 'a';
+	return 0;
+}
+
+int trigger_memcpy(char *addr)
+{
+	/* phantom arg3 so values in %rdi,%rsi,%ecx are right for "rep mov" */
+	do_memcpy(addr + 1024, addr, 0, 512);
+	return 0;
+}
+
+int trigger_copyin(char *addr)
+{
+	int	fd, ret;
+	char	filename[] = "/tmp/einj-XXXXXX";
+
+	if ((fd = mkstemp(filename)) == -1) {
+		fprintf(stderr, "%s: couldn't make temp file\n", progname);
+		return -1;
+	}
+	(void)unlink(filename);
+	if ((ret = write(fd, addr, 16) != 16)) {
+		if (ret == -1)
+			fprintf(stderr, "%s: couldn't write temp file\n", progname);
+		else
+			fprintf(stderr, "%s: short write to temp file\n", progname);
+	}
+	close(fd);
+	return 0;
+}
+
+int trigger_patrol(char *addr)
+{
+	sleep(1);
+}
+
+int trigger_instr(char *addr)
+{
+	int ret = dosums();
+
+	if (ret != 729)
+		printf("Corruption during instruction fault recovery (%d)\n", ret);
+
+	return ret;
+}
+
+/* attributes of the test and which events will follow our trigger */
+#define	F_MCE		1
+#define	F_CMCI		2
+#define F_SIGBUS	4
+#define	F_FATAL		8
+
+struct test {
+	char	*testname;
+	char	*testhelp;
+	void	*(*alloc)(void);
+	int	notrigger;
+	int	(*trigger)(char *);
+	int	flags;
+} tests[] = {
+	{
+		"single", "Single read in pipeline to target address, generates SRAR machine check",
+		data_alloc, 1, trigger_single, F_MCE|F_CMCI|F_SIGBUS,
+	},
+	{
+		"double", "Double read in pipeline to target address, generates SRAR machine check",
+		data_alloc, 1, trigger_double, F_MCE|F_CMCI|F_SIGBUS,
+	},
+	{
+		"split", "Unaligned read crosses cacheline from good to bad. Probably fatal",
+		data_alloc, 1, trigger_split, F_MCE|F_CMCI|F_SIGBUS|F_FATAL,
+	},
+	{
+		"THP", "Try to inject in transparent huge page, generates SRAR machine check",
+		thp_data_alloc, 1, trigger_single, F_MCE|F_CMCI|F_SIGBUS,
+	},
+	{
+		"store", "Write to target address. Should generate a UCNA/CMCI",
+		data_alloc, 1, trigger_write, F_CMCI,
+	},
+	{
+		"memcpy", "Streaming read from target address. Probably fatal",
+		data_alloc, 1, trigger_memcpy, F_MCE|F_CMCI|F_SIGBUS|F_FATAL,
+	},
+	{
+		"instr", "Instruction fetch. Generates SRAR that OS should transparently fix",
+		instr_alloc, 1, trigger_instr, F_MCE|F_CMCI,
+	},
+	{
+		"patrol", "Patrol scrubber, generates SRAO machine check",
+		data_alloc, 0, trigger_patrol, F_MCE,
+	},
+	{
+		"copyin", "Kernel copies data from user. Probably fatal",
+		data_alloc, 1, trigger_copyin, F_MCE|F_CMCI|F_SIGBUS|F_FATAL,
+	},
+	{ NULL }
+};
+
+static void show_help(void)
+{
+	struct test *t;
+
+	printf("Usage: %s [-a][-c count][-d delay][-f] [testname]\n", progname);
+	printf("  %-8s %-5s %s\n", "Testname", "Fatal", "Description");
+	for (t = tests; t->testname; t++)
+		printf("  %-8s %-5s %s\n", t->testname,
+			(t->flags & F_FATAL) ? "YES" : "no",
+			t->testhelp);
+	exit(0);
+}
+
+static struct test *lookup_test(char *s)
+{
+	struct test *t;
+
+	for (t = tests; t->testname; t++)
+		if (strcmp(s, t->testname) == 0)
+			return t;
+	fprintf(stderr, "%s: unknown test '%s'\n", progname, s);
+	exit(1);
+}
+
+static struct test *next_test(struct test *t)
+{
+	t++;
+	if (t->testname == NULL)
+		t = tests;
+	return t;
+}
+
+static jmp_buf env;
+
+static void recover(int sig, siginfo_t *si, void *v)
+{
+	printf("SIGBUS: addr = %p\n", si->si_addr);
+	siglongjmp(env, 1);
+}
+
+struct sigaction recover_act = {
+	.sa_sigaction = recover,
+	.sa_flags = SA_SIGINFO,
+};
+
+int main(int argc, char **argv)
+{
+	int c, i;
+	int	count = 1;
+	double	delay = 1.0;
+	struct test *t;
+	void	*vaddr;
+	long long paddr;
+	long	b_mce, b_cmci, a_mce, a_cmci;
+
+	progname = argv[0];
+
+	while ((c = getopt(argc, argv, "ac:d:fh")) != -1) switch (c) {
+	case 'a':
+		all_flag = 1;
+		break;
+	case 'c':
+		count = strtol(optarg, NULL, 0);
+		break;
+	case 'd':
+		delay = strtod(optarg, NULL);
+		break;
+	case 'f':
+		force_flag = 1;
+		break;
+	case 'h': case '?':
+		show_help();
+		break;
+	}
+
+	check_configuration();
+
+	if (optind < argc)
+		t = lookup_test(argv[optind]);
+	else
+		t = tests;
+
+	if ((t->flags & F_FATAL) && !force_flag) {
+		fprintf(stderr, "%s: selected test may be fatal. Use '-f' flag if you really want to do this\n", progname);
+		exit(1);
+	}
+
+	sigaction(SIGBUS, &recover_act, NULL);
+
+	for (i = 0; i < count; i++) {
+		vaddr = t->alloc();
+		paddr = vtop((long long)vaddr);
+		printf("%d: %-8s vaddr = %p paddr = %llx\n", i, t->testname, vaddr, paddr);
+
+		proc_interrupts(&b_mce, &b_cmci);
+		if (sigsetjmp(env, 1)) {
+			if ((t->flags & F_SIGBUS) == 0) {
+				printf("Unexpected SIGBUS\n");
+			}
+		} else {
+			inject_uc(paddr, t->notrigger);
+			t->trigger(vaddr);
+			if (t->flags & F_SIGBUS) {
+				printf("Expected SIGBUS, didn't get one\n");
+			}
+		}
+		/* if system didn't already take page offline, ask it to do so now */
+		if (paddr == vtop((long long)vaddr)) {
+			printf("Manually take page offline\n");
+			wfile("/sys/devices/system/memory/hard_offline_page", paddr);
+		}
+
+		/* Give system a chance to process on possibly deep C-state idle cpus */
+		usleep(1000);
+
+		proc_interrupts(&a_mce, &a_cmci);
+
+		if (t->flags & F_FATAL) {
+			printf("Big surprise ... still running. Thought that would be fatal\n");
+		}
+
+		if (t->flags & F_MCE) {
+			if (a_mce == b_mce) {
+				printf("Expected MCE, but none seen\n");
+			} else if (a_mce == b_mce + 1) {
+				printf("Saw local machine check\n");
+			} else if (a_mce == b_mce + ncpus) {
+				printf("Saw broadcast machine check\n");
+			} else {
+				printf("Unusual number of MCEs seen: %ld\n", a_mce - b_mce);
+			}
+		} else {
+			if (a_mce != b_mce) {
+				printf("Saw %ld unexpected MCEs (%ld systemwide)\n", b_mce - a_mce, (b_mce - a_mce) / ncpus);
+			}
+		}
+
+		if (t->flags & F_CMCI) {
+			if (a_cmci == b_cmci) {
+				printf("Expected CMCI, but none seen\n");
+			} else if (a_cmci < b_cmci + lcpus_persocket) {
+				printf("Unusual number of CMCIs seen: %ld\n", a_cmci - b_cmci);
+			}
+		} else {
+			if (a_cmci != b_cmci) {
+				printf("Saw %ld unexpected CMCIs (%ld per socket)\n", a_cmci - b_cmci, (a_cmci - b_cmci) / lcpus_persocket);
+			}
+		}
+
+		usleep((useconds_t)(delay * 1.0e6));
+		if (all_flag) {
+			t = next_test(t);
+			while (t->flags & F_FATAL)
+				t = next_test(t);
+		}
+	}
+
+	return 0;
+}
diff --git a/hornet.8 b/hornet.8
new file mode 100644
index 0000000..0488915
--- /dev/null
+++ b/hornet.8
@@ -0,0 +1,93 @@
+.TH HORNET 8 2015-07-07 Linux "System Validation Manual"
+.SH NAME
+hornet \- inject an uncorrectable memory error into a process
+.SH SYNOPSIS
+.B hornet
+[
+.B \-D
+.I delay
+] [
+.B \-v
+] [
+.I address\-options
+]
+.I COMMAND
+[
+.I ARGS
+]
+.br
+.B hornet
+[
+.B \-D
+.I delay
+] [
+.B \-v
+] [
+.I address\-options
+]
+\-p
+.I PID
+.SH CONFIGURATION
+.B hornet
+requires that error injection be enabled in ACPI 5.0 mode by the BIOS and that the
+error injection driver
+.B einj
+is loaded. Running on a processor that does not support error recovery
+will crash the system when the injected error is consumed.
+.SH DESCRIPTION
+.B hornet
+injects an uncorrectable memory error into a program to test recovery paths.
+It waits for the process to terminate and tries to take the target page offline
+if the application did not trigger an error by accessing the injected address.
+.SH OPTIONS
+.TP
+.BI \-D " delay"
+Pause for
+.I delay
+seconds before injecting the memory error. A floating point value may be specified.
+.TP
+.B \-v
+print verbose details about execution progress.
+.TP
+.B address-options
+.BI \-a " address"
+inject at the specified virtual address
+.br
+.B \-t
+choose an address randomly from the text (code) section
+.br
+.B \-d
+choose an address randomly from the initialized data section
+.br
+.B \-b
+choose an address randomly from the uninitialized data section (heap)
+.br
+.B \-s
+choose an address randomly from the stack section
+.br
+.B \-m
+choose an address randomly from an anonymous mapped section
+.br
+If none of the address options are specified
+.B hornet
+will choose an address from the largest writable address region
+in the process.
+.TP
+.BI \-p " PID"
+inject errors into an already running process
+.SH EXIT STATUS
+.SH FILES
+.PP
+/sys/kernel/debug/apei/einj/*
+.RS 4
+Error injection interface.
+.RE
+.PP
+/sys/devices/system/memory/hard_offline_page
+.RS 4
+Interface to ask kernel to take pages offline.
+.RE
+.SH NOTES
+.SH BUGS
+.SH EXAMPLE
+.SH SEE ALSO
diff --git a/hornet.c b/hornet.c
new file mode 100644
index 0000000..2c1b1fa
--- /dev/null
+++ b/hornet.c
@@ -0,0 +1,252 @@
+/*
+ * Copyright (C) 2014 Intel Corporation
+ * Authors: Tony Luck
+ *
+ * This software may be redistributed and/or modified under the terms of
+ * the GNU General Public License ("GPL") version 2 only as published by the
+ * Free Software Foundation.
+ */
+
+/*
+ * hornet: Start a process (or point to an existing one) and inject
+ * an uncorrectable memory error to a targeted or randomly chosen
+ * memory address
+ */
+
+#include <unistd.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/signal.h>
+#include <sys/wait.h>
+
+static char *progname;
+
+long	addr;
+double	delay;
+int	pid;
+int	tflag, dflag, bflag, sflag, mflag, pflag, vflag;
+
+static void usage(void)
+{
+	fprintf(stderr, "Usage: %s [hornetopts] -p PID\n", progname);
+	fprintf(stderr, "Usage: %s [hornetopts] command args ...\n", progname);
+	fprintf(stderr, "  hornetopts = [-D delay][ -a ADDRESS][-t|-d|-b|-s|-m]\n");
+	exit(1);
+}
+
+#define EINJ_ETYPE "/sys/kernel/debug/apei/einj/error_type"
+#define EINJ_ADDR "/sys/kernel/debug/apei/einj/param1"
+#define EINJ_MASK "/sys/kernel/debug/apei/einj/param2"
+#define EINJ_NOTRIGGER "/sys/kernel/debug/apei/einj/notrigger"
+#define EINJ_DOIT "/sys/kernel/debug/apei/einj/error_inject"
+
+static void wfile(char *file, unsigned long val)
+{
+	FILE *fp;
+
+	fp = fopen(file, "w");
+	if (fp == NULL) {
+		perror(file);
+		exit(1);
+	}
+	fprintf(fp, "0x%lx\n", val);
+	if (fclose(fp) == EOF) {
+		perror(file);
+		exit(1);
+	}
+}
+
+static int startproc(char **args)
+{
+	int	pid;
+
+	switch ((pid = fork())) {
+	case -1:
+		perror("fork");
+		exit(1);
+	case 0:
+		execvp(args[0], args);
+		fprintf(stderr, "%s: cannot run '%s'\n", progname, args[0]);
+		exit(1);
+	}
+	return pid;
+}
+
+static void parsemaps(int pid, long *lo, long *hi)
+{
+	char mapfile[32], perm[10], file[4096], line[4096];
+	long vstart, vend;
+	int pgoff, maj, min, ino;
+	char *p;
+	long sz, maxsz = -1, vmstart, vmend;
+	FILE *fp;
+
+	sprintf(mapfile, "/proc/%d/maps", pid);
+
+	if ((fp = fopen(mapfile, "r")) == NULL) {
+		fprintf(stderr, "%s: can't open %s\n", progname, mapfile);
+		exit(1);
+	}
+
+	while ((p = fgets(line, sizeof line, fp)) != NULL) {
+		file[0] = '\0';
+		if (sscanf(line, "%lx-%lx %s %x %d:%d %d %s\n",
+		&vstart, &vend, perm, &pgoff, &maj, &min, &ino, file) >= 7) {
+			sz = vend - vstart;
+			if (strcmp(perm, "rw-p") == 0 && file[0] == '\0' && sz > maxsz) {
+				vmstart = vstart;
+				vmend = vend;
+				maxsz = sz;
+			}
+			if (tflag && strcmp(perm, "r-xp") == 0 && file[0] == '/')
+				break;
+			if (dflag && strcmp(perm, "rw-p") == 0 && file[0] == '/')
+				break;
+			if (bflag && strcmp(perm, "rw-p") == 0 && file[0] == '\0' && vstart < 0x400000000000)
+				break;
+			if (sflag && strcmp(perm, "rw-p") == 0 && strcmp(file, "[stack]") == 0)
+				break;
+			if (mflag && strcmp(perm, "rw-p") == 0 && file[0] == '\0' && vstart > 0x400000000000)
+				break;
+			if (addr && addr >= vstart && addr < vend)
+				break;
+		}
+	}
+	fclose(fp);
+
+	if (p) {
+		*lo = vstart; *hi = vend;
+		return;
+	}
+	if (!tflag && !dflag && !bflag && !sflag && !mflag && addr == 0) {
+		*lo = vmstart; *hi = vmend;
+		return;
+	}
+	fprintf(stderr, "%s: can't find suitable address range\n", progname);
+	exit(1);
+}
+
+static long randaddr(long lo, long hi)
+{
+	long sz = hi - lo;
+	long a;
+
+	srandom(getpid() ^ time(0));
+	a = lo + sz/10 + (long)(sz * 0.8 * random() / RAND_MAX);
+
+	return a & ~0x3ful;
+}
+
+	
+static long pickaddr(int pid, long lo, long hi, long *phys)
+{
+	int pagesize = getpagesize();
+	unsigned long pinfo;
+	long offset;
+	int fd, skip = 0;
+	char pagemap[32];
+	long a;
+
+	sprintf(pagemap, "/proc/%d/pagemap", pid);
+	fd = open(pagemap, O_RDONLY);
+	if (fd == -1) {
+		fprintf(stderr, "%s: cannot open pagemap for pid=%d\n", progname, pid);
+		return -1;
+	}
+	if (addr)
+		a = addr;
+	else
+		a = randaddr(lo, hi);
+again:
+	if (vflag) printf("checking virtual address 0x%lx in [0x%lx,0x%lx]\n", a, lo, hi);
+	offset = a / pagesize * (sizeof pinfo);
+	if (pread(fd, &pinfo, sizeof pinfo, offset) != sizeof pinfo) {
+		fprintf(stderr, "%s: cannot read pagemap for pid=%d addr=%lx\n", progname, pid, a);
+		goto fail;
+	}
+	if ((pinfo & (1ul << 63)) == 0) {
+		if (addr) {
+			fprintf(stderr, "%s: chosen address %lx not allocated\n", progname, addr);
+			goto fail;
+		}
+		skip = (skip <= 0) ? -skip + 1 : -(skip + 1);
+		a += pagesize * skip;
+		if (vflag) printf("skip=%d new addr=0x%lx\n", skip, a);
+		if (a < lo || a >= hi) {
+			fprintf(stderr, "%s: could not find allocated address\n", progname);
+			goto fail;
+		}
+		goto again;
+	}
+	*phys = ((pinfo & 0x007ffffffffffffful) << 12) + (a & (pagesize - 1));
+	return a;
+fail:
+	close(fd);
+	return -1;
+}
+
+int main(int argc, char **argv)
+{
+	int	c;
+	int	status;
+	long	lo, hi, phys, virt;
+
+	progname = argv[0];
+
+	while ((c = getopt(argc, argv, "D:a:tdbsmp:v")) != -1) switch (c) {
+	case 'D': delay = atof(optarg); break;
+	case 'a': addr = strtol(optarg, NULL, 0); break;
+	case 't': tflag = 1; break;
+	case 'd': dflag = 1; break;
+	case 'b': bflag = 1; break;
+	case 's': sflag = 1; break;
+	case 'm': mflag = 1; break;
+	case 'p': pflag = 1; pid = atoi(optarg); break;
+	case 'v': vflag++; break;
+	default: usage(); break;
+	}
+
+	wfile(EINJ_ETYPE, 0x10);
+	wfile(EINJ_MASK, ~0x0ul);
+	wfile(EINJ_NOTRIGGER, 1);
+
+	if (!pflag)
+		pid = startproc(&argv[optind]);
+	if (delay != 0.0)
+		usleep((useconds_t)(delay * 1.0e6));
+	if (kill(pid, SIGSTOP) == -1) {
+		fprintf(stderr, "%s: cannot stop process\n", progname);
+		return 1;
+	}
+
+	parsemaps(pid, &lo, &hi);
+
+	if ((virt = pickaddr(pid, lo, hi, &phys)) == -1) {
+		kill(pid, SIGKILL);
+		return 1;
+	}
+
+	wfile(EINJ_ADDR, phys);
+	wfile(EINJ_DOIT, 1);
+
+	if (vflag) printf("%s: injected UC error at virt=%lx phys=%lx to pid=%d\n",
+		progname, virt, phys, pid);
+
+	if (kill(pid, SIGCONT) == -1) {
+		fprintf(stderr, "%s: cannot resume process\n", progname);
+		return 1;
+	}
+	if (pflag) {
+		while (kill(pid, 0) != -1)
+			usleep(1000000);
+	} else {
+		while (wait(&status) != pid)
+			;
+		if (WIFSIGNALED(status) && WTERMSIG(status) == SIGBUS)
+			printf("%s: process terminated by SIGBUS\n", progname);
+	}
+	wfile("/sys/devices/system/memory/hard_offline_page", phys);
+	return 0;
+}
diff --git a/proc_cpuinfo.c b/proc_cpuinfo.c
new file mode 100644
index 0000000..540dcd1
--- /dev/null
+++ b/proc_cpuinfo.c
@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2015 Intel Corporation
+ * Author: Tony Luck
+ *
+ * This software may be redistributed and/or modified under the terms of
+ * the GNU General Public License ("GPL") version 2 only as published by the
+ * Free Software Foundation.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+
+/*
+ * read /proc/cpuinfo to discover:
+ * 1) number of cpu sockets
+ * 2) Number of logical cpus
+ * 3) Model name of the cpu
+ */
+void proc_cpuinfo(int *nsockets, int *ncpus, char *model)
+{
+	FILE	*fp = fopen("/proc/cpuinfo", "r");
+	char	*p, line[4096];
+	long	s, sockmask = 0;
+	int	i;
+
+	*nsockets = 0;
+	*ncpus = 0;
+
+	if (fp == NULL)
+		return;
+
+	while (fgets(line, sizeof line, fp)) {
+		if (model[0] == '\0' && strncmp(line, "model name", 10) == 0) {
+			p = strchr(&line[10], ':');
+			while (isspace(*++p))
+				;
+			strcpy(model, p);
+		} else if (strncmp(line, "physical id", 11) == 0) {
+			(*ncpus)++;
+			p = strchr(&line[10], ':');
+			s = strtol(p+1, NULL, 10);
+			sockmask |= 1 << s;
+		}
+	}
+	for (i = 0; i < 8 * sizeof sockmask; i++)
+		if (sockmask & (1l << i))
+			(*nsockets)++;
+
+	fclose(fp);
+}
diff --git a/proc_interrupt.c b/proc_interrupt.c
new file mode 100644
index 0000000..eca5a03
--- /dev/null
+++ b/proc_interrupt.c
@@ -0,0 +1,53 @@
+/*
+ * Copyright (C) 2015 Intel Corporation
+ * Author: Tony Luck
+ *
+ * This software may be redistributed and/or modified under the terms of
+ * the GNU General Public License ("GPL") version 2 only as published by the
+ * Free Software Foundation.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
+
+static long sumint(char *s)
+{
+	long total = 0;
+	char	*se;
+
+	for (;;) {
+		total += strtol(s, &se, 10);
+		if (s == se)
+			break;
+		s = se;
+	}
+
+	return total;
+}
+
+/*
+ * Parse /proc/interrupts to sum the number of observed
+ * machine checks and corrected machine check interrupts
+ * across all cpus
+ */
+void proc_interrupts(long *nmce, long *ncmci)
+{
+	FILE *fp = fopen("/proc/interrupts", "r");
+	char	*p, line[4096];
+
+	*ncmci = *nmce = -1;
+	if (fp == NULL)
+		return;
+
+	while (fgets(line, sizeof(line), fp) != NULL) {
+		for (p = line; isspace(*p); p++)
+			;
+		if (strncmp(p, "MCE:", 4) == 0)
+			*nmce = sumint(p+4);
+		else if (strncmp(p, "THR:", 4) == 0)
+			*ncmci = sumint(p+4);
+	}
+
+	fclose(fp);
+}
diff --git a/proc_pagemap.c b/proc_pagemap.c
new file mode 100644
index 0000000..bfc29ed
--- /dev/null
+++ b/proc_pagemap.c
@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2015 Intel Corporation
+ * Author: Tony Luck
+ *
+ * This software may be redistributed and/or modified under the terms of
+ * the GNU General Public License ("GPL") version 2 only as published by the
+ * Free Software Foundation.
+ */
+
+/*
+ * Convert a user mode virtual address belonging to the
+ * current process to physical.
+ * Does not handle huge pages.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+/*
+ * get information about address from /proc/self/pagemap
+ */
+unsigned long long vtop(unsigned long long addr)
+{
+	static int pagesize;
+	unsigned long long pinfo;
+	long offset;
+	int fd;
+
+	if (pagesize == 0)
+		pagesize = getpagesize();
+	offset = addr / pagesize * (sizeof pinfo);
+	fd = open("/proc/self/pagemap", O_RDONLY);
+	if (fd == -1) {
+		perror("pagemap");
+		exit(1);
+	}
+	if (pread(fd, &pinfo, sizeof pinfo, offset) != sizeof pinfo) {
+		perror("pagemap");
+		exit(1);
+	}
+	close(fd);
+	if ((pinfo & (1ull << 63)) == 0) {
+		printf("page not present\n");
+		return ~0ull;
+	}
+	return ((pinfo & 0x007fffffffffffffull) << 12) + (addr & (pagesize - 1));
+}
diff --git a/vtop.c b/vtop.c
index f292218..f292218 100755..100644
--- a/vtop.c
+++ b/vtop.c