iowatcher: Handle traces to more than once device at a time

Signed-off-by: Chris Mason <chris.mason@fusionio.com>

7 files changed, 340 insertions, 72 deletions

diff --git a/iowatcher/blkparse.c b/iowatcher/blkparse.c
index f81b831..43630be 100644
--- a/iowatcher/blkparse.c
+++ b/iowatcher/blkparse.c
@@ -30,6 +30,7 @@
 #include <sys/mman.h>
 #include <time.h>
 #include <math.h>
+#include <dirent.h>
 
 #include "plot.h"
 #include "blkparse.h"
@@ -48,6 +49,9 @@ static struct list_head process_hash_table[PROCESS_HASH_TABLE_SIZE];
 extern int plot_io_action;
 extern int io_per_process;
 
+static const int line_len = 1024;
+static char line[1024];
+
 /*
  * Trace categories
  */
@@ -158,6 +162,9 @@ struct pending_io {
 	/* sector offset of this IO */
 	u64 sector;
 
+	/* dev_t for this IO */
+	u32 device;
+
 	/* time this IO was dispatched */
 	u64 dispatch_time;
 	/* time this IO was finished */
@@ -219,20 +226,21 @@ static inline u64 hash_sector(u64 val)
 static int io_hash_table_insert(struct pending_io *ins_pio)
 {
 	u64 sector = ins_pio->sector;
+	u32 dev = ins_pio->device;
 	int slot = hash_sector(sector);
 	struct list_head *head;
 	struct pending_io *pio;
 
 	head = io_hash_table + slot;
 	list_for_each_entry(pio, head, hash_list) {
-		if (pio->sector == sector)
+		if (pio->sector == sector && pio->device == dev)
 			return -EEXIST;
 	}
 	list_add_tail(&ins_pio->hash_list, head);
 	return 0;
 }
 
-static struct pending_io *io_hash_table_search(u64 sector)
+static struct pending_io *io_hash_table_search(u64 sector, u32 dev)
 {
 	int slot = hash_sector(sector);
 	struct list_head *head;
@@ -240,7 +248,7 @@ static struct pending_io *io_hash_table_search(u64 sector)
 
 	head = io_hash_table + slot;
 	list_for_each_entry(pio, head, hash_list) {
-		if (pio->sector == sector)
+		if (pio->sector == sector && pio->device == dev)
 			return pio;
 	}
 	return NULL;
@@ -253,6 +261,7 @@ static struct pending_io *hash_queued_io(struct blk_io_trace *io)
 
 	pio = calloc(1, sizeof(*pio));
 	pio->sector = io->sector;
+	pio->device = io->device;
 	pio->pid = io->pid;
 
 	ret = io_hash_table_insert(pio);
@@ -268,7 +277,7 @@ static struct pending_io *hash_dispatched_io(struct blk_io_trace *io)
 {
 	struct pending_io *pio;
 
-	pio = io_hash_table_search(io->sector);
+	pio = io_hash_table_search(io->sector, io->device);
 	if (!pio) {
 		pio = hash_queued_io(io);
 		if (!pio)
@@ -282,7 +291,7 @@ static struct pending_io *hash_completed_io(struct blk_io_trace *io)
 {
 	struct pending_io *pio;
 
-	pio = io_hash_table_search(io->sector);
+	pio = io_hash_table_search(io->sector, io->device);
 
 	if (!pio)
 		return NULL;
@@ -513,6 +522,66 @@ out:
 	return -1;
 }
 
+static struct dev_info *lookup_dev(struct trace *trace, struct blk_io_trace *io)
+{
+	u32 dev = io->device;
+	int i;
+	struct dev_info *di = NULL;
+
+	for (i = 0; i < trace->num_devices; i++) {
+		if (trace->devices[i].device == dev) {
+			di = trace->devices + i;
+			goto found;
+		}
+	}
+	i = trace->num_devices++;
+	if (i >= MAX_DEVICES_PER_TRACE) {
+		fprintf(stderr, "Trace contains too many devices (%d)\n", i);
+		exit(1);
+	}
+	di = trace->devices + i;
+	di->device = dev;
+found:
+	return di;
+}
+
+static void map_devices(struct trace *trace)
+{
+	struct dev_info *di;
+	u64 found;
+	u64 map_start = 0;
+	int i;
+
+	first_record(trace);
+	while (1) {
+		if (!(trace->io->action & BLK_TC_ACT(BLK_TC_NOTIFY))) {
+			di = lookup_dev(trace, trace->io);
+			found = trace->io->sector << 9;
+			if (found < di->min)
+				di->min = found;
+
+			found += trace->io->bytes;
+			if (di->max < found)
+				di->max = found;
+		}
+		if (next_record(trace))
+			break;
+	}
+	first_record(trace);
+	for (i = 0; i < trace->num_devices; i++) {
+		di = trace->devices + i;
+		di->map = map_start;
+		map_start += di->max - di->min;
+	}
+}
+
+u64 map_io(struct trace *trace, struct blk_io_trace *io)
+{
+	struct dev_info *di = lookup_dev(trace, io);
+	u64 val = trace->io->sector << 9;
+	return di->map + val - di->min;
+}
+
 void find_extreme_offsets(struct trace *trace, u64 *min_ret, u64 *max_ret, u64 *max_bank_ret,
 			  u64 *max_offset_ret)
 {
@@ -521,10 +590,13 @@ void find_extreme_offsets(struct trace *trace, u64 *min_ret, u64 *max_ret, u64 *
 	u64 max_bank = 0;
 	u64 max_bank_offset = 0;
 	u64 num_banks = 0;
+
+	map_devices(trace);
+
 	first_record(trace);
 	while (1) {
 		if (!(trace->io->action & BLK_TC_ACT(BLK_TC_NOTIFY))) {
-			found = trace->io->sector << 9;
+			found = map_io(trace, trace->io);
 			if (found < min)
 				min = found;
 
@@ -591,7 +663,7 @@ int filter_outliers(struct trace *trace, u64 min_offset, u64 max_offset,
 		check_io_types(trace);
 		if (!(trace->io->action & BLK_TC_ACT(BLK_TC_NOTIFY)) &&
 		    (trace->io->action & BLK_TA_MASK) == __BLK_TA_QUEUE) {
-			u64 off = (trace->io->sector << 9) - min_offset;
+			u64 off = map_io(trace, trace->io) - min_offset;
 
 			slot = (int)(off / bytes_per_bucket);
 			hits[slot]++;
@@ -636,33 +708,139 @@ int filter_outliers(struct trace *trace, u64 min_offset, u64 max_offset,
 	return 0;
 }
 
+static char footer[] = ".blktrace.0";
+static int footer_len = sizeof(footer);
+
+static void match_trace(char *name, char **traces)
+{
+	int match_len;
+	char *match;
+	int footer_start;
+
+	match_len = strlen(name);
+	if (match_len <= footer_len)
+		return;
+
+	footer_start = match_len - footer_len;
+	if (strcmp(name + footer_start + 1, footer) != 0)
+		return;
+
+	match = strdup(name);
+	if (!match)
+		goto enomem;
+
+	match[footer_start + 1] = '\0';
+	snprintf(line, line_len, "%s -i '%s'", *traces ? *traces : "", match);
+	free(match);
+
+	match = strdup(line);
+	if (!match)
+		goto enomem;
+
+	free(*traces);
+	*traces = match;
+	return;
+
+enomem:
+	perror("memory allocation failed");
+	exit(1);
+	return;
+}
+
+static char *combine_blktrace_devs(char *dir_name)
+{
+	DIR *dir;
+	char *traces = NULL;
+	struct dirent *d;
+	int len;
+	int ret;
+
+	dir = opendir(dir_name);
+	if (!dir)
+		return NULL;
+
+	while (1) {
+		d = readdir(dir);
+		if (!d)
+			break;
+
+		len = strlen(d->d_name);
+		if (len > footer_len)
+			match_trace(d->d_name, &traces);
+	}
+
+	closedir(dir);
+
+	if (!traces)
+		return NULL;
+
+	snprintf(line, line_len, "blkparse -O %s -D %s -d '%s.%s'",
+		 traces, dir_name, dir_name, "dump");
+
+	ret = system(line);
+	if (ret) {
+		fprintf(stderr, "blkparse failure %s\n", line);
+		exit(1);
+	}
+	snprintf(line, line_len, "%s.%s", dir_name, "dump");
+	return strdup(line);
+}
+
 static char *find_trace_file(char *filename)
 {
 	int ret;
 	struct stat st;
-	char line[1024];
 	char *dot;
 	char *try;
+	int found_dir = 0;
 
+	/* look for an exact match of whatever they pass in.
+	 * If it is a file, assume it is the dump file.
+	 * If a directory, remember that it existed so we
+	 * can combine traces in that directory later
+	 */
 	ret = stat(filename, &st);
-	if (ret == 0)
-		return strdup(filename);
+	if (ret == 0) {
+		if (S_ISREG(st.st_mode))
+			return strdup(filename);
+
+		if (S_ISDIR(st.st_mode))
+			found_dir = 1;
+	}
 
-	snprintf(line, 1024, "%s.%s", filename, "dump");
+	/*
+	 * try tacking .dump onto the end and see if that already
+	 * has been generated
+	 */
+	snprintf(line, line_len, "%s.%s", filename, "dump");
 	ret = stat(line, &st);
 	if (ret == 0)
 		return strdup(line);
 
+	/*
+	 * try to generate the .dump from all the traces in
+	 * a single dir.
+	 */
+	if (found_dir) {
+		try = combine_blktrace_devs(filename);
+		if (try)
+			return try;
+	}
+
+	/*
+	 * try to generate the .dump from all the blktrace
+	 * files for a named trace
+	 */
 	try = strdup(filename);
 	dot = strrchr(try, '.');
 	if (!dot || strcmp(".dump", dot) != 0) {
 		if (dot && dot != try)
 			*dot = '\0';
-		snprintf(line, 1024, "%s%s", try, ".blktrace.0");
+		snprintf(line, line_len, "%s%s", try, ".blktrace.0");
 		ret = stat(line, &st);
 		if (ret == 0) {
 			blktrace_to_dump(try);
-			snprintf(line, 1024, "%s.%s", try, "dump");
+			snprintf(line, line_len, "%s.%s", try, "dump");
 			ret = stat(line, &st);
 			if (ret == 0) {
 				free(try);
@@ -762,9 +940,11 @@ static inline int io_event(struct trace *trace)
 	return __BLK_TA_COMPLETE;
 }
 
-void add_tput(struct trace *trace, struct graph_line_data *gld)
+void add_tput(struct trace *trace, struct graph_line_data *writes_gld,
+	      struct graph_line_data *reads_gld)
 {
 	struct blk_io_trace *io = trace->io;
+	struct graph_line_data *gld;
 	int action = io->action & BLK_TA_MASK;
 	int seconds;
 
@@ -774,11 +954,17 @@ void add_tput(struct trace *trace, struct graph_line_data *gld)
 	if (action != tput_event(trace))
 		return;
 
+	if (BLK_DATADIR(io->action) & BLK_TC_READ)
+		gld = reads_gld;
+	else
+		gld = writes_gld;
+
 	seconds = SECONDS(io->time);
 	if (seconds > gld->max_seconds)
 		return;
 
 	gld->data[seconds].sum += io->bytes;
+
 	gld->data[seconds].count = 1;
 	if (gld->data[seconds].sum > gld->max)
 		gld->max = gld->data[seconds].sum;
@@ -834,7 +1020,7 @@ void add_io(struct trace *trace, struct trace_file *tf)
 	if (action != io_event(trace))
 		return;
 
-	offset = io->sector << 9;
+	offset = map_io(trace, io);
 
 	pm = get_pid_map(tf, io->pid);
 	if (!pm) {
diff --git a/iowatcher/blkparse.h b/iowatcher/blkparse.h
index 0222e5e..d435a87 100644
--- a/iowatcher/blkparse.h
+++ b/iowatcher/blkparse.h
@@ -25,6 +25,15 @@
 #define DOUBLE_TO_NANO_ULL(d)   ((unsigned long long)((d) * 1000000000))
 #define CHECK_MAGIC(t)          (((t)->magic & 0xffffff00) == BLK_IO_TRACE_MAGIC)
 
+struct dev_info {
+	u32 device;
+	u64 min;
+	u64 max;
+	u64 map;
+};
+
+#define MAX_DEVICES_PER_TRACE 64
+
 struct trace {
 	int fd;
 	u64 len;
@@ -48,6 +57,8 @@ struct trace {
 	int mpstat_fd;
 	int mpstat_seconds;
 	int mpstat_num_cpus;
+	int num_devices;
+	struct dev_info devices[MAX_DEVICES_PER_TRACE];
 };
 
 struct trace_file {
@@ -61,9 +72,12 @@ struct trace_file {
 	u64 min_offset;
 	u64 max_offset;
 
+	char *reads_color;
+	char *writes_color;
 	char *line_color;
 
-	struct graph_line_data *tput_gld;
+	struct graph_line_data *tput_writes_gld;
+	struct graph_line_data *tput_reads_gld;
 	struct graph_line_data *iop_gld;
 	struct graph_line_data *latency_gld;
 	struct graph_line_data *queue_depth_gld;
@@ -104,7 +118,8 @@ void check_record(struct trace *trace);
 void add_completed_io(struct trace *trace,
 		      struct graph_line_data *latency_gld);
 void add_io(struct trace *trace, struct trace_file *tf);
-void add_tput(struct trace *trace, struct graph_line_data *gld);
+void add_tput(struct trace *trace, struct graph_line_data *writes_gld,
+	      struct graph_line_data *reads_gld);
 void add_pending_io(struct trace *trace, struct graph_line_data *gld);
 int next_record(struct trace *trace);
 void first_record(struct trace *trace);
diff --git a/iowatcher/main.c b/iowatcher/main.c
index 41b2828..fbd8a88 100644
--- a/iowatcher/main.c
+++ b/iowatcher/main.c
@@ -139,7 +139,8 @@ static int longest_label = 0;
 
 static char *graph_title = "";
 static char *output_filename = "trace.svg";
-static char *blktrace_device = NULL;
+static char *blktrace_devices[MAX_DEVICES_PER_TRACE];
+static int num_blktrace_devices = 0;
 static char *blktrace_outfile = "trace";
 static char *blktrace_dest_dir = ".";
 static char *program_to_run = NULL;
@@ -257,7 +258,8 @@ static void setup_trace_file_graphs(void)
 	else
 		alloc_ptrs = 1;
 	list_for_each_entry(tf, &all_traces, list) {
-		tf->tput_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
+		tf->tput_reads_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
+		tf->tput_writes_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
 		tf->latency_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
 		tf->queue_depth_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
 		tf->iop_gld = alloc_line_data(tf->min_seconds, tf->max_seconds, tf->stop_seconds);
@@ -300,6 +302,7 @@ static void read_traces(void)
 		tf->trace = trace;
 		tf->max_seconds = SECONDS(last_time) + 1;
 		tf->stop_seconds = SECONDS(last_time) + 1;
+
 		find_extreme_offsets(trace, &tf->min_offset, &tf->max_offset,
 				    &max_bank, &max_bank_offset);
 		filter_outliers(trace, tf->min_offset, tf->max_offset, &ymin, &ymax);
@@ -324,13 +327,19 @@ static void pick_line_graph_color(void)
 		for (i = 0; i < tf->io_plots; i++) {
 			if (tf->gdd_reads[i]) {
 				tf->line_color = tf->gdd_reads[i]->color;
-				break;
+				tf->reads_color = tf->gdd_reads[i]->color;
 			}
 			if (tf->gdd_writes[i]) {
 				tf->line_color = tf->gdd_writes[i]->color;
-				break;
+				tf->writes_color = tf->gdd_writes[i]->color;
 			}
+			if (tf->writes_color && tf->reads_color)
+				break;
 		}
+		if (!tf->reads_color)
+			tf->reads_color = tf->line_color;
+		if (!tf->writes_color)
+			tf->writes_color = tf->line_color;
 	}
 }
 
@@ -351,7 +360,7 @@ static void read_trace_events(void)
 		first_record(trace);
 		while (1) {
 			check_record(trace);
-			add_tput(trace, tf->tput_gld);
+			add_tput(trace, tf->tput_writes_gld, tf->tput_reads_gld);
 			add_iop(trace, tf->iop_gld);
 			add_io(trace, tf);
 			add_pending_io(trace, tf->queue_depth_gld);
@@ -663,6 +672,12 @@ static void plot_io(struct plot *plot, int min_seconds, int max_seconds, u64 min
 		pos = label + strlen(label);
 
 		for (i = 0; i < tf->io_plots; i++) {
+			if (tf->gdd_writes[i]) {
+				svg_io_graph(plot, tf->gdd_writes[i]);
+				if (io_per_process)
+					strcpy(pos, tf->gdd_writes[i]->label);
+				svg_add_legend(plot, label, " Writes", tf->gdd_writes[i]->color);
+			}
 			if (tf->gdd_reads[i]) {
 				svg_io_graph(plot, tf->gdd_reads[i]);
 				if (io_per_process)
@@ -670,12 +685,6 @@ static void plot_io(struct plot *plot, int min_seconds, int max_seconds, u64 min
 				svg_add_legend(plot, label, " Reads", tf->gdd_reads[i]->color);
 			}
 
-			if (tf->gdd_writes[i]) {
-				svg_io_graph(plot, tf->gdd_writes[i]);
-				if (io_per_process)
-					strcpy(pos, tf->gdd_writes[i]->label);
-				svg_add_legend(plot, label, " Writes", tf->gdd_writes[i]->color);
-			}
 		}
 	}
 	if (plot->add_xlabel)
@@ -694,14 +703,20 @@ static void plot_tput(struct plot *plot, int min_seconds, int max_seconds)
 	if (active_graphs[TPUT_GRAPH_INDEX] == 0)
 		return;
 
-	if (num_traces > 1)
-		svg_alloc_legend(plot, num_traces);
+	svg_alloc_legend(plot, num_traces * 2);
+
 	list_for_each_entry(tf, &all_traces, list) {
-		if (tf->tput_gld->max > max)
-			max = tf->tput_gld->max;
+		if (tf->tput_writes_gld->max > max)
+			max = tf->tput_writes_gld->max;
+		if (tf->tput_reads_gld->max > max)
+			max = tf->tput_reads_gld->max;
+	}
+	list_for_each_entry(tf, &all_traces, list) {
+		if (tf->tput_writes_gld->max > 0)
+			tf->tput_writes_gld->max = max;
+		if (tf->tput_reads_gld->max > 0)
+			tf->tput_reads_gld->max = max;
 	}
-	list_for_each_entry(tf, &all_traces, list)
-		tf->tput_gld->max = max;
 
 	setup_axis(plot);
 	set_plot_label(plot, "Throughput");
@@ -715,15 +730,20 @@ static void plot_tput(struct plot *plot, int min_seconds, int max_seconds)
 	set_xticks(plot, num_xticks, min_seconds, max_seconds);
 
 	list_for_each_entry(tf, &all_traces, list) {
-		svg_line_graph(plot, tf->tput_gld, tf->line_color, 0, 0);
-		if (num_traces > 1)
-			svg_add_legend(plot, tf->label, "", tf->line_color);
+		if (tf->tput_writes_gld->max > 0) {
+			svg_line_graph(plot, tf->tput_writes_gld, tf->writes_color, 0, 0);
+			svg_add_legend(plot, tf->label, "Writes", tf->writes_color);
+		}
+		if (tf->tput_reads_gld->max > 0) {
+			svg_line_graph(plot, tf->tput_reads_gld, tf->reads_color, 0, 0);
+			svg_add_legend(plot, tf->label, "Reads", tf->reads_color);
+		}
 	}
 
 	if (plot->add_xlabel)
 		set_xlabel(plot, "Time (seconds)");
-	if (num_traces > 1)
-		svg_write_legend(plot);
+
+	svg_write_legend(plot);
 	close_plot(plot);
 	total_graphs_written++;
 }
@@ -1300,7 +1320,11 @@ static int parse_options(int ac, char **av)
 			disable_one_graph(optarg);
 			break;
 		case 'd':
-			blktrace_device = strdup(optarg);
+			if (num_blktrace_devices == MAX_DEVICES_PER_TRACE - 1) {
+				fprintf(stderr, "Too many blktrace devices provided\n");
+				exit(1);
+			}
+			blktrace_devices[num_blktrace_devices++] = strdup(optarg);
 			break;
 		case 'D':
 			blktrace_dest_dir = strdup(optarg);
@@ -1386,11 +1410,11 @@ action_err:
 	return 0;
 }
 
-static void dest_mkdir()
+static void dest_mkdir(char *dir)
 {
 	int ret;
 
-	ret = mkdir(blktrace_dest_dir, 0777);
+	ret = mkdir(dir, 0777);
 	if (ret && errno != EEXIST) {
 		fprintf(stderr, "failed to mkdir error %s\n", strerror(errno));
 		exit(errno);
@@ -1440,14 +1464,23 @@ int main(int ac, char **av)
 		exit(1);
 	}
 
-	if (blktrace_device) {
+	if (num_blktrace_devices) {
 		char *path;
 
-		dest_mkdir();
+		dest_mkdir(blktrace_dest_dir);
+		if (num_blktrace_devices > 1) {
+			snprintf(line, line_len, "%s/%s", blktrace_dest_dir,
+				 blktrace_outfile);
+			dest_mkdir(line);
+		}
+
 		path = join_path(blktrace_dest_dir, blktrace_outfile);
 
-		ret = start_blktrace(blktrace_device, blktrace_outfile,
-				     blktrace_dest_dir);
+		snprintf(line, line_len, "%s.dump", path);
+		unlink(line);
+
+		ret = start_blktrace(blktrace_devices, num_blktrace_devices,
+				     blktrace_outfile, blktrace_dest_dir);
 		if (ret) {
 			fprintf(stderr, "exiting due to blktrace failure\n");
 			exit(1);
@@ -1462,7 +1495,6 @@ int main(int ac, char **av)
 				exit(1);
 			}
 			wait_for_tracers();
-			blktrace_to_dump(path);
 		} else {
 			/* no program specified, just wait for
 			 * blktrace to exit
diff --git a/iowatcher/mpstat.c b/iowatcher/mpstat.c
index f2292b9..022f589 100644
--- a/iowatcher/mpstat.c
+++ b/iowatcher/mpstat.c
@@ -38,8 +38,11 @@
 char line[1024];
 int line_len = 1024;
 
-char *record_header = "CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest   %idle\n";
-int record_header_len = 0;
+static char record_header[] = "CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest   %idle\n";
+static char record_header_v2[] = "CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle\n";
+
+int record_header_len = sizeof(record_header);
+int record_header_v2_len = sizeof(record_header_v2);
 
 static int past_eof(struct trace *trace, char *cur)
 {
@@ -67,11 +70,17 @@ char *next_mpstat(struct trace *trace)
 {
 	char *cur = trace->mpstat_cur;
 
-	cur = strstr(cur, record_header);
+	cur = strstr(trace->mpstat_cur, record_header);
+	if (cur) {
+		cur += record_header_len;
+	} else {
+		cur = strstr(trace->mpstat_cur, record_header_v2);
+		if (cur)
+			cur += record_header_v2_len;
+	}
 	if (!cur)
 		return NULL;
 
-	cur += record_header_len;
 	if (past_eof(trace, cur))
 		return NULL;
 	trace->mpstat_cur = cur;
@@ -111,8 +120,7 @@ static int count_mpstat_cpus(struct trace *trace)
 	char *cur = trace->mpstat_start;
 	char *cpu;
 	char *record;
-	int len;
-	char *line;
+	int len; char *line;
 
 	first_mpstat(trace);
 
@@ -134,6 +142,7 @@ static int count_mpstat_cpus(struct trace *trace)
 	trace->mpstat_num_cpus = atoi(cur);
 	first_mpstat(trace);
 	free(line);
+
 	return trace->mpstat_num_cpus;
 }
 
@@ -176,7 +185,6 @@ int read_mpstat(struct trace *trace, char *trace_name)
 	}
 
 	snprintf(line, line_len, "%s.mpstat", guess_filename(trace_name));
-
 	fd = open(line, O_RDONLY);
 	if (fd < 0)
 		return 0;
@@ -191,7 +199,6 @@ int read_mpstat(struct trace *trace, char *trace_name)
 		fprintf(stderr, "Unable to mmap trace file %s, err %s\n", line, strerror(errno));
 		goto fail_fd;
 	}
-
 	trace->mpstat_start = p;
 	trace->mpstat_len = st.st_size;
 	trace->mpstat_cur = p;
@@ -211,6 +218,11 @@ fail_fd:
 /*
  * 09:56:26 AM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest   %idle
  *
+ * or
+ *
+ * 10:18:51 AM  CPU    %usr   %nice    %sys %iowait    %irq   %soft  %steal  %guest  %gnice   %idle
+ *
+ *
  * this reads just one line in the mpstat
  */
 int read_mpstat_event(struct trace *trace, double *user,
diff --git a/iowatcher/plot.c b/iowatcher/plot.c
index 7b4e18c..e5af3ad 100644
--- a/iowatcher/plot.c
+++ b/iowatcher/plot.c
@@ -71,8 +71,10 @@ static int final_width = 0;
 
 static char *colors[] = {
 	"blue", "darkgreen",
-	"red", "aqua",
-	"orange", "darkviolet",
+	"red",
+	"darkviolet",
+	"orange",
+	"aqua",
 	"brown", "#00FF00",
 	"yellow", "coral",
 	"black", "darkred",
diff --git a/iowatcher/tracers.c b/iowatcher/tracers.c
index a995e11..433b530 100644
--- a/iowatcher/tracers.c
+++ b/iowatcher/tracers.c
@@ -42,16 +42,14 @@ static char line[1024];
 static pid_t blktrace_pid = 0;
 static pid_t mpstat_pid = 0;
 
-char *blktrace_args[] = {
+char *blktrace_args[17 + MAX_DEVICES_PER_TRACE * 2] = {
 	"blktrace",
-	"-d", NULL,
-	"-b", "16384",
-	"-o", "trace",
-	"-D", ".",
+	"-b", "8192",
 	"-a", "queue",
 	"-a", "complete",
 	"-a", "issue",
 	"-a", "notify",
+	"-D", ".",
 	NULL,
 };
 
@@ -61,9 +59,8 @@ char *mpstat_args[] = {
 	NULL,
 };
 
-#define DEVICE_INDEX 2
-#define DEST_DIR_INDEX 8
-#define TRACE_NAME_INDEX 6
+#define DEST_DIR_INDEX 12
+#define LAST_ARG 13
 
 int stop_tracer(pid_t *tracer_pid)
 {
@@ -103,18 +100,42 @@ void sig_handler_for_quit(int val)
 }
 
 
-int start_blktrace(char *device, char *trace_name, char *dest)
+int start_blktrace(char **devices, int num_devices, char *trace_name, char *dest)
 {
 	pid_t pid;
 	int ret;
 	char **arg = blktrace_args;
-	blktrace_args[DEVICE_INDEX] = device;
+	int i;
+	int arg_index;
 
 	fprintf(stderr, "running blktrace");
-	if (dest)
-		blktrace_args[DEST_DIR_INDEX] = dest;
-	if (trace_name)
-		blktrace_args[TRACE_NAME_INDEX] = trace_name;
+	if (!trace_name)
+		trace_name = "trace";
+
+	arg_index = LAST_ARG;
+	for (i = 0; i < num_devices; i++) {
+		blktrace_args[arg_index++] = "-d";
+		blktrace_args[arg_index++] = devices[i];
+	}
+
+	/*
+	 * single device traces use -o and are put into
+	 * the dest dir if provided
+	 */
+	if (num_devices == 1) {
+		blktrace_args[arg_index++] = "-o";
+		blktrace_args[arg_index++] = trace_name;
+		if (dest)
+			blktrace_args[DEST_DIR_INDEX] = dest;
+	} else {
+		/*
+		 * multi device traces are put into a dest
+		 * dir based on the trace name
+		 */
+		blktrace_args[DEST_DIR_INDEX] = trace_name;
+	}
+
+	blktrace_args[arg_index] = NULL;
 
 	while(*arg) {
 		fprintf(stderr, " %s", *arg);
diff --git a/iowatcher/tracers.h b/iowatcher/tracers.h
index 7ceeefe..81a2fe7 100644
--- a/iowatcher/tracers.h
+++ b/iowatcher/tracers.h
@@ -19,7 +19,7 @@
 #define __IOWATCH_TRACERS
 int run_program(char *str);
 int stop_blktrace(void);
-int start_blktrace(char *device, char *trace_name, char *dest);
+int start_blktrace(char **devices, int num_devices, char *trace_name, char *dest);
 int start_mpstat(char *trace_name);
 int wait_for_tracers(void);
 int blktrace_to_dump(char *trace_name);