diff options
author | Chris Mason <clm@fb.com> | 2017-05-18 13:26:47 -0700 |
---|---|---|
committer | Chris Mason <clm@fb.com> | 2017-05-31 09:55:04 -0700 |
commit | e70dbfbb31bd7759bfd34d141c80653f51c10273 (patch) | |
tree | 59623446e72df65d09b831ffb30296d0688994e8 | |
parent | 1fe42e423765b2048bcf55c094e5dfa4a758afb0 (diff) | |
download | simoop-e70dbfbb31bd7759bfd34d141c80653f51c10273.tar.gz |
Add support for more vmstat counters
Signed-off-by: Chris Mason <clm@fb.com>
-rw-r--r-- | simoop.c | 398 |
1 files changed, 294 insertions, 104 deletions
@@ -23,6 +23,8 @@ #include <sys/mman.h> #include <libgen.h> #include <locale.h> +#include <ctype.h> +#include <limits.h> /* these are part of the histogram accounting */ #define PLAT_BITS 8 @@ -71,6 +73,10 @@ static int du_threads = 1; static int thinking_mem = 128 * 1024 * 1024; /* should we do a truncate and fsync after every write */ static int funksync = 0; +/* are we just appending bytes onto the ends of the working set files */ +static int append_mode = 0; +/* randomize the write size */ +static int oddsizes = 0; /* -M how much memory we allocate to benchmark allocations */ static int mmap_size = 64 * 1024 * 1024; @@ -132,8 +138,130 @@ char *stat_labels[] = { NULL, }; -char *option_string = "t:s:C:c:r:n:f:FR:T:m:W:M:w:i:D:"; + +/* match this with vmstat_labels */ +enum { + ALLOCSTALLS, + VMSCAN_NR_WRITE, + TOTAL_VMSTATS, +}; + +char *vmstat_labels[] = { + "allocstall", + "nr_vmscan_write", + NULL, +}; + +struct vmstat_info { + double instant_rate[TOTAL_VMSTATS]; + double last_rate[TOTAL_VMSTATS]; + double rate[TOTAL_VMSTATS]; + struct stats stats[TOTAL_VMSTATS]; +}; + +static void save_vmstat_rates(struct vmstat_info *vmstat_info) +{ + int i; + for (i = 0; i < TOTAL_VMSTATS; i++) { + vmstat_info->last_rate[i] = vmstat_info->rate[i]; + } +} + +static void save_instant_vmstat_rates(struct vmstat_info *vmstat_info) +{ + int i; + for (i = 0; i < TOTAL_VMSTATS; i++) { + vmstat_info->instant_rate[i] = vmstat_info->rate[i]; + } +} + +/* + * A not-so-good version fls64. No fascinating optimization since + * no one except parse_size use it + */ +static int fls64(unsigned long long x) +{ + int i; + + for (i = 0; i <64; i++) + if (x << i & (1ULL << 63)) + return 64 - i; + return 64 - i; +} + +unsigned long long parse_size(char *s) +{ + char c; + char *endptr; + unsigned long long mult = 1; + unsigned long long ret; + + if (!s) { + fprintf(stderr, "size value is empty\n"); + exit(1); + } + if (s[0] == '-') { + fprintf(stderr, "size value '%s' is less equal than 0\n", s); + exit(1); + } + ret = strtoull(s, &endptr, 10); + if (endptr == s) { + fprintf(stderr, "size value '%s' is invalid\n", s); + exit(1); + } + if (endptr[0] && endptr[1]) { + fprintf(stderr, "illegal suffix contains character '%c' in wrong position\n", + endptr[1]); + exit(1); + } + /* + * strtoll returns LLONG_MAX when overflow, if this happens, + * need to call strtoull to get the real size + */ + if (errno == ERANGE && ret == ULLONG_MAX) { + fprintf(stderr, "size value '%s' is too large for unsigned long long", s); + exit(1); + } + if (endptr[0]) { + c = tolower(endptr[0]); + switch (c) { + case 'e': + mult *= 1024; + /* fallthrough */ + case 'p': + mult *= 1024; + /* fallthrough */ + case 't': + mult *= 1024; + /* fallthrough */ + case 'g': + mult *= 1024; + /* fallthrough */ + case 'm': + mult *= 1024; + /* fallthrough */ + case 'k': + mult *= 1024; + /* fallthrough */ + case 'b': + break; + default: + fprintf(stderr, "unknown size descriptor '%c'", c); + exit(1); + } + } + /* Check whether ret * mult overflow */ + if (fls64(ret) + fls64(mult) - 1 > 64) { + fprintf(stderr, "size value '%s' is too large for unsigned long long\n", s); + exit(1); + } + ret *= mult; + return ret; +} + +char *option_string = "t:s:C:c:r:n:f:FR:T:m:W:M:w:i:D:oa"; static struct option long_options[] = { + {"appendmode", required_argument, 0, 'a'}, {"mmapsize", required_argument, 0, 'M'}, {"filesize", required_argument, 0, 'f'}, {"numfiles", required_argument, 0, 'n'}, @@ -150,6 +278,7 @@ static struct option long_options[] = { {"cputhreads", required_argument, 0, 'C'}, {"memory", required_argument, 0, 'm'}, {"funksync", no_argument, 0, 'F'}, + {"oddsizes", no_argument, 0, 'o'}, {"help", no_argument, 0, HELP_LONG_OPT}, {0, 0, 0, 0} }; @@ -157,9 +286,10 @@ static struct option long_options[] = { static void print_usage(void) { fprintf(stderr, "simoop usage:\n" + "\t-a (--appendmode): append onto working files\n" "\t-t (--threads): worker threads (def: 16)\n" - "\t-m (--memory): memory in MB to allocate during think time in each worker (def 128)\n" - "\t-M (--mmapsize): amount in MB to mmap to time allocator (64MB)\n" + "\t-m (--memory): memory to allocate during think time in each worker (def 128m)\n" + "\t-M (--mmapsize): amount to mmap to time allocator (64M)\n" "\t-r (--runtime): How long to run before exiting (seconds, def: 30)\n" "\t-w (--warmuptime): How long to warmup before resetting the stats (seconds, def: 60)\n" "\t-i (--interval): Sleep time in seconds between latency reports (sec, def: 120\n" @@ -167,13 +297,15 @@ static void print_usage(void) "\t-c (--cputime): How long to think during each worker loop (seconds, def: 3)\n" "\t-C (--cputhreads): How many threads do the cpu time loop (24)\n" "\t-n (--numfiles): Number of files per directory tree (65536)\n" - "\t-f (--filesize): Size of each file in MB (64MB)\n" - "\t-R (--readsize): amount in MB to read from each file (2MB)\n" - "\t-W (--writesize): amount in MB to write to tmp files (2MB)\n" + "\t-f (--filesize): Size of each file (64M)\n" + "\t-R (--readsize): amount to read from each file (2M)\n" + "\t-W (--writesize): amount to write to tmp files (2M)\n" "\t-T (--rwthreads): how many threads to read/write (8)\n" "\t-D (--duthraeds): how many threads to scanning the working dirs (1)\n" "\t-F (--funksync): should we fsync;truncate(0);fsync after writes\n" + "\t-o (--oddsizes): randomize sizes to unaligned values\n" "\t dir1 [dir2 ... dirN]\n" + "\nall sizes are in bytes k,m,g,t modifiers can be used\n" ); exit(1); } @@ -195,16 +327,17 @@ static void parse_options(int ac, char **av) break; switch(c) { + case 'a': + append_mode = 1; + break; case 's': found_sleeptime = atoi(optarg); break; case 'm': - thinking_mem = atoi(optarg); - thinking_mem *= 1024 *1024; + thinking_mem = parse_size(optarg); break; case 'M': - mmap_size = atoi(optarg); - mmap_size *= 1024 *1024; + mmap_size = parse_size(optarg); break; case 'c': found_cputime = atoi(optarg); @@ -228,19 +361,16 @@ static void parse_options(int ac, char **av) funksync = 0; break; case 'f': - file_size = atoi(optarg); - file_size *= 1024 * 1024; + file_size = parse_size(optarg); break; case 'n': num_files = atoi(optarg); break; case 'R': - read_size = atoi(optarg); - read_size *= 1024 * 1024; + read_size = parse_size(optarg); break; case 'W': - write_size = atoi(optarg); - write_size *= 1024 * 1024; + write_size = parse_size(optarg); break; case 'T': rw_threads = atoi(optarg); @@ -248,6 +378,9 @@ static void parse_options(int ac, char **av) case 'D': du_threads = atoi(optarg); break; + case 'o': + oddsizes = 1; + break; case '?': case HELP_LONG_OPT: print_usage(); @@ -281,6 +414,13 @@ static void parse_options(int ac, char **av) if (found_cputime >= 0) cputime = found_cputime * 1000000; + if (cputime == 0 || cpu_threads == 0) { + cputime = 0; + cpu_threads = 0; + if (!found_sleeptime) + sleeptime = 0; + } + if (optind < ac) { fprintf(stderr, "Error Extra arguments '%s'\n", av[optind]); exit(1); @@ -610,6 +750,14 @@ static int open_path(char *path, int seq, char *postfix, int flags) return fd; } +static int randomize_size(int sz) +{ + if (!oddsizes) + return sz; + + return rand() % sz; +} + /* helper for startup, do initial writes to a given fd */ static void fill_one_file(int fd) { @@ -617,6 +765,7 @@ static void fill_one_file(int fd) int ret; unsigned long long cur_size; char *buf; + unsigned long long this_size = randomize_size(file_size); ret = fstat(fd, &st); if (ret < 0) { @@ -625,8 +774,16 @@ static void fill_one_file(int fd) } cur_size = st.st_size; - if (cur_size >= file_size) + if (append_mode && oddsizes && this_size > 4096 && rand() % 2) { + this_size = this_size % 4096; + } + + if (cur_size >= this_size) { + if (append_mode) { + ftruncate(fd, this_size); + } return; + } buf = malloc(BUF_SIZE); if (!buf) { @@ -634,21 +791,24 @@ static void fill_one_file(int fd) exit(1); } - memset(buf, 'a', BUF_SIZE); - while (cur_size < file_size) { - ret = write(fd, buf, BUF_SIZE); + while (cur_size < this_size) { + int this_write = this_size - cur_size; + + if (this_write > BUF_SIZE) + this_write = BUF_SIZE; + + ret = write(fd, buf, this_write); if (ret < 0) { perror("write"); exit(1); } - if (ret < BUF_SIZE) { + if (ret < this_write) { fprintf(stderr, "short write\n"); exit(1); } cur_size += ret; } - free(buf); } @@ -761,13 +921,23 @@ static void write_to_file(char *path, int seq, char *buf) int fd; int ret; int i; - unsigned long write_bytes = write_size; + int write_bytes = randomize_size(write_size); unsigned long long offset = 0; - fd = open_path(path, seq, RESULT_FILE, 0); + if (append_mode) + fd = open_path(path, seq, DATA_FILE, O_APPEND); + else + fd = open_path(path, seq, RESULT_FILE, 0); + + if (oddsizes && write_bytes > 4096) + write_bytes = write_bytes % 4096; while (write_bytes > 0) { - ret = write(fd, buf, BUF_SIZE); + int this_write = write_bytes; + if (this_write > BUF_SIZE) + this_write = BUF_SIZE; + + ret = write(fd, buf, this_write); if (ret == 0) break; if (ret < 0) { @@ -786,8 +956,13 @@ static void write_to_file(char *path, int seq, char *buf) close(fd); /* make some dirty inodes */ - for (i = 0; i < 8; i++) - dirty_an_inode(path); + if (!append_mode) { + for (i = 0; i < 8; i++) + dirty_an_inode(path); + } else if (rand() % 10 == 0) { + /* delete some files */ + ftruncate(fd, 0); + } } /* make all the worker files under a main path */ @@ -797,7 +972,7 @@ static void make_files(char *path) int fd; for (seq = 0; seq < num_files; seq++) { - fd = open_path(path, seq, DATA_FILE, 0); + fd = open_path(path, seq, DATA_FILE, O_APPEND); fill_one_file(fd); close(fd); @@ -919,9 +1094,10 @@ void *worker_thread(void *arg) read_tids = malloc(sizeof(*read_tids) * rw_threads); write_tids = malloc(sizeof(*write_tids) * rw_threads); - mem = malloc(thinking_mem); + if (thinking_mem) + mem = malloc(thinking_mem); - if (!read_tids || !write_tids || !mem) { + if (!read_tids || !write_tids || (thinking_mem && !mem)) { perror("allocation failed\n"); exit(1); } @@ -944,28 +1120,31 @@ void *worker_thread(void *arg) } /* if someone swapped out our thinking mem, bring it back */ - memset(mem, 0, thinking_mem); + if (thinking_mem) + memset(mem, 0, thinking_mem); gettimeofday(&start, NULL); /* Start the threads to read files */ - read_some_files(read_buf, read_tids); + if (read_size) { + read_some_files(read_buf, read_tids); - /* think in parallel */ - usec_spin(cputime); + /* think in parallel */ + usec_spin(cputime); - /* wait for our reads to finish */ - for (i = 0; i < rw_threads; i++) { - pthread_join(read_tids[i], NULL); - } - gettimeofday(&now, NULL); + /* wait for our reads to finish */ + for (i = 0; i < rw_threads; i++) { + pthread_join(read_tids[i], NULL); + } + gettimeofday(&now, NULL); - /* - * record how long the reading stage took. This - * includes all of the latencies for thread creation, - * doing the reads and waiting for completeion - */ - record_one_lat(&td->stats[READ_STATS], &start, &now); + /* + * record how long the reading stage took. This + * includes all of the latencies for thread creation, + * doing the reads and waiting for completeion + */ + record_one_lat(&td->stats[READ_STATS], &start, &now); + } /* write out the (pretend) results */ if (write_size) { @@ -983,35 +1162,38 @@ void *worker_thread(void *arg) * a chunk of pages. This is basicaly the user-visible * impact of allocation stalls */ - gettimeofday(&start, NULL); + if (mmap_size) { + gettimeofday(&start, NULL); - mmap_ptr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, - MAP_ANONYMOUS | MAP_PRIVATE, - -1, 0); + mmap_ptr = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE, + MAP_ANONYMOUS | MAP_PRIVATE, + -1, 0); - if (mmap_ptr == MAP_FAILED) { - perror("mmap"); - exit(1); - } + if (mmap_ptr == MAP_FAILED) { + perror("mmap"); + exit(1); + } - /* fault in all those pages */ - for (i = 0; i < mmap_size; i += 4096) { - mmap_ptr[i] = 'a'; - } + /* fault in all those pages */ + for (i = 0; i < mmap_size; i += 4096) { + mmap_ptr[i] = 'a'; + } - /* measure how long all of this took */ - gettimeofday(&now, NULL); - record_one_lat(&td->stats[ALLOC_STATS], &start, &now); + /* measure how long all of this took */ + gettimeofday(&now, NULL); + record_one_lat(&td->stats[ALLOC_STATS], &start, &now); - /* think again, pretending we've done something useful */ - munmap(mmap_ptr, mmap_size); + /* think again, pretending we've done something useful */ + munmap(mmap_ptr, mmap_size); + } usec_spin(cputime); + free(read_buf); free(write_buf); - td->work_done++; - usleep(sleeptime); + if (sleeptime) + usleep(sleeptime); } free(mem); @@ -1039,29 +1221,32 @@ static void *cpu_thread(void *arg) * read in /proc/vmstat so we can sum the allocation stall lines and * print them out */ -static int read_allocstalls(void) +static void read_vmstat(struct vmstat_info *vmstat_info) { - int stalls = 0; int val; FILE * fp; char * line = NULL; size_t len = 0; ssize_t read; + int i; fp = fopen("/proc/vmstat", "r"); if (fp == NULL) - return 0; + return; + memset(vmstat_info->rate, 0, sizeof(double) * TOTAL_VMSTATS); while ((read = getline(&line, &len, fp)) != -1) { /* * newer kernels break out different types of allocstall, * just add them all together */ - if (strstr(line, "allocstall")) { - char *p = strchr(line, ' '); - if (p && p[1] != '\0') { - val = atoi(p + 1); - stalls += val; + for (i = 0; i < TOTAL_VMSTATS; i++) { + if (strstr(line, vmstat_labels[i])) { + char *p = strchr(line, ' '); + if (p && p[1] != '\0') { + val = atoi(p + 1); + vmstat_info->rate[i] += val; + } } } } @@ -1069,7 +1254,6 @@ static int read_allocstalls(void) if (line) free(line); fclose(fp); - return stalls; } /* @@ -1101,10 +1285,8 @@ static void collect_stats(struct stats *dest, struct thread_data *worker_threads static void print_latencies(struct thread_data *worker_threads_mem, struct stats *stats, struct stats *work_done_stats, - struct stats *allocstall_stats, + struct vmstat_info *vmstat_info, double work_done, double instant_work_done, - double allocstalls, - double instant_allocstalls, unsigned long long delta, unsigned long long instant_delta) { @@ -1112,6 +1294,7 @@ static void print_latencies(struct thread_data *worker_threads_mem, double instant_rate; double seconds = (double)delta / 1000000; unsigned int p50, p95, p99; + int i; printf("___\n"); printf("Run time: %.0f seconds\n", seconds); @@ -1128,13 +1311,24 @@ static void print_latencies(struct thread_data *worker_threads_mem, printf("work rate = %.2f/sec (avg %.2f/sec) (p50: %.2f) (p95: %.2f) (p99: %.2f)\n", instant_rate, rate, (double)p50/100.00, (double)p95/100.00, (double)p99/100.00); - /* do the same for the allocation stall rate */ - rate = (allocstalls * 1000000) / delta; - instant_rate = (instant_allocstalls * 1000000) / instant_delta; - add_lat(allocstall_stats, rate * 100); - calc_p99(allocstall_stats, &p50, &p95, &p99); - printf("alloc stall rate = %.2f/sec (avg: %.2f) (p50: %.2f) (p95: %.2f) (p99: %.2f)\n", - instant_rate, rate, (double)p50/100.00, (double)p95/100.00, (double)p99/100.00); + for (i = 0; i < TOTAL_VMSTATS; i++) { + rate = vmstat_info->rate[i] - vmstat_info->last_rate[i]; + if (rate < 0) + rate = 0; + + instant_rate = vmstat_info->rate[i] - vmstat_info->instant_rate[i]; + if (instant_rate < 0) + instant_rate = 0; + + rate = (rate * 1000000) / delta; + instant_rate = (instant_rate * 1000000) / delta; + + add_lat(&vmstat_info->stats[i], rate * 100); + calc_p99(&vmstat_info->stats[i], &p50, &p95, &p99); + printf("%s rate = %.2f/sec (avg: %.2f) (p50: %.2f) (p95: %.2f) (p99: %.2f)\n", + vmstat_labels[i], instant_rate, rate, + (double)p50/100.00, (double)p95/100.00, (double)p99/100.00); + } } @@ -1153,22 +1347,20 @@ static void sleep_for_runtime(struct thread_data *worker_threads_mem) double work_done = 0; double instant_work_done = 0; double last_work_done = 0; - double allocstalls = 0; - double instant_allocstalls = 0; - double last_allocstalls = 0; struct stats stats[TOTAL_STATS]; + struct vmstat_info vmstat_info; struct stats work_done_stats; - struct stats allocstall_stats; int i; gettimeofday(&start, NULL); rate_start = start; memset(&work_done_stats, 0, sizeof(work_done_stats)); - memset(&allocstall_stats, 0, sizeof(allocstall_stats)); + memset(&vmstat_info, 0, sizeof(vmstat_info)); - last_allocstalls = read_allocstalls(); - allocstalls = last_allocstalls; + read_vmstat(&vmstat_info); + save_vmstat_rates(&vmstat_info); + save_instant_vmstat_rates(&vmstat_info); while(1) { gettimeofday(&now, NULL); instant_start = now; @@ -1179,13 +1371,14 @@ static void sleep_for_runtime(struct thread_data *worker_threads_mem) __sync_synchronize(); warmup_done = 1; memset(&work_done_stats, 0, sizeof(work_done_stats)); - memset(&allocstall_stats, 0, sizeof(allocstall_stats)); - last_allocstalls = read_allocstalls(); + memset(&vmstat_info, 0, sizeof(vmstat_info)); + read_vmstat(&vmstat_info); + save_vmstat_rates(&vmstat_info); + save_instant_vmstat_rates(&vmstat_info); last_work_done = work_done; rate_start = now; } - instant_allocstalls = allocstalls; instant_work_done = work_done; if (delta < runtime_usec) sleep(interval_seconds); @@ -1200,19 +1393,15 @@ static void sleep_for_runtime(struct thread_data *worker_threads_mem) for (i = 0; i < worker_threads; i++) work_done += worker_threads_mem[i].work_done; - allocstalls = read_allocstalls(); - if (allocstalls < last_allocstalls) - allocstalls = last_allocstalls; - + read_vmstat(&vmstat_info); print_latencies(worker_threads_mem, stats, &work_done_stats, - &allocstall_stats, + &vmstat_info, work_done - last_work_done, work_done - instant_work_done, - allocstalls - last_allocstalls, - allocstalls - instant_allocstalls, rate_delta, instant_delta); + save_instant_vmstat_rates(&vmstat_info); } __sync_synchronize(); stopping = 1; @@ -1228,13 +1417,12 @@ static void sleep_for_runtime(struct thread_data *worker_threads_mem) gettimeofday(&now, NULL); rate_delta = tvdelta(&rate_start, &now); instant_delta = tvdelta(&instant_start, &now); + read_vmstat(&vmstat_info); print_latencies(worker_threads_mem, stats, &work_done_stats, - &allocstall_stats, + &vmstat_info, work_done - last_work_done, work_done - instant_work_done, - allocstalls - last_allocstalls, - allocstalls - instant_allocstalls, rate_delta, instant_delta); } @@ -1252,7 +1440,9 @@ int main(int ac, char **av) if (du_threads > total_paths) du_threads = total_paths; - du_tids = calloc(du_threads, sizeof(pthread_t)); + + /* du threads might be zero */ + du_tids = calloc(du_threads + 1, sizeof(pthread_t)); worker_threads_mem = calloc(worker_threads + cpu_threads, sizeof(struct thread_data)); |