kbench_mod: Add kernel module route lookup tester.

Flow keys can be specified on the kernel command line.

A warmup of "warmup_count" lookups are performed, then
a single cycle counted lookup is performed with the
cycle count reported in the kernel logs.

X86, Powerpc, and Sparc64 are currently supported.

Signed-off-by: David S. Miller <davem@davemloft.net>

2 files changed, 505 insertions, 2 deletions

diff --git a/Makefile b/Makefile
index 38b071c..f69ffb3 100644
--- a/Makefile
+++ b/Makefile
@@ -1,12 +1,21 @@
+ifneq ($(KERNELRELEASE),)
+obj-m := kbench_mod.o
+else
 CC := gcc
 CFLAGS := -Wall -O2 -I/usr/local -L/usr/local
 LDFLAGS := -lmnl
 
-all: route_bench udpflood
+KDIR ?= /lib/modules/`uname -r`/build
+
+all: route_bench udpflood kbench_mod
 
 route_bench: route_bench.c
 
+kbench_mod: kbench_mod.c
+	$(MAKE) -C $(KDIR) M=$$PWD
+
 udpflood: udpflood.c
 
 clean:
-	rm route_bench udpflood
+	rm route_bench udpflood kbench_mod
+endif
diff --git a/kbench_mod.c b/kbench_mod.c
new file mode 100644
index 0000000..35dc0dc
--- /dev/null
+++ b/kbench_mod.c
@@ -0,0 +1,494 @@
+#define pr_fmt(fmt) "kbench: " fmt
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/inet.h>
+
+#include <net/route.h>
+#include <net/ip_fib.h>
+
+#include <linux/timex.h>
+
+/* We can't just use "get_cycles()" as on some platforms, such
+ * as sparc64, that gives system cycles rather than cpu clock
+ * cycles.
+ */
+
+#ifdef CONFIG_SPARC64
+static inline unsigned long long get_tick(void)
+{
+	unsigned long long t;
+
+	__asm__ __volatile__("rd %%tick, %0" : "=r" (t));
+	return t;
+}
+#elif defined(CONFIG_X86)
+static inline unsigned long long get_tick(void)
+{
+	unsigned long long t;
+
+	rdtscll(t);
+
+	return t;
+}
+#elif defined(CONFIG_POWERPC)
+static inline unsigned long long get_tick(void)
+{
+	return get_cycles();
+}
+#else
+#error Unsupported architecture, please implement get_tick()
+#endif
+
+#define IP_ROUTE_RETURNS_PTR
+#undef IP_ROUTE_HAVE_PREALLOC
+#undef IP_ROUTE_CYCLE_SAMPLES
+#undef IP_FIB_LOOKUP_EXPORTED
+
+#ifdef IP_ROUTE_CYCLE_SAMPLES
+extern int ip_route_output_cycles_1;
+extern int ip_route_output_cycles_2;
+extern int ip_route_output_cycles_3;
+extern int ip_route_output_cycles_4;
+extern int ip_route_output_cycles_5;
+extern int ip_route_output_cycles_6;
+extern int ip_route_output_cycles_7;
+#endif
+
+#define DEFAULT_WARMUP_COUNT 100000
+
+#define DEFAULT_DST_IP_ADDR	0x4a800001
+#define DEFAULT_SRC_IP_ADDR	0x00000000
+#define DEFAULT_OIF		0
+#define DEFAULT_IIF		0
+#define DEFAULT_MARK		0x00000000
+#define DEFAULT_TOS		0x00
+
+static int flow_oif = DEFAULT_OIF;
+static int flow_iif = DEFAULT_IIF;
+static u32 flow_mark = DEFAULT_MARK;
+static u32 flow_dst_ip_addr = DEFAULT_DST_IP_ADDR;
+static u32 flow_src_ip_addr = DEFAULT_SRC_IP_ADDR;
+static int flow_tos = DEFAULT_TOS;
+
+static char dst_string[64];
+static char src_string[64];
+
+module_param_string(dst, dst_string, sizeof(dst_string), 0);
+module_param_string(src, src_string, sizeof(src_string), 0);
+
+static void __init flow_setup(void)
+{
+	if (dst_string[0])
+		flow_dst_ip_addr = in_aton(dst_string);
+	if (src_string[0])
+		flow_src_ip_addr = in_aton(src_string);
+}
+
+module_param_named(oif, flow_oif, int, 0);
+module_param_named(iif, flow_iif, int, 0);
+module_param_named(mark, flow_mark, uint, 0);
+module_param_named(tos, flow_tos, int, 0);
+
+static int warmup_count = DEFAULT_WARMUP_COUNT;
+module_param_named(count, warmup_count, int, 0);
+
+static void flow_init(struct flowi *fl)
+{
+	memset(fl, 0, sizeof(*fl));
+	fl->oif = flow_oif;
+	fl->iif = flow_iif;
+	fl->mark = flow_mark;
+	fl->fl4_dst = flow_dst_ip_addr;
+	fl->fl4_src = flow_src_ip_addr;
+	fl->fl4_tos = flow_tos;
+}
+
+#ifdef IP_ROUTE_RETURNS_PTR
+static struct rtable *route_output(struct net *net, struct flowi *fl)
+{
+	return ip_route_output_key(net, fl);
+}
+#else
+static struct rtable *route_output(struct net *net, struct flowi *fl)
+{
+	struct rtable *rt;
+	int err;
+
+	err = ip_route_output_key(net, &rt, fl);
+	if (err)
+		return ERR_PTR(err);
+	return rt;
+}
+#endif
+
+static void do_full_output_lookup_bench(void)
+{
+	unsigned long long t1, t2, tdiff;
+	struct rtable *rt;
+	struct flowi fl;
+	int i;
+
+	rt = NULL;
+
+	for (i = 0; i < warmup_count; i++) {
+		flow_init(&fl);
+
+		rt = route_output(&init_net, &fl);
+		if (IS_ERR(rt))
+			break;
+		ip_rt_put(rt);
+	}
+	if (IS_ERR(rt)) {
+		pr_info("ip_route_output_key: err=%ld\n", PTR_ERR(rt));
+		return;
+	}
+
+#ifdef IP_ROUTE_CYCLE_SAMPLES
+	ip_route_output_cycles_1 = 0;
+	ip_route_output_cycles_2 = 0;
+	ip_route_output_cycles_3 = 0;
+	ip_route_output_cycles_4 = 0;
+	ip_route_output_cycles_5 = 0;
+	ip_route_output_cycles_6 = 0;
+	ip_route_output_cycles_7 = 0;
+#endif
+
+	flow_init(&fl);
+
+	t1 = get_tick();
+	rt = route_output(&init_net, &fl);
+	t2 = get_tick();
+	if (!IS_ERR(rt))
+		ip_rt_put(rt);
+
+	tdiff = t2 - t1;
+	pr_info("ip_route_output_key tdiff: %llu\n", tdiff);
+#ifdef IP_ROUTE_CYCLE_SAMPLES
+	pr_info("ip_route_output_key cycls: [%d %d %d %d %d %d]\n",
+		ip_route_output_cycles_1,
+		ip_route_output_cycles_2,
+		ip_route_output_cycles_3,
+		ip_route_output_cycles_4,
+		ip_route_output_cycles_5,
+		ip_route_output_cycles_6);
+#endif
+}
+
+static void do_full_input_lookup_bench(void)
+{
+	unsigned long long t1, t2, tdiff;
+	struct net_device *dev;
+	struct sk_buff *skb;
+	int err, i;
+
+	skb = alloc_skb(4096, GFP_KERNEL);
+	if (!skb) {
+		pr_info("Cannot alloc SKB for test\n");
+		return;
+	}
+	skb_reset_mac_header(skb);
+	skb_reset_network_header(skb);
+	ip_hdr(skb)->protocol = IPPROTO_ICMP;
+	skb_reserve(skb, MAX_HEADER + sizeof(struct iphdr));
+
+	dev = __dev_get_by_index(&init_net, flow_iif);
+	if (dev == NULL) {
+		pr_info("Input device does not exist\n");
+		goto out_free;
+	}
+	skb->protocol = htons(ETH_P_IP);
+	skb->dev = dev;
+	skb->mark = flow_mark;
+	local_bh_disable();
+	err = 0;
+	for (i = 0; i < warmup_count; i++) {
+		err = ip_route_input(skb, flow_dst_ip_addr, flow_src_ip_addr, flow_tos, dev);
+		if (err)
+			break;
+		skb_dst_drop(skb);
+	}
+	local_bh_enable();
+
+	if (err) {
+		pr_info("Input route lookup fails with err=%d\n", err);
+		goto out_free;
+	}
+
+	local_bh_disable();
+	t1 = get_tick();
+	err = ip_route_input(skb, flow_dst_ip_addr, flow_src_ip_addr, flow_tos, dev);
+	t2 = get_tick();
+	local_bh_enable();
+
+	if (err) {
+		pr_info("Input route lookup fails with err=%d\n", err);
+		goto out_free;
+	}
+
+	skb_dst_drop(skb);
+
+	tdiff = t2 - t1;
+	pr_info("ip_route_input tdiff: %llu\n", tdiff);
+
+out_free:
+	kfree_skb(skb);
+}
+
+static void do_full_lookup_bench(void)
+{
+	if (!flow_iif)
+		do_full_output_lookup_bench();
+	else
+		do_full_input_lookup_bench();
+}
+
+#ifdef IP_ROUTE_HAVE_PREALLOC
+static void do_full_lookup_prealloc_bench(void)
+{
+	unsigned long long t1, t2, tdiff;
+	struct rtable *rt, rt_stack;
+	struct flowi fl;
+	int err, i;
+
+	err = 0;
+
+	for (i = 0; i < warmup_count; i++) {
+		flow_init(&fl);
+
+		rt = ip_route_output_flow_prealloc(&init_net, &fl, NULL, &rt_stack.dst);
+		if (IS_ERR(rt)) {
+			err = PTR_ERR(rt);
+			break;
+		}
+		ip_rt_put(rt);
+	}
+	if (err) {
+		pr_info("ip_route_output_key prealloc: err=%d\n", err);
+		return;
+	}
+
+#ifdef IP_ROUTE_CYCLE_SAMPLES
+	ip_route_output_cycles_1 = 0;
+	ip_route_output_cycles_2 = 0;
+	ip_route_output_cycles_3 = 0;
+	ip_route_output_cycles_4 = 0;
+	ip_route_output_cycles_5 = 0;
+	ip_route_output_cycles_6 = 0;
+	ip_route_output_cycles_7 = 0;
+#endif
+
+	flow_init(&fl);
+
+	t1 = get_tick();
+	rt = ip_route_output_flow_prealloc(&init_net, &fl, NULL, &rt_stack.dst);
+	t2 = get_tick();
+	if (!IS_ERR(rt))
+		ip_rt_put(rt);
+
+	tdiff = t2 - t1;
+	pr_info("ip_route_output_key prealloc tdiff: %llu\n", tdiff);
+#ifdef IP_ROUTE_CYCLE_SAMPLES
+	pr_info("ip_route_output_key prealloc cycls: [%d %d %d %d %d %d %d]\n",
+		ip_route_output_cycles_1,
+		ip_route_output_cycles_2,
+		ip_route_output_cycles_3,
+		ip_route_output_cycles_4,
+		ip_route_output_cycles_5,
+		ip_route_output_cycles_6,
+		ip_route_output_cycles_7);
+#endif
+}
+#endif
+
+#ifdef IP_FIB_LOOKUP_EXPORTED
+static void do_fib_lookup_bench(void)
+{
+	unsigned long long t1, t2, tdiff;
+	struct fib_result res;
+	struct flowi fl;
+	int err, i;
+
+	flow_init(&fl);
+
+	for (i = 0; i < warmup_count; i++) {
+		struct fib_table *table;
+
+		table = fib_get_table(&init_net, RT_TABLE_MAIN);
+		if (!table) {
+			pr_info("fib_lookup: No main table.\n");
+			return;
+		}
+		err = fib_table_lookup(table, &fl, &res, FIB_LOOKUP_NOREF);
+		if (err) {
+			pr_info("fib_lookup: err=%d\n", err);
+			return;
+		}
+	}
+
+	{
+		struct fib_table *table;
+
+		t1 = get_tick();
+		table = fib_get_table(&init_net, RT_TABLE_MAIN);
+		if (!table) {
+			pr_info("fib_lookup: No main table.\n");
+			return;
+		}
+		err = fib_table_lookup(table, &fl, &res, FIB_LOOKUP_NOREF);
+		t2 = get_tick();
+	}
+	tdiff = t2 - t1;
+	pr_info("fib_lookup tdiff: %llu\n", tdiff);
+}
+
+struct net_route {
+	struct net_device	*dev;
+	unsigned long		_metrics;
+	struct neighbour	*neighbour;
+	struct hh_cache		*hh;
+	atomic_t		__refcnt;
+};
+
+struct ipv4_route {
+	struct net_route	base;
+
+	int			rt_genid;
+
+	__be32			rt_dst;
+	__be32			rt_src;
+	__u16			rt_type;
+	__u8			rt_tos;
+
+	int			rt_iif;
+	int			rt_oif;
+	__u32			rt_mark;
+
+	__be32			rt_gateway;
+	__be32			rt_spec_dst;
+
+	u32			rt_peer_genid;
+	struct inet_peer	*peer;
+	struct fib_info		*fi;
+};
+
+static atomic_t foo_id = ATOMIC_INIT(1);
+
+static int new_output_lookup(const struct flowi *fl, struct ipv4_route *rt)
+{
+	struct fib_table *table = fib_get_table(&init_net, RT_TABLE_MAIN);
+	struct fib_result res;
+	int err;
+
+	if (!table) {
+		pr_info("new_output_lookup: No main table.\n");
+		return -ENODEV;
+	}
+	err = fib_table_lookup(table, fl, &res, FIB_LOOKUP_NOREF);
+	if (err)
+		return err;
+
+	rt->rt_genid = atomic_read(&foo_id);
+	rt->rt_dst = fl->fl4_dst;
+	rt->rt_src = fl->fl4_src;
+	rt->rt_type = res.type;
+	rt->rt_tos = fl->fl4_tos;
+	rt->rt_iif = fl->iif;
+	rt->rt_oif = fl->oif;
+	rt->rt_mark = fl->mark;
+	rt->rt_gateway = FIB_RES_GW(res);
+	rt->rt_spec_dst = fl->fl4_src;
+	rt->rt_peer_genid = 0;
+	rt->peer = NULL;
+	rt->fi = NULL;
+
+	return 0;
+}
+
+static void do_new_lookup_bench(void)
+{
+	unsigned long long t1, t2, tdiff;
+	struct ipv4_route rt;
+	struct flowi fl;
+	int err, i;
+
+	flow_init(&fl);
+
+	for (i = 0; i < warmup_count; i++) {
+		err = new_output_lookup(&fl, &rt);
+		if (err) {
+			pr_info("new_output_lookup: err=%d\n", err);
+			return;
+		}
+	}
+	t1 = get_tick();
+	err = new_output_lookup(&fl, &rt);
+	t2 = get_tick();
+
+	if (err) {
+		pr_info("new_output_lookup: err=%d\n", err);
+		return;
+	}
+
+	tdiff = t2 - t1;
+	pr_info("new_output_lookup tdiff: %llu\n", tdiff);
+}
+#endif
+
+static void do_bench(void)
+{
+	do_full_lookup_bench();
+	do_full_lookup_bench();
+	do_full_lookup_bench();
+	do_full_lookup_bench();
+
+#ifdef IP_ROUTE_HAVE_PREALLOC
+	do_full_lookup_prealloc_bench();
+	do_full_lookup_prealloc_bench();
+	do_full_lookup_prealloc_bench();
+	do_full_lookup_prealloc_bench();
+#endif
+
+#ifdef IP_FIB_LOOKUP_EXPORTED
+	do_fib_lookup_bench();
+	do_fib_lookup_bench();
+	do_fib_lookup_bench();
+	do_fib_lookup_bench();
+
+	do_new_lookup_bench();
+	do_new_lookup_bench();
+	do_new_lookup_bench();
+	do_new_lookup_bench();
+#endif
+}
+
+static int __init kbench_init(void)
+{
+	flow_setup();
+
+	pr_info("flow [IIF(%d),OIF(%d),MARK(0x%08x),D(%pI4),S(%pI4),TOS(0x%02x)]\n",
+		flow_iif, flow_oif, flow_mark,
+		&flow_dst_ip_addr,
+		&flow_src_ip_addr, flow_tos);
+
+#if defined(CONFIG_X86)
+	if (!cpu_has_tsc) {
+		pr_err("X86 TSC is required, but is unavailable.\n");
+		return -EINVAL;
+	}
+#endif
+
+	pr_info("sizeof(struct rtable)==%Zd\n", sizeof(struct rtable));
+
+	do_bench();
+
+	return -ENODEV;
+}
+
+static void __exit kbench_exit(void)
+{
+}
+
+module_init(kbench_init);
+module_exit(kbench_exit);
+MODULE_LICENSE("GPL");