x86-32: Add stack smashing debugging

This patch introduces -Xdebug:stack command line option that enables JIT code
stack smashing detection.

Signed-off-by: Pekka Enberg <penberg@kernel.org>

5 files changed, 61 insertions, 5 deletions

diff --git a/Documentation/options.txt b/Documentation/options.txt
index b5fc9258..d1a89f3a 100644
--- a/Documentation/options.txt
+++ b/Documentation/options.txt
@@ -17,3 +17,6 @@ Jato supports the following command line options:
 
     -Xtrace:trampoline
       Trace executed trampolines.
+
+    -Xdebug:stack
+      Enable stack smashing debugging.
diff --git a/arch/x86/emit_32.c b/arch/x86/emit_32.c
index ed81d256..8e62e1a0 100644
--- a/arch/x86/emit_32.c
+++ b/arch/x86/emit_32.c
@@ -47,6 +47,7 @@
 #include "jit/compiler.h"
 #include "jit/exception.h"
 #include "jit/emit-code.h"
+#include "jit/debug.h"
 #include "jit/text.h"
 
 #include "lib/buffer.h"
@@ -567,6 +568,11 @@ static void __emit_sub_imm_reg(struct buffer *buf, unsigned long imm,
 	emit_alu_imm_reg(buf, 0x05, imm, reg);
 }
 
+static void __emit_cmp_imm_reg(struct buffer *buf, int rex_w, long imm, enum machine_reg reg)
+{
+	emit_alu_imm_reg(buf, 0x07, imm, reg);
+}
+
 static void __emit_test_imm_memdisp(struct buffer *buf,
 	long imm, long disp)
 {
@@ -610,6 +616,8 @@ static void emit_restore_callee_save_regs(struct buffer *buf)
 	}
 }
 
+#define STACK_FRAME_REDZONE_END		0xdeadbeef
+
 void emit_prolog(struct buffer *buf, struct stack_frame *frame,
 					unsigned long frame_size)
 {
@@ -620,6 +628,32 @@ void emit_prolog(struct buffer *buf, struct stack_frame *frame,
 
 	if (frame_size)
 		__emit_sub_imm_reg(buf, frame_size, MACH_REG_ESP);
+
+	if (opt_debug_stack)
+		__emit_push_imm(buf, STACK_FRAME_REDZONE_END);
+}
+
+/* magic is in ecx */
+static void __attribute__((regparm(1)))
+stack_frame_redzone_fail(void *magic)
+{
+	printf("Redzone overwritten: %p\n", magic);
+	abort();
+}
+
+static void emit_stack_redzone_check(struct buffer *buf)
+{
+	__emit_pop_reg(buf, MACH_REG_ECX);
+
+	__emit_cmp_imm_reg(buf, 1, STACK_FRAME_REDZONE_END, MACH_REG_ECX);
+
+	/* open-coded "jne" */
+	emit(buf, 0x0f);
+	emit(buf, 0x85);
+
+	uint8_t *jne_addr = buffer_current(buf);
+	emit_imm32(buf, 0);
+	fixup_branch_target(jne_addr, stack_frame_redzone_fail);
 }
 
 static void emit_restore_regs(struct buffer *buf)
@@ -629,6 +663,9 @@ static void emit_restore_regs(struct buffer *buf)
 
 void emit_epilog(struct buffer *buf)
 {
+	if (opt_debug_stack)
+		emit_stack_redzone_check(buf);
+
 	emit_leave(buf);
 	emit_restore_regs(buf);
 	emit_ret(buf);
@@ -640,6 +677,9 @@ void emit_epilog(struct buffer *buf)
  */
 void emit_unwind(struct buffer *buf)
 {
+	if (opt_debug_stack)
+		emit_stack_redzone_check(buf);
+
 	emit_leave(buf);
 	emit_restore_regs(buf);
 	__emit_jmp(buf, (unsigned long)&unwind);
@@ -760,11 +800,6 @@ static void __emit_add_imm_reg(struct buffer *buf, long imm, enum machine_reg re
 	emit_alu_imm_reg(buf, 0x00, imm, reg);
 }
 
-static void __emit_cmp_imm_reg(struct buffer *buf, int rex_w, long imm, enum machine_reg reg)
-{
-	emit_alu_imm_reg(buf, 0x07, imm, reg);
-}
-
 static void emit_indirect_jump_reg(struct buffer *buf, enum machine_reg reg)
 {
 	emit(buf, 0xff);
diff --git a/include/jit/debug.h b/include/jit/debug.h
new file mode 100644
index 00000000..11b01e3e
--- /dev/null
+++ b/include/jit/debug.h
@@ -0,0 +1,8 @@
+#ifndef JATO_JIT_DEBUG_H
+#define JATO_JIT_DEBUG_H
+
+#include <stdbool.h>
+
+extern bool opt_debug_stack;
+
+#endif /* JATO_JIT_DEBUG_H */
diff --git a/jit/emit.c b/jit/emit.c
index ef1803ec..e4342d9d 100644
--- a/jit/emit.c
+++ b/jit/emit.c
@@ -24,6 +24,7 @@
 #include "jit/gdb.h"
 #include "jit/instruction.h"
 #include "jit/statement.h"
+#include "jit/debug.h"
 #include "jit/text.h"
 
 #include <stdlib.h>
@@ -31,6 +32,8 @@
 #include <errno.h>
 #include <stdio.h>
 
+bool opt_debug_stack;
+
 static void emit_monitorenter(struct compilation_unit *cu)
 {
 	if (vm_method_is_static(cu->method))
diff --git a/vm/jato.c b/vm/jato.c
index 247457ea..261df2be 100644
--- a/vm/jato.c
+++ b/vm/jato.c
@@ -59,6 +59,7 @@
 #include "jit/exception.h"
 #include "jit/inline-cache.h"
 #include "jit/perf-map.h"
+#include "jit/debug.h"
 #include "jit/text.h"
 
 #include "lib/string.h"
@@ -754,6 +755,11 @@ static void handle_trace_gate(const char *arg)
 	regex_compile(&method_trace_gate_regex, arg);
 }
 
+static void handle_debug_stack(void)
+{
+	opt_debug_stack = true;
+}
+
 static void handle_trace_asm(void)
 {
 	opt_trace_machine_code = true;
@@ -927,6 +933,7 @@ const struct option options[] = {
 	DEFINE_OPTION("Xnoic",			handle_no_ic),
 	DEFINE_OPTION("Xint",			handle_int),
 
+	DEFINE_OPTION("Xdebug:stack",		handle_debug_stack),
 	DEFINE_OPTION("Xtrace:asm",		handle_trace_asm),
 	DEFINE_OPTION("Xtrace:bytecode",	handle_trace_bytecode),
 	DEFINE_OPTION("Xtrace:bytecode-offset",	handle_trace_bytecode_offset),