diff options
| author | Peter Zijlstra <peterz@infradead.org> | 2023-11-20 15:33:46 +0100 |
|---|---|---|
| committer | Ingo Molnar <mingo@kernel.org> | 2023-11-21 13:57:31 +0100 |
| commit | 1e4d3001f59fb7a9917cb746544b65e616b5f809 (patch) | |
| tree | 6d04c0058f5f125f755e883acbd8558e7f8773ee | |
| parent | c516213726fb572700cce4a5909aa8d82b77192a (diff) | |
| download | queue-1e4d3001f59fb7a9917cb746544b65e616b5f809.tar.gz | |
x86/entry: Harden return-to-userx86/entry
Make the CONFIG_DEBUG_ENTRY=y check that validates CS is a user segment
unconditional and move it nearer to IRET.
PRE:
140,026,608 cycles:k ( +- 0.01% )
236,696,176 instructions:k # 1.69 insn per cycle ( +- 0.00% )
POST:
139,957,681 cycles:k ( +- 0.01% )
236,681,819 instructions:k # 1.69 insn per cycle ( +- 0.00% )
(this is with --repeat 100 and the run-to-run variance is bigger than
the difference shown)
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20231120143626.753200755@infradead.org
| -rw-r--r-- | arch/x86/entry/entry_64.S | 18 |
1 files changed, 8 insertions, 10 deletions
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index dfbf799b7311db..c40f89ab1b4c70 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -559,13 +559,6 @@ __irqentry_text_end: SYM_CODE_START_LOCAL(common_interrupt_return) SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) IBRS_EXIT -#ifdef CONFIG_DEBUG_ENTRY - /* Assert that pt_regs indicates user mode. */ - testb $3, CS(%rsp) - jnz 1f - ud2 -1: -#endif #ifdef CONFIG_XEN_PV ALTERNATIVE "", "jmp xenpv_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV #endif @@ -576,8 +569,14 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) STACKLEAK_ERASE POP_REGS add $8, %rsp /* orig_ax */ + UNWIND_HINT_IRET_REGS + +.Lswapgs_and_iret: swapgs - jmp .Lnative_iret + /* Assert that the IRET frame indicates user mode. */ + testb $3, 8(%rsp) + jnz .Lnative_iret + ud2 #ifdef CONFIG_PAGE_TABLE_ISOLATION .Lpti_restore_regs_and_return_to_usermode: @@ -613,8 +612,7 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) /* Restore RDI. */ popq %rdi - swapgs - jmp .Lnative_iret + jmp .Lswapgs_and_iret #endif SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL) |