summaryrefslogtreecommitdiff
path: root/arch/x86/entry/entry_64.S
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/entry/entry_64.S')
-rw-r--r--arch/x86/entry/entry_64.S23
1 files changed, 21 insertions, 2 deletions
diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S
index 870e941c1947..10ecfba43dff 100644
--- a/arch/x86/entry/entry_64.S
+++ b/arch/x86/entry/entry_64.S
@@ -313,6 +313,7 @@ ENTRY(__switch_to_asm)
pushq %r13
pushq %r14
pushq %r15
+ pushfq
/* switch stack */
movq %rsp, TASK_threadsp(%rdi)
@@ -335,6 +336,7 @@ ENTRY(__switch_to_asm)
#endif
/* restore callee-saved registers */
+ popfq
popq %r15
popq %r14
popq %r13
@@ -418,6 +420,7 @@ END(irq_entries_start)
* tracking that we're in kernel mode.
*/
SWAPGS
+ FENCE_SWAPGS_USER_ENTRY
SWITCH_KERNEL_CR3
/*
@@ -431,8 +434,10 @@ END(irq_entries_start)
TRACE_IRQS_OFF
CALL_enter_from_user_mode
-
+ jmp 2f
1:
+ FENCE_SWAPGS_KERNEL_ENTRY
+2:
/*
* Save previous stack pointer, optionally switch to interrupt stack.
* irq_count is used to check if a CPU is already on an interrupt stack
@@ -1002,6 +1007,13 @@ ENTRY(paranoid_entry)
movq %rax, %cr3
2:
#endif
+ /*
+ * The above doesn't do an unconditional CR3 write, even in the PTI
+ * case. So do an lfence to prevent GS speculation, regardless of
+ * whether PTI is enabled.
+ */
+ FENCE_SWAPGS_KERNEL_ENTRY
+
ret
END(paranoid_entry)
@@ -1063,6 +1075,7 @@ ENTRY(error_entry)
* from user mode due to an IRET fault.
*/
SWAPGS
+ FENCE_SWAPGS_USER_ENTRY
.Lerror_entry_from_usermode_after_swapgs:
/*
@@ -1074,6 +1087,8 @@ ENTRY(error_entry)
CALL_enter_from_user_mode
ret
+.Lerror_entry_done_lfence:
+ FENCE_SWAPGS_KERNEL_ENTRY
.Lerror_entry_done:
TRACE_IRQS_OFF
ret
@@ -1092,7 +1107,7 @@ ENTRY(error_entry)
cmpq %rax, RIP+8(%rsp)
je .Lbstep_iret
cmpq $.Lgs_change, RIP+8(%rsp)
- jne .Lerror_entry_done
+ jne .Lerror_entry_done_lfence
/*
* hack: .Lgs_change can fail with user gsbase. If this happens, fix up
@@ -1100,6 +1115,7 @@ ENTRY(error_entry)
* .Lgs_change's error handler with kernel gsbase.
*/
SWAPGS
+ FENCE_SWAPGS_USER_ENTRY
jmp .Lerror_entry_done
.Lbstep_iret:
@@ -1113,6 +1129,7 @@ ENTRY(error_entry)
* Switch to kernel gsbase:
*/
SWAPGS
+ FENCE_SWAPGS_USER_ENTRY
/*
* Pretend that the exception came from user mode: set up pt_regs
@@ -1209,6 +1226,7 @@ ENTRY(nmi)
* to switch CR3 here.
*/
cld
+ FENCE_SWAPGS_USER_ENTRY
movq %rsp, %rdx
movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
pushq 5*8(%rdx) /* pt_regs->ss */
@@ -1497,6 +1515,7 @@ end_repeat_nmi:
movq %rax, %cr3
2:
#endif
+ FENCE_SWAPGS_KERNEL_ENTRY
/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
call do_nmi