[v11,09/12] x86/smpboot: Remove initial_stack on 64-bit
Commit Message
From: Brian Gerst <brgerst@gmail.com>
Load RSP from current_task->thread.sp instead.
Signed-off-by: Brian Gerst <brgerst@gmail.com>
Reviewed-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Tested-by: Usama Arif <usama.arif@bytedance.com>
Signed-off-by: Usama Arif <usama.arif@bytedance.com>
---
arch/x86/include/asm/processor.h | 6 +++++-
arch/x86/kernel/acpi/sleep.c | 2 +-
arch/x86/kernel/head_64.S | 35 ++++++++++++++++++--------------
arch/x86/xen/xen-head.S | 2 +-
4 files changed, 27 insertions(+), 18 deletions(-)
Comments
From: Usama Arif <usama.arif@bytedance.com> Sent: Thursday, February 23, 2023 11:12 AM
>
> From: Brian Gerst <brgerst@gmail.com>
>
> Load RSP from current_task->thread.sp instead.
>
> Signed-off-by: Brian Gerst <brgerst@gmail.com>
> Reviewed-by: David Woodhouse <dwmw@amazon.co.uk>
> Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
> Tested-by: Usama Arif <usama.arif@bytedance.com>
> Signed-off-by: Usama Arif <usama.arif@bytedance.com>
> ---
> arch/x86/include/asm/processor.h | 6 +++++-
> arch/x86/kernel/acpi/sleep.c | 2 +-
> arch/x86/kernel/head_64.S | 35 ++++++++++++++++++--------------
> arch/x86/xen/xen-head.S | 2 +-
> 4 files changed, 27 insertions(+), 18 deletions(-)
>
> diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
> index 4e35c66edeb7..9c4a5c4d46c1 100644
> --- a/arch/x86/include/asm/processor.h
> +++ b/arch/x86/include/asm/processor.h
> @@ -648,7 +648,11 @@ static inline void spin_lock_prefetch(const void *x)
> #define KSTK_ESP(task) (task_pt_regs(task)->sp)
>
> #else
> -#define INIT_THREAD { }
> +extern unsigned long __end_init_task[];
> +
> +#define INIT_THREAD { \
> + .sp = (unsigned long)&__end_init_task - PTREGS_SIZE, \
> +}
I'm getting a compile error on the new reference to PTREGS_SIZE:
In file included from ./arch/x86/include/asm/cpufeature.h:5:0,
from ./arch/x86/include/asm/thread_info.h:53,
from ./include/linux/thread_info.h:60,
from ./arch/x86/include/asm/preempt.h:9,
from ./include/linux/preempt.h:78,
from ./include/linux/rcupdate.h:27,
from ./include/linux/init_task.h:5,
from init/init_task.c:2:
./arch/x86/include/asm/processor.h:654:42: error: 'PTREGS_SIZE' undeclared here (not in a function); did you mean 'TLS_SIZE'?
.sp = (unsigned long)&__end_init_task - PTREGS_SIZE, \
^
init/init_task.c:115:13: note: in expansion of macro 'INIT_THREAD'
.thread = INIT_THREAD,
^~~~~~~~~~~
scripts/Makefile.build:252: recipe for target 'init/init_task.o' failed
Michael
On Fri, 2023-02-24 at 18:24 +0000, Michael Kelley (LINUX) wrote:
> I'm getting a compile error on the new reference to PTREGS_SIZE:
Is it just that <asm/asm-offsets.h> is included conditionally, and not
in your build? What if you include it directly from <asm/processor.h>
On 24/02/2023 18:28, David Woodhouse wrote:
> On Fri, 2023-02-24 at 18:24 +0000, Michael Kelley (LINUX) wrote:
>> I'm getting a compile error on the new reference to PTREGS_SIZE:
>
> Is it just that <asm/asm-offsets.h> is included conditionally, and not
> in your build? What if you include it directly from <asm/processor.h>
This is fixed in the v2 patch that Brian sent.
diff --git a/arch/x86/include/asm/processor.h
b/arch/x86/include/asm/processor.h
index 9c4a5c4d46c1..bdde7316e75b 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -650,8 +650,8 @@ static inline void spin_lock_prefetch(const void *x)
#else
extern unsigned long __end_init_task[];
-#define INIT_THREAD { \
- .sp = (unsigned long)&__end_init_task - PTREGS_SIZE, \
+#define INIT_THREAD {
\
+ .sp = (unsigned long)&__end_init_task - sizeof(struct
pt_regs), \
}
extern unsigned long KSTK_ESP(struct task_struct *task);
@@ -648,7 +648,11 @@ static inline void spin_lock_prefetch(const void *x)
#define KSTK_ESP(task) (task_pt_regs(task)->sp)
#else
-#define INIT_THREAD { }
+extern unsigned long __end_init_task[];
+
+#define INIT_THREAD { \
+ .sp = (unsigned long)&__end_init_task - PTREGS_SIZE, \
+}
extern unsigned long KSTK_ESP(struct task_struct *task);
@@ -113,7 +113,7 @@ int x86_acpi_suspend_lowlevel(void)
saved_magic = 0x12345678;
#else /* CONFIG_64BIT */
#ifdef CONFIG_SMP
- initial_stack = (unsigned long)temp_stack + sizeof(temp_stack);
+ current->thread.sp = (unsigned long)temp_stack + sizeof(temp_stack);
early_gdt_descr.address =
(unsigned long)get_cpu_gdt_rw(smp_processor_id());
initial_gs = per_cpu_offset(smp_processor_id());
@@ -62,8 +62,8 @@ SYM_CODE_START_NOALIGN(startup_64)
* tables and then reload them.
*/
- /* Set up the stack for verify_cpu(), similar to initial_stack below */
- leaq (__end_init_task - FRAME_SIZE)(%rip), %rsp
+ /* Set up the stack for verify_cpu() */
+ leaq (__end_init_task - PTREGS_SIZE)(%rip), %rsp
leaq _text(%rip), %rdi
@@ -245,11 +245,11 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
#ifdef CONFIG_SMP
/*
* Is this the boot CPU coming up? If so everything is available
- * in initial_gs, initial_stack and early_gdt_descr.
+ * in initial_gs and early_gdt_descr.
*/
movl smpboot_control(%rip), %edx
testl $STARTUP_SECONDARY, %edx
- jz .Lsetup_cpu
+ jz .Linit_cpu0_data
/*
* For parallel boot, the APIC ID is retrieved from CPUID, and then
@@ -302,6 +302,10 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
hlt
jmp 1b
+.Linit_cpu0_data:
+ movq __per_cpu_offset(%rip), %rdx
+ jmp .Lsetup_cpu
+
.Linit_cpu_data:
/* Get the per cpu offset for the given CPU# which is in ECX */
leaq __per_cpu_offset(%rip), %rbx
@@ -314,13 +318,21 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
addq %rbx, %rcx
movq %rcx, early_gdt_descr_base(%rip)
- /* Find the idle task stack */
- movq pcpu_hot + X86_current_task(%rbx), %rcx
- movq TASK_threadsp(%rcx), %rcx
- movq %rcx, initial_stack(%rip)
+ movq %rbx, %rdx
+#else
+ xorl %edx, %edx
#endif /* CONFIG_SMP */
.Lsetup_cpu:
+ /*
+ * Setup a boot time stack - Any secondary CPU will have lost its stack
+ * by now because the cr3-switch above unmaps the real-mode stack
+ *
+ * RDX contains the per-cpu offset
+ */
+ movq pcpu_hot + X86_current_task(%rdx), %rax
+ movq TASK_threadsp(%rax), %rsp
+
/*
* We must switch to a new descriptor in kernel space for the GDT
* because soon the kernel won't have access anymore to the userspace
@@ -355,12 +367,6 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
movl initial_gs+4(%rip),%edx
wrmsr
- /*
- * Setup a boot time stack - Any secondary CPU will have lost its stack
- * by now because the cr3-switch above unmaps the real-mode stack
- */
- movq initial_stack(%rip), %rsp
-
/* Drop the realmode protection. For the boot CPU the pointer is NULL! */
movq trampoline_lock(%rip), %rax
testq %rax, %rax
@@ -517,7 +523,6 @@ SYM_DATA(initial_vc_handler, .quad handle_vc_boot_ghcb)
* The FRAME_SIZE gap is a convention which helps the in-kernel unwinder
* reliably detect the end of the stack.
*/
-SYM_DATA(initial_stack, .quad init_thread_union + THREAD_SIZE - FRAME_SIZE)
SYM_DATA(trampoline_lock, .quad 0);
__FINITDATA
@@ -49,7 +49,7 @@ SYM_CODE_START(startup_xen)
ANNOTATE_NOENDBR
cld
- mov initial_stack(%rip), %rsp
+ leaq (__end_init_task - PTREGS_SIZE)(%rip), %rsp
/* Set up %gs.
*