diff mbox series

[v16,7/7] x86/crash: add x86 crash hotplug support

Message ID 20230105151709.1845-8-eric.devolder@oracle.com
State New
Headers
Series crash: Kernel handling of CPU and memory hot un/plug |

Commit Message

Eric DeVolder Jan. 5, 2023, 3:17 p.m. UTC 
  When CPU or memory is hot un/plugged, the crash elfcorehdr, which
describes the CPUs and memory in the system, must also be updated.

A new elfcorehdr is generated from the available CPUs and memory
into a buffer, and then installed over the top of the existing
elfcorehdr. The segment containing the elfcorehdr is identified
at run time in crash_core:handle_hotplug_event(), which works for
both the kexec_load() and kexec_file_load() syscalls.

In the patch 'kexec: exclude elfcorehdr from the segment digest'
the need to update purgatory due to the change in elfcorehdr was
eliminated.  As a result, no changes to purgatory or boot_params
(as the elfcorehdr= kernel command line parameter pointer
remains unchanged and correct) are needed, just elfcorehdr.

To accommodate a growing number of resources via hotplug, the
elfcorehdr segment must be sufficiently large enough to accommodate
changes, see the CRASH_MAX_MEMORY_RANGES description.

Signed-off-by: Eric DeVolder <eric.devolder@oracle.com>
---
 arch/x86/Kconfig             |  12 ++++
 arch/x86/include/asm/kexec.h |  15 +++++
 arch/x86/kernel/crash.c      | 112 ++++++++++++++++++++++++++++++++++-
 3 files changed, 136 insertions(+), 3 deletions(-)

Comments

Baoquan He Jan. 9, 2023, 7:36 a.m. UTC | #1 
On 01/05/23 at 10:17am, Eric DeVolder wrote:
......
> @@ -394,10 +409,37 @@ int crash_load_segments(struct kimage *image)
>  	if (ret)
>  		return ret;
>  
> -	image->elf_headers = kbuf.buffer;
> -	image->elf_headers_sz = kbuf.bufsz;
> +	image->elf_headers	= kbuf.buffer;
> +	image->elf_headers_sz	= kbuf.bufsz;
> +	kbuf.memsz		= kbuf.bufsz;
> +
> +	if (IS_ENABLED(CONFIG_CRASH_HOTPLUG)) {
> +		/*
> +		 * Ensure the elfcorehdr segment large enough for hotplug changes.
> +		 * Start with VMCOREINFO and kernel_map.
> +		 */
> +		unsigned long pnum = 2;
> +
> +		if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
> +			pnum += CONFIG_NR_CPUS_DEFAULT;
> +
> +		if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG))
> +			pnum += CRASH_MAX_MEMORY_RANGES;

Logic of pnum calculating is a little confusing to me. If I only enable
one of CONFIG_HOTPLUG_CPU and CONFIG_MEMORY_HOTPLUG, is it OK? Say I
only enable CONFIG_HOTPLUG_CPU and CONFIG_SMP, on x86_64,
CONFIG_NR_CPUS_DEFAULT will be 64. pnum will be 64, is it OK. Am I miss
anything?

> +
> +		if (pnum < (unsigned long)PN_XNUM) {
> +			kbuf.memsz = pnum * sizeof(Elf64_Phdr);
> +			kbuf.memsz += sizeof(Elf64_Ehdr);
> +
> +			image->elfcorehdr_index = image->nr_segments;
> +			image->elfcorehdr_index_valid = true;
> +
> +			/* Mark as usable to crash kernel, else crash kernel fails on boot */
> +			image->elf_headers_sz = kbuf.memsz;
> +		} else {
> +			pr_err("number of Phdrs %lu exceeds max\n", pnum);
> +		}
> +	}
>  
> -	kbuf.memsz = kbuf.bufsz;
>  	kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
>  	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
>  	ret = kexec_add_buffer(&kbuf);
> @@ -412,3 +454,67 @@ int crash_load_segments(struct kimage *image)
>  	return ret;
>  }
>  #endif /* CONFIG_KEXEC_FILE */
Eric DeVolder Jan. 9, 2023, 7:43 p.m. UTC | #2 
On 1/9/23 01:36, Baoquan He wrote:
> On 01/05/23 at 10:17am, Eric DeVolder wrote:
> ......
>> @@ -394,10 +409,37 @@ int crash_load_segments(struct kimage *image)
>>   	if (ret)
>>   		return ret;
>>   
>> -	image->elf_headers = kbuf.buffer;
>> -	image->elf_headers_sz = kbuf.bufsz;
>> +	image->elf_headers	= kbuf.buffer;
>> +	image->elf_headers_sz	= kbuf.bufsz;
>> +	kbuf.memsz		= kbuf.bufsz;
>> +
>> +	if (IS_ENABLED(CONFIG_CRASH_HOTPLUG)) {
>> +		/*
>> +		 * Ensure the elfcorehdr segment large enough for hotplug changes.
>> +		 * Start with VMCOREINFO and kernel_map.
>> +		 */
>> +		unsigned long pnum = 2;
>> +
>> +		if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
>> +			pnum += CONFIG_NR_CPUS_DEFAULT;
>> +
>> +		if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG))
>> +			pnum += CRASH_MAX_MEMORY_RANGES;
> 
> Logic of pnum calculating is a little confusing to me. If I only enable
> one of CONFIG_HOTPLUG_CPU and CONFIG_MEMORY_HOTPLUG, is it OK? Say I
> only enable CONFIG_HOTPLUG_CPU and CONFIG_SMP, on x86_64,
> CONFIG_NR_CPUS_DEFAULT will be 64. pnum will be 64, is it OK. Am I miss
> anything?

Ah, your understanding is correct, and the issue you point out I need to fix.
Specifically is only one of HOTPLUG_CPU or MEMORY_HOTPLUG is set, then I need
to allow a default for the other. Using the example you have above, there would
not be room for a single memory region, so I need to allow for a default number
of memory regions.

Good find!
eric

> 
>> +
>> +		if (pnum < (unsigned long)PN_XNUM) {
>> +			kbuf.memsz = pnum * sizeof(Elf64_Phdr);
>> +			kbuf.memsz += sizeof(Elf64_Ehdr);
>> +
>> +			image->elfcorehdr_index = image->nr_segments;
>> +			image->elfcorehdr_index_valid = true;
>> +
>> +			/* Mark as usable to crash kernel, else crash kernel fails on boot */
>> +			image->elf_headers_sz = kbuf.memsz;
>> +		} else {
>> +			pr_err("number of Phdrs %lu exceeds max\n", pnum);
>> +		}
>> +	}
>>   
>> -	kbuf.memsz = kbuf.bufsz;
>>   	kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
>>   	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
>>   	ret = kexec_add_buffer(&kbuf);
>> @@ -412,3 +454,67 @@ int crash_load_segments(struct kimage *image)
>>   	return ret;
>>   }
>>   #endif /* CONFIG_KEXEC_FILE */
>
Baoquan He Jan. 9, 2023, 11:47 p.m. UTC | #3 
On 01/09/23 at 01:43pm, Eric DeVolder wrote:
> 
> 
> On 1/9/23 01:36, Baoquan He wrote:
> > On 01/05/23 at 10:17am, Eric DeVolder wrote:
> > ......
> > > @@ -394,10 +409,37 @@ int crash_load_segments(struct kimage *image)
> > >   	if (ret)
> > >   		return ret;
> > > -	image->elf_headers = kbuf.buffer;
> > > -	image->elf_headers_sz = kbuf.bufsz;
> > > +	image->elf_headers	= kbuf.buffer;
> > > +	image->elf_headers_sz	= kbuf.bufsz;
> > > +	kbuf.memsz		= kbuf.bufsz;
> > > +
> > > +	if (IS_ENABLED(CONFIG_CRASH_HOTPLUG)) {
> > > +		/*
> > > +		 * Ensure the elfcorehdr segment large enough for hotplug changes.
> > > +		 * Start with VMCOREINFO and kernel_map.
> > > +		 */
> > > +		unsigned long pnum = 2;
> > > +
> > > +		if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
> > > +			pnum += CONFIG_NR_CPUS_DEFAULT;
> > > +
> > > +		if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG))
> > > +			pnum += CRASH_MAX_MEMORY_RANGES;
> > 
> > Logic of pnum calculating is a little confusing to me. If I only enable
> > one of CONFIG_HOTPLUG_CPU and CONFIG_MEMORY_HOTPLUG, is it OK? Say I
> > only enable CONFIG_HOTPLUG_CPU and CONFIG_SMP, on x86_64,
> > CONFIG_NR_CPUS_DEFAULT will be 64. pnum will be 64, is it OK. Am I miss
> > anything?
> 
> Ah, your understanding is correct, and the issue you point out I need to fix.
> Specifically is only one of HOTPLUG_CPU or MEMORY_HOTPLUG is set, then I need
> to allow a default for the other. Using the example you have above, there would
> not be room for a single memory region, so I need to allow for a default number
> of memory regions.

Hmm, for the above example, should we get real number of memory
regions? I am worried if the default number of memory regions 
need be big enough. Because on system w/o memory hotplug, it could have
very many memory regions. The cpu number is similar.
diff mbox series

Patch

diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 3604074a878b..09e9b7a091ed 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -2119,6 +2119,18 @@  config CRASH_DUMP
 	  (CONFIG_RELOCATABLE=y).
 	  For more details see Documentation/admin-guide/kdump/kdump.rst
 
+config CRASH_HOTPLUG
+	bool "Update the crash elfcorehdr on system configuration changes"
+	default n
+	depends on CRASH_DUMP && (HOTPLUG_CPU || MEMORY_HOTPLUG)
+	help
+	  Enable direct update to the crash elfcorehdr (which contains
+	  the list of CPUs and memory regions to be dumped upon a crash)
+	  in response to hot plug/unplug of CPUs or memory. This is a much
+	  more advanced approach than userspace attempting that.
+
+	  If unsure, say Y.
+
 config KEXEC_JUMP
 	bool "kexec jump"
 	depends on KEXEC && HIBERNATION
diff --git a/arch/x86/include/asm/kexec.h b/arch/x86/include/asm/kexec.h
index a3760ca796aa..1bc852ce347d 100644
--- a/arch/x86/include/asm/kexec.h
+++ b/arch/x86/include/asm/kexec.h
@@ -212,6 +212,21 @@  typedef void crash_vmclear_fn(void);
 extern crash_vmclear_fn __rcu *crash_vmclear_loaded_vmcss;
 extern void kdump_nmi_shootdown_cpus(void);
 
+#ifdef CONFIG_CRASH_HOTPLUG
+void arch_crash_handle_hotplug_event(struct kimage *image);
+#define arch_crash_handle_hotplug_event arch_crash_handle_hotplug_event
+
+#ifdef CONFIG_HOTPLUG_CPU
+static inline int crash_hotplug_cpu_support(void) { return 1; }
+#define crash_hotplug_cpu_support crash_hotplug_cpu_support
+#endif
+
+#ifdef CONFIG_MEMORY_HOTPLUG
+static inline int crash_hotplug_memory_support(void) { return 1; }
+#define crash_hotplug_memory_support crash_hotplug_memory_support
+#endif
+#endif
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* _ASM_X86_KEXEC_H */
diff --git a/arch/x86/kernel/crash.c b/arch/x86/kernel/crash.c
index 9ceb93c176a6..76e2431dec63 100644
--- a/arch/x86/kernel/crash.c
+++ b/arch/x86/kernel/crash.c
@@ -42,6 +42,21 @@ 
 #include <asm/crash.h>
 #include <asm/cmdline.h>
 
+/*
+ * For the kexec_file_load() syscall path, specify the maximum number of
+ * memory regions that the elfcorehdr buffer/segment can accommodate.
+ * These regions are obtained via walk_system_ram_res(); eg. the
+ * 'System RAM' entries in /proc/iomem.
+ * This value is combined with NR_CPUS_DEFAULT and multiplied by
+ * sizeof(Elf64_Phdr) to determine the final elfcorehdr memory buffer/
+ * segment size.
+ * The value 8192, for example, covers a (sparsely populated) 1TiB system
+ * consisting of 128MiB memblocks, while resulting in an elfcorehdr
+ * memory buffer/segment size under 1MiB. This represents a sane choice
+ * to accommodate both baremetal and virtual machine configurations.
+ */
+#define CRASH_MAX_MEMORY_RANGES 8192
+
 /* Used while preparing memory map entries for second kernel */
 struct crash_memmap_data {
 	struct boot_params *params;
@@ -394,10 +409,37 @@  int crash_load_segments(struct kimage *image)
 	if (ret)
 		return ret;
 
-	image->elf_headers = kbuf.buffer;
-	image->elf_headers_sz = kbuf.bufsz;
+	image->elf_headers	= kbuf.buffer;
+	image->elf_headers_sz	= kbuf.bufsz;
+	kbuf.memsz		= kbuf.bufsz;
+
+	if (IS_ENABLED(CONFIG_CRASH_HOTPLUG)) {
+		/*
+		 * Ensure the elfcorehdr segment large enough for hotplug changes.
+		 * Start with VMCOREINFO and kernel_map.
+		 */
+		unsigned long pnum = 2;
+
+		if (IS_ENABLED(CONFIG_HOTPLUG_CPU))
+			pnum += CONFIG_NR_CPUS_DEFAULT;
+
+		if (IS_ENABLED(CONFIG_MEMORY_HOTPLUG))
+			pnum += CRASH_MAX_MEMORY_RANGES;
+
+		if (pnum < (unsigned long)PN_XNUM) {
+			kbuf.memsz = pnum * sizeof(Elf64_Phdr);
+			kbuf.memsz += sizeof(Elf64_Ehdr);
+
+			image->elfcorehdr_index = image->nr_segments;
+			image->elfcorehdr_index_valid = true;
+
+			/* Mark as usable to crash kernel, else crash kernel fails on boot */
+			image->elf_headers_sz = kbuf.memsz;
+		} else {
+			pr_err("number of Phdrs %lu exceeds max\n", pnum);
+		}
+	}
 
-	kbuf.memsz = kbuf.bufsz;
 	kbuf.buf_align = ELF_CORE_HEADER_ALIGN;
 	kbuf.mem = KEXEC_BUF_MEM_UNKNOWN;
 	ret = kexec_add_buffer(&kbuf);
@@ -412,3 +454,67 @@  int crash_load_segments(struct kimage *image)
 	return ret;
 }
 #endif /* CONFIG_KEXEC_FILE */
+
+#ifdef CONFIG_CRASH_HOTPLUG
+
+#undef pr_fmt
+#define pr_fmt(fmt) "crash hp: " fmt
+
+/**
+ * arch_crash_handle_hotplug_event() - Handle hotplug elfcorehdr changes
+ * @image: the active struct kimage
+ *
+ * To accurately reflect hot un/plug changes, the new elfcorehdr
+ * is prepared in a kernel buffer, and then it is written on top
+ * of the existing/old elfcorehdr.
+ */
+void arch_crash_handle_hotplug_event(struct kimage *image)
+{
+	void *elfbuf = NULL, *old_elfcorehdr;
+	unsigned long mem, memsz;
+	unsigned long elfsz = 0;
+
+	/*
+	 * Create the new elfcorehdr reflecting the changes to CPU and/or
+	 * memory resources.
+	 */
+	if (prepare_elf_headers(image, &elfbuf, &elfsz)) {
+		pr_err("unable to prepare elfcore headers");
+		goto out;
+	}
+
+	/*
+	 * Obtain address and size of the elfcorehdr segment, and
+	 * check it against the new elfcorehdr buffer.
+	 */
+	mem = image->segment[image->elfcorehdr_index].mem;
+	memsz = image->segment[image->elfcorehdr_index].memsz;
+	if (elfsz > memsz) {
+		pr_err("update elfcorehdr elfsz %lu > memsz %lu",
+			elfsz, memsz);
+		goto out;
+	}
+
+	/*
+	 * Copy new elfcorehdr over the old elfcorehdr at destination.
+	 */
+	old_elfcorehdr = kmap_local_page(pfn_to_page(mem >> PAGE_SHIFT));
+	if (!old_elfcorehdr) {
+		pr_err("updating elfcorehdr failed\n");
+		goto out;
+	}
+
+	/*
+	 * Temporarily invalidate the crash image while the
+	 * elfcorehdr is updated.
+	 */
+	xchg(&kexec_crash_image, NULL);
+	memcpy_flushcache(old_elfcorehdr, elfbuf, elfsz);
+	xchg(&kexec_crash_image, image);
+	kunmap_local(old_elfcorehdr);
+	pr_debug("updated elfcorehdr\n");
+
+out:
+	vfree(elfbuf);
+}
+#endif