[V3,7/9] x86/hyperv: Initialize cpu and memory for SEV-SNP enlightened guest

  From: Tianyu Lan <tiala@microsoft.com>

Hyper-V enlightened guest doesn't have boot loader support.
Boot Linux kernel directly from hypervisor with data (kernel
image, initrd and parameter page) and memory for boot up that
is initialized via AMD SEV PSP protocol (Please reference
Section 4.5 Launching a Guest of [1]).

Kernel needs to read processor and memory info from EN_SEV_
SNP_PROCESSOR/MEM_INFO_ADDR address which are populated by
Hyper-V. The data is prepared by hypervisor via SNP_
LAUNCH_UPDATE with page type SNP_PAGE_TYPE_UNMEASURED and
Initialize smp cpu related ops, validate system memory and
add them into e820 table.

[1]: https://www.amd.com/system/files/TechDocs/56860.pdf
Reviewed-by: Michael Kelley <mikelley@microsoft.com>
Signed-off-by: Tianyu Lan <tiala@microsoft.com>
---
Change since v2:
	* Update change log.
---
 arch/x86/hyperv/ivm.c           | 93 +++++++++++++++++++++++++++++++++
 arch/x86/include/asm/mshyperv.h | 17 ++++++
 arch/x86/kernel/cpu/mshyperv.c  |  3 ++
 3 files changed, 113 insertions(+)
  

Hi Tianyu,

On 7/18/2023 8:53 AM, Tianyu Lan wrote:
> From: Tianyu Lan <tiala@microsoft.com>
>
> Hyper-V enlightened guest doesn't have boot loader support.
> Boot Linux kernel directly from hypervisor with data (kernel
> image, initrd and parameter page) and memory for boot up that
> is initialized via AMD SEV PSP protocol (Please reference
> Section 4.5 Launching a Guest of [1]).
>
> Kernel needs to read processor and memory info from EN_SEV_
> SNP_PROCESSOR/MEM_INFO_ADDR address which are populated by
> Hyper-V. The data is prepared by hypervisor via SNP_
> LAUNCH_UPDATE with page type SNP_PAGE_TYPE_UNMEASURED and
> Initialize smp cpu related ops, validate system memory and
> add them into e820 table.
>
> [1]: https://www.amd.com/system/files/TechDocs/56860.pdf
> Reviewed-by: Michael Kelley <mikelley@microsoft.com>
> Signed-off-by: Tianyu Lan <tiala@microsoft.com>
> ---
> Change since v2:
> 	* Update change log.
> ---
>   arch/x86/hyperv/ivm.c           | 93 +++++++++++++++++++++++++++++++++
>   arch/x86/include/asm/mshyperv.h | 17 ++++++
>   arch/x86/kernel/cpu/mshyperv.c  |  3 ++
>   3 files changed, 113 insertions(+)
>
> diff --git a/arch/x86/hyperv/ivm.c b/arch/x86/hyperv/ivm.c
> index b2b5cb19fac9..ede47c8264e0 100644
> --- a/arch/x86/hyperv/ivm.c
> +++ b/arch/x86/hyperv/ivm.c
> @@ -18,6 +18,11 @@
>   #include <asm/mshyperv.h>
>   #include <asm/hypervisor.h>
>   #include <asm/mtrr.h>
> +#include <asm/coco.h>
> +#include <asm/io_apic.h>
> +#include <asm/sev.h>
> +#include <asm/realmode.h>
> +#include <asm/e820/api.h>
>   
>   #ifdef CONFIG_AMD_MEM_ENCRYPT
>   
> @@ -58,6 +63,8 @@ union hv_ghcb {
>   
>   static u16 hv_ghcb_version __ro_after_init;
>   
> +static u32 processor_count;
> +
>   u64 hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size)
>   {
>   	union hv_ghcb *hv_ghcb;
> @@ -357,6 +364,92 @@ static bool hv_is_private_mmio(u64 addr)
>   	return false;
>   }
>   
> +static __init void hv_snp_get_smp_config(unsigned int early)
> +{
> +	/*
> +	 * The "early" parameter can be true only if old-style AMD
> +	 * Opteron NUMA detection is enabled, which should never be
> +	 * the case for an SEV-SNP guest.  See CONFIG_AMD_NUMA.
> +	 * For safety, just do nothing if "early" is true.
> +	 */
> +	if (early)
> +		return;
> +
> +	/*
> +	 * There is no firmware and ACPI MADT table support in
> +	 * in the Hyper-V SEV-SNP enlightened guest. Set smp
> +	 * related config variable here.
> +	 */
> +	while (num_processors < processor_count) {
> +		early_per_cpu(x86_cpu_to_apicid, num_processors) = num_processors;
> +		early_per_cpu(x86_bios_cpu_apicid, num_processors) = num_processors;
> +		physid_set(num_processors, phys_cpu_present_map);
> +		set_cpu_possible(num_processors, true);
> +		set_cpu_present(num_processors, true);
> +		num_processors++;
> +	}
> +}
> +
> +__init void hv_sev_init_mem_and_cpu(void)
> +{
> +	struct memory_map_entry *entry;
> +	struct e820_entry *e820_entry;
> +	u64 e820_end;
> +	u64 ram_end;
> +	u64 page;
> +
> +	/*
> +	 * Hyper-V enlightened snp guest boots kernel
> +	 * directly without bootloader. So roms, bios
> +	 * regions and reserve resources are not available.
> +	 * Set these callback to NULL.
> +	 */
> +	x86_platform.legacy.rtc			= 0;
> +	x86_platform.legacy.reserve_bios_regions = 0;
> +	x86_platform.set_wallclock		= set_rtc_noop;
> +	x86_platform.get_wallclock		= get_rtc_noop;
> +	x86_init.resources.probe_roms		= x86_init_noop;
> +	x86_init.resources.reserve_resources	= x86_init_noop;
> +	x86_init.mpparse.find_smp_config	= x86_init_noop;
> +	x86_init.mpparse.get_smp_config		= hv_snp_get_smp_config;
> +
> +	/*
> +	 * Hyper-V SEV-SNP enlightened guest doesn't support ioapic
> +	 * and legacy APIC page read/write. Switch to hv apic here.
> +	 */
> +	disable_ioapic_support();

Where are we switching hv_apic? May I am missing something here?

Also in my experiments I have seen that if we don't enable I/O Apic 
legacy serial console does not seem to work for SEV-SNP guests.

> +
> +	/* Get processor and mem info. */
> +	processor_count = *(u32 *)__va(EN_SEV_SNP_PROCESSOR_INFO_ADDR);
> +	entry = (struct memory_map_entry *)__va(EN_SEV_SNP_MEM_INFO_ADDR);
> +
> +	/*
> +	 * There is no bootloader/EFI firmware in the SEV SNP guest.
> +	 * E820 table in the memory just describes memory for kernel,
> +	 * ACPI table, cmdline, boot params and ramdisk. The dynamic
> +	 * data(e.g, vcpu number and the rest memory layout) needs to
> +	 * be read from EN_SEV_SNP_PROCESSOR_INFO_ADDR.
> +	 */
> +	for (; entry->numpages != 0; entry++) {
> +		e820_entry = &e820_table->entries[
> +				e820_table->nr_entries - 1];
> +		e820_end = e820_entry->addr + e820_entry->size;
> +		ram_end = (entry->starting_gpn +
> +			   entry->numpages) * PAGE_SIZE;
> +
> +		if (e820_end < entry->starting_gpn * PAGE_SIZE)
> +			e820_end = entry->starting_gpn * PAGE_SIZE;
> +
> +		if (e820_end < ram_end) {
> +			pr_info("Hyper-V: add e820 entry [mem %#018Lx-%#018Lx]\n", e820_end, ram_end - 1);
> +			e820__range_add(e820_end, ram_end - e820_end,
> +					E820_TYPE_RAM);
> +			for (page = e820_end; page < ram_end; page += PAGE_SIZE)
> +				pvalidate((unsigned long)__va(page), RMP_PG_SIZE_4K, true);
> +		}
> +	}
> +}
> +
>   void __init hv_vtom_init(void)
>   {
>   	/*
> diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
> index 025eda129d99..e57df590846a 100644
> --- a/arch/x86/include/asm/mshyperv.h
> +++ b/arch/x86/include/asm/mshyperv.h
> @@ -50,6 +50,21 @@ extern bool hv_isolation_type_en_snp(void);
>   
>   extern union hv_ghcb * __percpu *hv_ghcb_pg;
>   
> +/*
> + * Hyper-V puts processor and memory layout info
> + * to this address in SEV-SNP enlightened guest.
> + */
> +#define EN_SEV_SNP_PROCESSOR_INFO_ADDR  0x802000
> +#define EN_SEV_SNP_MEM_INFO_ADDR	0x802018
> +
> +struct memory_map_entry {
> +	u64 starting_gpn;
> +	u64 numpages;
> +	u16 type;
> +	u16 flags;
> +	u32 reserved;
> +};
> +
>   int hv_call_deposit_pages(int node, u64 partition_id, u32 num_pages);
>   int hv_call_add_logical_proc(int node, u32 lp_index, u32 acpi_id);
>   int hv_call_create_vp(int node, u64 partition_id, u32 vp_index, u32 flags);
> @@ -234,12 +249,14 @@ void hv_ghcb_msr_read(u64 msr, u64 *value);
>   bool hv_ghcb_negotiate_protocol(void);
>   void __noreturn hv_ghcb_terminate(unsigned int set, unsigned int reason);
>   void hv_vtom_init(void);
> +void hv_sev_init_mem_and_cpu(void);
>   #else
>   static inline void hv_ghcb_msr_write(u64 msr, u64 value) {}
>   static inline void hv_ghcb_msr_read(u64 msr, u64 *value) {}
>   static inline bool hv_ghcb_negotiate_protocol(void) { return false; }
>   static inline void hv_ghcb_terminate(unsigned int set, unsigned int reason) {}
>   static inline void hv_vtom_init(void) {}
> +static inline void hv_sev_init_mem_and_cpu(void) {}
>   #endif
>   
>   extern bool hv_isolation_type_snp(void);
> diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
> index 5398fb2f4d39..d3bb921ee7fe 100644
> --- a/arch/x86/kernel/cpu/mshyperv.c
> +++ b/arch/x86/kernel/cpu/mshyperv.c
> @@ -529,6 +529,9 @@ static void __init ms_hyperv_init_platform(void)
>   	if (!(ms_hyperv.features & HV_ACCESS_TSC_INVARIANT))
>   		mark_tsc_unstable("running on Hyper-V");
>   
> +	if (hv_isolation_type_en_snp())
> +		hv_sev_init_mem_and_cpu();
> +
>   	hardlockup_detector_disable();
>   }
>   

Regards,

Jinank
  

On 7/26/2023 12:26 PM, Jinank Jain wrote:
>> +    /*
>> +     * Hyper-V SEV-SNP enlightened guest doesn't support ioapic
>> +     * and legacy APIC page read/write. Switch to hv apic here.
>> +     */
>> +    disable_ioapic_support();
> 
> Where are we switching hv_apic? May I am missing something here?
>

Nice catch! It's fossil comment when there is no x2apic support.
Will fix it in the next version.