diff mbox series

[v10,064/108] KVM: TDX: Create initial guest memory

Message ID	2b04a33103e12b476a7f3547eb54abd16fb5d21a.1667110240.git.isaku.yamahata@intel.com
State	New
Headers	Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20; From: isaku.yamahata@intel.com To: kvm@vger.kernel.org, linux-kernel@vger.kernel.org Cc: isaku.yamahata@intel.com, isaku.yamahata@gmail.com, Paolo Bonzini <pbonzini@redhat.com>, erdemaktas@google.com, Sean Christopherson <seanjc@google.com>, Sagi Shahar <sagis@google.com>, David Matlack <dmatlack@google.com> Subject: [PATCH v10 064/108] KVM: TDX: Create initial guest memory Date: Sat, 29 Oct 2022 23:23:05 -0700 Message-Id: <2b04a33103e12b476a7f3547eb54abd16fb5d21a.1667110240.git.isaku.yamahata@intel.com> In-Reply-To: <cover.1667110240.git.isaku.yamahata@intel.com> References: <cover.1667110240.git.isaku.yamahata@intel.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Precedence: bulk
Series	KVM TDX basic feature support \| [v10,000/108] KVM TDX basic feature support [v10,001/108] KVM: VMX: Move out vmx_x86_ops to 'main.c' to wrap VMX and TDX [v10,002/108] KVM: x86: Refactor KVM VMX module init/exit functions [v10,003/108] KVM: TDX: Add placeholders for TDX VM/vcpu structure [v10,004/108] x86/virt/tdx: Add a helper function to return system wide info about TDX module [v10,005/108] KVM: TDX: Initialize the TDX module when loading the KVM intel kernel module [v10,006/108] KVM: x86: Introduce vm_type to differentiate default VMs from confidential VMs [v10,007/108] KVM: TDX: Make TDX VM type supported [v10,008/108,MARKER] The start of TDX KVM patch series: TDX architectural definitions [v10,009/108] KVM: TDX: Define TDX architectural definitions [v10,010/108] KVM: TDX: Add TDX "architectural" error codes [v10,011/108] KVM: TDX: Add C wrapper functions for SEAMCALLs to the TDX module [v10,012/108] KVM: TDX: Add helper functions to print TDX SEAMCALL error [v10,013/108,MARKER] The start of TDX KVM patch series: TD VM creation/destruction [v10,014/108] KVM: TDX: Stub in tdx.h with structs, accessors, and VMCS helpers [v10,015/108] x86/cpu: Add helper functions to allocate/free TDX private host key id [v10,016/108] KVM: TDX: create/destroy VM structure [v10,017/108] KVM: TDX: Refuse to unplug the last cpu on the package [v10,018/108] KVM: TDX: x86: Add ioctl to get TDX systemwide parameters [v10,019/108] KVM: TDX: Add place holder for TDX VM specific mem_enc_op ioctl [v10,020/108] KVM: Support KVM_CAP_MAX_VCPUS for KVM_ENABLE_CAP [v10,021/108] KVM: TDX: initialize VM with TDX specific parameters [v10,022/108] KVM: TDX: Make pmu_intel.c ignore guest TD case [v10,023/108,MARKER] The start of TDX KVM patch series: TD vcpu creation/destruction [v10,024/108] KVM: TDX: allocate/free TDX vcpu structure [v10,025/108] KVM: TDX: Do TDX specific vcpu initialization [v10,026/108] KVM: TDX: Use private memory for TDX [v10,027/108,MARKER] The start of TDX KVM patch series: KVM MMU GPA shared bits [v10,028/108] KVM: x86/mmu: introduce config for PRIVATE KVM MMU [v10,029/108] KVM: x86/mmu: Add address conversion functions for TDX shared bit of GPA [v10,030/108,MARKER] The start of TDX KVM patch series: KVM TDP refactoring for TDX [v10,031/108] KVM: x86/mmu: Replace hardcoded value 0 for the initial value for SPTE [v10,032/108] KVM: x86/mmu: Make sync_page not use hard-coded 0 as the initial SPTE value [v10,033/108] KVM: x86/mmu: Allow non-zero value for non-present SPTE and removed SPTE [v10,034/108] KVM: x86/mmu: Add Suppress VE bit to shadow_mmio_{value, mask} [v10,035/108] KVM: x86/mmu: Track shadow MMIO value on a per-VM basis [v10,036/108] KVM: TDX: Enable mmio spte caching always for TDX [v10,037/108] KVM: x86/mmu: Disallow fast page fault on private GPA [v10,038/108] KVM: x86/mmu: Allow per-VM override of the TDP max page level [v10,039/108] KVM: VMX: Introduce test mode related to EPT violation VE [v10,040/108,MARKER] The start of TDX KVM patch series: KVM TDP MMU hooks [v10,041/108] KVM: x86/tdp_mmu: refactor kvm_tdp_mmu_map() [v10,042/108] KVM: x86/tdp_mmu: Init role member of struct kvm_mmu_page at allocation [v10,043/108] KVM: x86/mmu: Require TDP MMU for TDX [v10,044/108] KVM: x86/mmu: Add a new is_private member for union kvm_mmu_page_role [v10,045/108] KVM: x86/mmu: Add a private pointer to struct kvm_mmu_page [v10,046/108] KVM: Add flags to struct kvm_gfn_range [v10,047/108] KVM: x86/tdp_mmu: Don't zap private pages for unsupported cases [v10,048/108] KVM: x86/tdp_mmu: Make handle_changed_spte() return value [v10,049/108] KVM: x86/tdp_mmu: Support TDX private mapping for TDP MMU [v10,050/108,MARKER] The start of TDX KVM patch series: TDX EPT violation [v10,051/108] KVM: x86/mmu: Disallow dirty logging for x86 TDX [v10,052/108] KVM: x86/tdp_mmu: Ignore unsupported mmu operation on private GFNs [v10,053/108] KVM: VMX: Split out guts of EPT violation to common/exposed function [v10,054/108] KVM: VMX: Move setting of EPT MMU masks to common VT-x code [v10,055/108] KVM: TDX: Add load_mmu_pgd method for TDX [v10,056/108] KVM: TDX: don't request KVM_REQ_APIC_PAGE_RELOAD [v10,057/108] KVM: x86/VMX: introduce vmx tlb_remote_flush and tlb_remote_flush_with_range [v10,058/108] KVM: TDX: TDP MMU TDX support [v10,059/108,MARKER] The start of TDX KVM patch series: KVM TDP MMU MapGPA [v10,060/108] KVM: Add functions to set GFN to private or shared [v10,061/108] KVM: x86/mmu: Introduce kvm_mmu_map_tdp_page() for use by TDX [v10,062/108] KVM: x86/tdp_mmu: implement MapGPA hypercall for TDX [v10,063/108,MARKER] The start of TDX KVM patch series: TD finalization [v10,064/108] KVM: TDX: Create initial guest memory [v10,065/108] KVM: TDX: Finalize VM initialization [v10,066/108,MARKER] The start of TDX KVM patch series: TD vcpu enter/exit [v10,067/108] KVM: TDX: Add helper assembly function to TDX vcpu [v10,068/108] KVM: TDX: Implement TDX vcpu enter/exit path [v10,069/108] KVM: TDX: vcpu_run: save/restore host state(host kernel gs) [v10,070/108] KVM: TDX: restore host xsave state when exit from the guest TD [v10,071/108] KVM: x86: Allow to update cached values in kvm_user_return_msrs w/o wrmsr [v10,072/108] KVM: TDX: restore user ret MSRs [v10,073/108,MARKER] The start of TDX KVM patch series: TD vcpu exits/interrupts/hypercalls [v10,074/108] KVM: TDX: complete interrupts after tdexit [v10,075/108] KVM: TDX: restore debug store when TD exit [v10,076/108] KVM: TDX: handle vcpu migration over logical processor [v10,077/108] KVM: x86: Add a switch_db_regs flag to handle TDX's auto-switched behavior [v10,078/108] KVM: TDX: Add support for find pending IRQ in a protected local APIC [v10,079/108] KVM: x86: Assume timer IRQ was injected if APIC state is proteced [v10,080/108] KVM: TDX: remove use of struct vcpu_vmx from posted_interrupt.c [v10,081/108] KVM: TDX: Implement interrupt injection [v10,082/108] KVM: TDX: Implements vcpu request_immediate_exit [v10,083/108] KVM: TDX: Implement methods to inject NMI [v10,084/108] KVM: VMX: Modify NMI and INTR handlers to take intr_info as function argument [v10,085/108] KVM: VMX: Move NMI/exception handler to common helper [v10,086/108] KVM: x86: Split core of hypercall emulation to helper function [v10,087/108] KVM: TDX: Add a place holder to handle TDX VM exit [v10,088/108] KVM: TDX: Retry seamcall when TDX_OPERAND_BUSY with operand SEPT [v10,089/108] KVM: TDX: handle EXIT_REASON_OTHER_SMI [v10,090/108] KVM: TDX: handle ept violation/misconfig exit [v10,091/108] KVM: TDX: handle EXCEPTION_NMI and EXTERNAL_INTERRUPT [v10,092/108] KVM: TDX: Add a place holder for handler of TDX hypercalls (TDG.VP.VMCALL) [v10,093/108] KVM: TDX: handle KVM hypercall with TDG.VP.VMCALL [v10,094/108] KVM: TDX: Handle TDX PV CPUID hypercall [v10,095/108] KVM: TDX: Handle TDX PV HLT hypercall [v10,096/108] KVM: TDX: Handle TDX PV port io hypercall [v10,097/108] KVM: TDX: Handle TDX PV MMIO hypercall [v10,098/108] KVM: TDX: Implement callbacks for MSR operations for TDX [v10,099/108] KVM: TDX: Handle TDX PV rdmsr/wrmsr hypercall [v10,100/108] KVM: TDX: Handle TDX PV report fatal error hypercall [v10,101/108] KVM: TDX: Handle TDX PV map_gpa hypercall [v10,102/108] KVM: TDX: Handle TDG.VP.VMCALL<GetTdVmCallInfo> hypercall [v10,103/108] KVM: TDX: Silently discard SMI request [v10,104/108] KVM: TDX: Silently ignore INIT/SIPI [v10,105/108] KVM: TDX: Add methods to ignore accesses to CPU state [v10,106/108] Documentation/virt/kvm: Document on Trust Domain Extensions(TDX) [v10,107/108] KVM: x86: design documentation on TDX support of x86 KVM TDP MMU [v10,108/108,MARKER] the end of (the first phase of) TDX KVM patch series

Commit Message

Isaku Yamahata Oct. 30, 2022, 6:23 a.m. UTC

  From: Isaku Yamahata <isaku.yamahata@intel.com>

Because the guest memory is protected in TDX, the creation of the initial
guest memory requires a dedicated TDX module API, tdh_mem_page_add, instead
of directly copying the memory contents into the guest memory in the case
of the default VM type.  KVM MMU page fault handler callback,
private_page_add, handles it.

Define new subcommand, KVM_TDX_INIT_MEM_REGION, of VM-scoped
KVM_MEMORY_ENCRYPT_OP.  It assigns the guest page, copies the initial
memory contents into the guest memory, encrypts the guest memory.  At the
same time, optionally it extends memory measurement of the TDX guest.  It
calls the KVM MMU page fault(EPT-violation) handler to trigger the
callbacks for it.

Signed-off-by: Isaku Yamahata <isaku.yamahata@intel.com>
---
 arch/x86/include/uapi/asm/kvm.h       |   9 ++
 arch/x86/kvm/mmu/mmu.c                |   1 +
 arch/x86/kvm/vmx/tdx.c                | 158 +++++++++++++++++++++++++-
 arch/x86/kvm/vmx/tdx.h                |   2 +
 tools/arch/x86/include/uapi/asm/kvm.h |   9 ++
 5 files changed, 174 insertions(+), 5 deletions(-)

diff mbox series

Patch

diff --git a/arch/x86/include/uapi/asm/kvm.h b/arch/x86/include/uapi/asm/kvm.h
index 80db152430e4..6ae52926e05a 100644
--- a/arch/x86/include/uapi/asm/kvm.h
+++ b/arch/x86/include/uapi/asm/kvm.h
@@ -540,6 +540,7 @@  enum kvm_tdx_cmd_id {
 	KVM_TDX_CAPABILITIES = 0,
 	KVM_TDX_INIT_VM,
 	KVM_TDX_INIT_VCPU,
+	KVM_TDX_INIT_MEM_REGION,
 
 	KVM_TDX_CMD_NR_MAX,
 };
@@ -615,4 +616,12 @@  struct kvm_tdx_init_vm {
 	};
 };
 
+#define KVM_TDX_MEASURE_MEMORY_REGION	(1UL << 0)
+
+struct kvm_tdx_init_mem_region {
+	__u64 source_addr;
+	__u64 gpa;
+	__u64 nr_pages;
+};
+
 #endif /* _ASM_X86_KVM_H */
diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 37b378bf60df..8e24dd0e3c3c 100644
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -5492,6 +5492,7 @@  int kvm_mmu_load(struct kvm_vcpu *vcpu)
 out:
 	return r;
 }
+EXPORT_SYMBOL(kvm_mmu_load);
 
 void kvm_mmu_unload(struct kvm_vcpu *vcpu)
 {
diff --git a/arch/x86/kvm/vmx/tdx.c b/arch/x86/kvm/vmx/tdx.c
index 5378d2c35e27..7c00f71d42af 100644
--- a/arch/x86/kvm/vmx/tdx.c
+++ b/arch/x86/kvm/vmx/tdx.c
@@ -417,6 +417,21 @@  void tdx_load_mmu_pgd(struct kvm_vcpu *vcpu, hpa_t root_hpa, int pgd_level)
 	td_vmcs_write64(to_tdx(vcpu), SHARED_EPT_POINTER, root_hpa & PAGE_MASK);
 }
 
+static void tdx_measure_page(struct kvm_tdx *kvm_tdx, hpa_t gpa)
+{
+	struct tdx_module_output out;
+	u64 err;
+	int i;
+
+	for (i = 0; i < PAGE_SIZE; i += TDX_EXTENDMR_CHUNKSIZE) {
+		err = tdh_mr_extend(kvm_tdx->tdr.pa, gpa + i, &out);
+		if (KVM_BUG_ON(err, &kvm_tdx->kvm)) {
+			pr_tdx_error(TDH_MR_EXTEND, err, &out);
+			break;
+		}
+	}
+}
+
 static void tdx_unpin(struct kvm *kvm, kvm_pfn_t pfn)
 {
 	struct page *page = pfn_to_page(pfn);
@@ -431,20 +446,23 @@  static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn,
 	hpa_t hpa = pfn_to_hpa(pfn);
 	gpa_t gpa = gfn_to_gpa(gfn);
 	struct tdx_module_output out;
+	hpa_t source_pa;
+	bool measure;
 	u64 err;
 
 	if (WARN_ON_ONCE(is_error_noslot_pfn(pfn) ||
 			 !kvm_pfn_to_refcounted_page(pfn)))
 		return 0;
 
-	/* TODO: handle large pages. */
-	if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
-		return -EINVAL;
-
 	/* To prevent page migration, do nothing on mmu notifier. */
 	get_page(pfn_to_page(pfn));
 
+	/* Build-time faults are induced and handled via TDH_MEM_PAGE_ADD. */
 	if (likely(is_td_finalized(kvm_tdx))) {
+		/* TODO: handle large pages. */
+		if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
+			return -EINVAL;
+
 		err = tdh_mem_page_aug(kvm_tdx->tdr.pa, gpa, hpa, &out);
 		if (err == TDX_ERROR_SEPT_BUSY) {
 			tdx_unpin(kvm, pfn);
@@ -453,11 +471,50 @@  static int tdx_sept_set_private_spte(struct kvm *kvm, gfn_t gfn,
 		if (KVM_BUG_ON(err, kvm)) {
 			pr_tdx_error(TDH_MEM_PAGE_AUG, err, &out);
 			tdx_unpin(kvm, pfn);
+			return -EIO;
 		}
 		return 0;
 	}
 
-	/* TODO: tdh_mem_page_add() comes here */
+	/*
+	 * KVM_INIT_MEM_REGION, tdx_init_mem_region(), supports only 4K page
+	 * because tdh_mem_page_add() supports only 4K page.
+	 */
+	if (KVM_BUG_ON(level != PG_LEVEL_4K, kvm))
+		return -EINVAL;
+
+	/*
+	 * In case of TDP MMU, fault handler can run concurrently.  Note
+	 * 'source_pa' is a TD scope variable, meaning if there are multiple
+	 * threads reaching here with all needing to access 'source_pa', it
+	 * will break.  However fortunately this won't happen, because below
+	 * TDH_MEM_PAGE_ADD code path is only used when VM is being created
+	 * before it is running, using KVM_TDX_INIT_MEM_REGION ioctl (which
+	 * always uses vcpu 0's page table and protected by vcpu->mutex).
+	 */
+	if (KVM_BUG_ON(kvm_tdx->source_pa == INVALID_PAGE, kvm)) {
+		tdx_unpin(kvm, pfn);
+		return -EINVAL;
+	}
+
+	source_pa = kvm_tdx->source_pa & ~KVM_TDX_MEASURE_MEMORY_REGION;
+	measure = kvm_tdx->source_pa & KVM_TDX_MEASURE_MEMORY_REGION;
+	kvm_tdx->source_pa = INVALID_PAGE;
+
+	do {
+		err = tdh_mem_page_add(kvm_tdx->tdr.pa, gpa, hpa, source_pa,
+				       &out);
+		/*
+		 * This path is executed during populating initial guest memory
+		 * image. i.e. before running any vcpu.  Race is rare.
+		 */
+	} while (err == TDX_ERROR_SEPT_BUSY);
+	if (KVM_BUG_ON(err, kvm)) {
+		pr_tdx_error(TDH_MEM_PAGE_ADD, err, &out);
+		tdx_unpin(kvm, pfn);
+		return -EIO;
+	} else if (measure)
+		tdx_measure_page(kvm_tdx, gpa);
 
 	return 0;
 }
@@ -1091,6 +1148,94 @@  void tdx_flush_tlb(struct kvm_vcpu *vcpu)
 		cpu_relax();
 }
 
+#define TDX_SEPT_PFERR	PFERR_WRITE_MASK
+
+static int tdx_init_mem_region(struct kvm *kvm, struct kvm_tdx_cmd *cmd)
+{
+	struct kvm_tdx *kvm_tdx = to_kvm_tdx(kvm);
+	struct kvm_tdx_init_mem_region region;
+	struct kvm_vcpu *vcpu;
+	struct page *page;
+	kvm_pfn_t pfn;
+	int idx, ret = 0;
+
+	/* The BSP vCPU must be created before initializing memory regions. */
+	if (!atomic_read(&kvm->online_vcpus))
+		return -EINVAL;
+
+	if (cmd->flags & ~KVM_TDX_MEASURE_MEMORY_REGION)
+		return -EINVAL;
+
+	if (copy_from_user(&region, (void __user *)cmd->data, sizeof(region)))
+		return -EFAULT;
+
+	/* Sanity check */
+	if (!IS_ALIGNED(region.source_addr, PAGE_SIZE) ||
+	    !IS_ALIGNED(region.gpa, PAGE_SIZE) ||
+	    !region.nr_pages ||
+	    region.gpa + (region.nr_pages << PAGE_SHIFT) <= region.gpa ||
+	    !kvm_is_private_gpa(kvm, region.gpa) ||
+	    !kvm_is_private_gpa(kvm, region.gpa + (region.nr_pages << PAGE_SHIFT)))
+		return -EINVAL;
+
+	vcpu = kvm_get_vcpu(kvm, 0);
+	if (mutex_lock_killable(&vcpu->mutex))
+		return -EINTR;
+
+	vcpu_load(vcpu);
+	idx = srcu_read_lock(&kvm->srcu);
+
+	kvm_mmu_reload(vcpu);
+
+	while (region.nr_pages) {
+		if (signal_pending(current)) {
+			ret = -ERESTARTSYS;
+			break;
+		}
+
+		if (need_resched())
+			cond_resched();
+
+
+		/* Pin the source page. */
+		ret = get_user_pages_fast(region.source_addr, 1, 0, &page);
+		if (ret < 0)
+			break;
+		if (ret != 1) {
+			ret = -ENOMEM;
+			break;
+		}
+
+		kvm_tdx->source_pa = pfn_to_hpa(page_to_pfn(page)) |
+				     (cmd->flags & KVM_TDX_MEASURE_MEMORY_REGION);
+
+		pfn = kvm_mmu_map_tdp_page(vcpu, region.gpa, TDX_SEPT_PFERR,
+					   PG_LEVEL_4K);
+		if (is_error_noslot_pfn(pfn) || kvm->vm_bugged)
+			ret = -EFAULT;
+		else
+			ret = 0;
+
+		put_page(page);
+		if (ret)
+			break;
+
+		region.source_addr += PAGE_SIZE;
+		region.gpa += PAGE_SIZE;
+		region.nr_pages--;
+	}
+
+	srcu_read_unlock(&kvm->srcu, idx);
+	vcpu_put(vcpu);
+
+	mutex_unlock(&vcpu->mutex);
+
+	if (copy_to_user((void __user *)cmd->data, &region, sizeof(region)))
+		ret = -EFAULT;
+
+	return ret;
+}
+
 int tdx_vm_ioctl(struct kvm *kvm, void __user *argp)
 {
 	struct kvm_tdx_cmd tdx_cmd;
@@ -1107,6 +1252,9 @@  int tdx_vm_ioctl(struct kvm *kvm, void __user *argp)
 	case KVM_TDX_INIT_VM:
 		r = tdx_td_init(kvm, &tdx_cmd);
 		break;
+	case KVM_TDX_INIT_MEM_REGION:
+		r = tdx_init_mem_region(kvm, &tdx_cmd);
+		break;
 	default:
 		r = -EINVAL;
 		goto out;
diff --git a/arch/x86/kvm/vmx/tdx.h b/arch/x86/kvm/vmx/tdx.h
index 80d595c5f96f..686da2321683 100644
--- a/arch/x86/kvm/vmx/tdx.h
+++ b/arch/x86/kvm/vmx/tdx.h
@@ -23,6 +23,8 @@  struct kvm_tdx {
 	u64 xfam;
 	int hkid;
 
+	hpa_t source_pa;
+
 	bool finalized;
 	atomic_t tdh_mem_track;
 
diff --git a/tools/arch/x86/include/uapi/asm/kvm.h b/tools/arch/x86/include/uapi/asm/kvm.h
index 35e3b4aa2e96..37e713ffab72 100644
--- a/tools/arch/x86/include/uapi/asm/kvm.h
+++ b/tools/arch/x86/include/uapi/asm/kvm.h
@@ -540,6 +540,7 @@  enum kvm_tdx_cmd_id {
 	KVM_TDX_CAPABILITIES = 0,
 	KVM_TDX_INIT_VM,
 	KVM_TDX_INIT_VCPU,
+	KVM_TDX_INIT_MEM_REGION,
 
 	KVM_TDX_CMD_NR_MAX,
 };
@@ -617,4 +618,12 @@  struct kvm_tdx_init_vm {
 	};
 };
 
+#define KVM_TDX_MEASURE_MEMORY_REGION	(1UL << 0)
+
+struct kvm_tdx_init_mem_region {
+	__u64 source_addr;
+	__u64 gpa;
+	__u64 nr_pages;
+};
+
 #endif /* _ASM_X86_KVM_H */