@@ -1,5 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
-gunyah_rsc_mgr-y += rsc_mgr.o rsc_mgr_rpc.o vm_mgr.o
+gunyah_rsc_mgr-y += rsc_mgr.o rsc_mgr_rpc.o vm_mgr.o vm_mgr_mem.o
obj-$(CONFIG_GUNYAH) += gunyah.o gunyah_rsc_mgr.o gunyah_vcpu.o
@@ -17,6 +17,16 @@
#include "rsc_mgr.h"
#include "vm_mgr.h"
+#define GUNYAH_VM_ADDRSPACE_LABEL 0
+// "To" extent for memory private to guest
+#define GUNYAH_VM_MEM_EXTENT_GUEST_PRIVATE_LABEL 0
+// "From" extent for memory shared with guest
+#define GUNYAH_VM_MEM_EXTENT_HOST_SHARED_LABEL 1
+// "To" extent for memory shared with the guest
+#define GUNYAH_VM_MEM_EXTENT_GUEST_SHARED_LABEL 3
+// "From" extent for memory private to guest
+#define GUNYAH_VM_MEM_EXTENT_HOST_PRIVATE_LABEL 2
+
static DEFINE_XARRAY(gunyah_vm_functions);
static void gunyah_vm_put_function(struct gunyah_vm_function *fn)
@@ -175,6 +185,16 @@ void gunyah_vm_function_unregister(struct gunyah_vm_function *fn)
}
EXPORT_SYMBOL_GPL(gunyah_vm_function_unregister);
+static bool gunyah_vm_resource_ticket_populate_noop(
+ struct gunyah_vm_resource_ticket *ticket, struct gunyah_resource *ghrsc)
+{
+ return true;
+}
+static void gunyah_vm_resource_ticket_unpopulate_noop(
+ struct gunyah_vm_resource_ticket *ticket, struct gunyah_resource *ghrsc)
+{
+}
+
int gunyah_vm_add_resource_ticket(struct gunyah_vm *ghvm,
struct gunyah_vm_resource_ticket *ticket)
{
@@ -342,6 +362,17 @@ static void gunyah_vm_stop(struct gunyah_vm *ghvm)
ghvm->vm_status != GUNYAH_RM_VM_STATUS_RUNNING);
}
+static inline void setup_extent_ticket(struct gunyah_vm *ghvm,
+ struct gunyah_vm_resource_ticket *ticket,
+ u32 label)
+{
+ ticket->resource_type = GUNYAH_RESOURCE_TYPE_MEM_EXTENT;
+ ticket->label = label;
+ ticket->populate = gunyah_vm_resource_ticket_populate_noop;
+ ticket->unpopulate = gunyah_vm_resource_ticket_unpopulate_noop;
+ gunyah_vm_add_resource_ticket(ghvm, ticket);
+}
+
static __must_check struct gunyah_vm *gunyah_vm_alloc(struct gunyah_rm *rm)
{
struct gunyah_vm *ghvm;
@@ -365,6 +396,25 @@ static __must_check struct gunyah_vm *gunyah_vm_alloc(struct gunyah_rm *rm)
INIT_LIST_HEAD(&ghvm->resources);
INIT_LIST_HEAD(&ghvm->resource_tickets);
+ mt_init(&ghvm->mm);
+
+ ghvm->addrspace_ticket.resource_type = GUNYAH_RESOURCE_TYPE_ADDR_SPACE;
+ ghvm->addrspace_ticket.label = GUNYAH_VM_ADDRSPACE_LABEL;
+ ghvm->addrspace_ticket.populate =
+ gunyah_vm_resource_ticket_populate_noop;
+ ghvm->addrspace_ticket.unpopulate =
+ gunyah_vm_resource_ticket_unpopulate_noop;
+ gunyah_vm_add_resource_ticket(ghvm, &ghvm->addrspace_ticket);
+
+ setup_extent_ticket(ghvm, &ghvm->host_private_extent_ticket,
+ GUNYAH_VM_MEM_EXTENT_HOST_PRIVATE_LABEL);
+ setup_extent_ticket(ghvm, &ghvm->host_shared_extent_ticket,
+ GUNYAH_VM_MEM_EXTENT_HOST_SHARED_LABEL);
+ setup_extent_ticket(ghvm, &ghvm->guest_private_extent_ticket,
+ GUNYAH_VM_MEM_EXTENT_GUEST_PRIVATE_LABEL);
+ setup_extent_ticket(ghvm, &ghvm->guest_shared_extent_ticket,
+ GUNYAH_VM_MEM_EXTENT_GUEST_SHARED_LABEL);
+
return ghvm;
}
@@ -528,6 +578,21 @@ static void _gunyah_vm_put(struct kref *kref)
gunyah_vm_stop(ghvm);
gunyah_vm_remove_functions(ghvm);
+
+ /**
+ * If this fails, we're going to lose the memory for good and is
+ * BUG_ON-worthy, but not unrecoverable (we just lose memory).
+ * This call should always succeed though because the VM is in not
+ * running and RM will let us reclaim all the memory.
+ */
+ WARN_ON(gunyah_vm_reclaim_range(ghvm, 0, U64_MAX));
+
+ gunyah_vm_remove_resource_ticket(ghvm, &ghvm->addrspace_ticket);
+ gunyah_vm_remove_resource_ticket(ghvm, &ghvm->host_shared_extent_ticket);
+ gunyah_vm_remove_resource_ticket(ghvm, &ghvm->host_private_extent_ticket);
+ gunyah_vm_remove_resource_ticket(ghvm, &ghvm->guest_shared_extent_ticket);
+ gunyah_vm_remove_resource_ticket(ghvm, &ghvm->guest_private_extent_ticket);
+
gunyah_vm_clean_resources(ghvm);
if (ghvm->vm_status != GUNYAH_RM_VM_STATUS_NO_STATE &&
@@ -541,6 +606,8 @@ static void _gunyah_vm_put(struct kref *kref)
ghvm->vm_status == GUNYAH_RM_VM_STATUS_RESET);
}
+ mtree_destroy(&ghvm->mm);
+
if (ghvm->vm_status > GUNYAH_RM_VM_STATUS_NO_STATE) {
gunyah_rm_notifier_unregister(ghvm->rm, &ghvm->nb);
@@ -8,6 +8,7 @@
#include <linux/device.h>
#include <linux/kref.h>
+#include <linux/maple_tree.h>
#include <linux/mutex.h>
#include <linux/rwsem.h>
#include <linux/wait.h>
@@ -16,12 +17,42 @@
#include "rsc_mgr.h"
+static inline u64 gunyah_gpa_to_gfn(u64 gpa)
+{
+ return gpa >> PAGE_SHIFT;
+}
+
+static inline u64 gunyah_gfn_to_gpa(u64 gfn)
+{
+ return gfn << PAGE_SHIFT;
+}
+
long gunyah_dev_vm_mgr_ioctl(struct gunyah_rm *rm, unsigned int cmd,
unsigned long arg);
/**
* struct gunyah_vm - Main representation of a Gunyah Virtual machine
* @vmid: Gunyah's VMID for this virtual machine
+ * @mm: A maple tree of all memory that has been mapped to a VM.
+ * Indices are guest frame numbers; entries are either folios or
+ * RM mem parcels
+ * @addrspace_ticket: Resource ticket to the capability for guest VM's
+ * address space
+ * @host_private_extent_ticket: Resource ticket to the capability for our
+ * memory extent from which to lend private
+ * memory to the guest
+ * @host_shared_extent_ticket: Resource ticket to the capaiblity for our
+ * memory extent from which to share memory
+ * with the guest. Distinction with
+ * @host_private_extent_ticket needed for
+ * current Qualcomm platforms; on non-Qualcomm
+ * platforms, this is the same capability ID
+ * @guest_private_extent_ticket: Resource ticket to the capaiblity for
+ * the guest's memory extent to lend private
+ * memory to
+ * @guest_shared_extent_ticket: Resource ticket to the capability for
+ * the memory extent that represents
+ * memory shared with the guest.
* @rm: Pointer to the resource manager struct to make RM calls
* @parent: For logging
* @nb: Notifier block for RM notifications
@@ -43,6 +74,11 @@ long gunyah_dev_vm_mgr_ioctl(struct gunyah_rm *rm, unsigned int cmd,
*/
struct gunyah_vm {
u16 vmid;
+ struct maple_tree mm;
+ struct gunyah_vm_resource_ticket addrspace_ticket,
+ host_private_extent_ticket, host_shared_extent_ticket,
+ guest_private_extent_ticket, guest_shared_extent_ticket;
+
struct gunyah_rm *rm;
struct notifier_block nb;
@@ -63,4 +99,14 @@ struct gunyah_vm {
};
+int gunyah_vm_parcel_to_paged(struct gunyah_vm *ghvm,
+ struct gunyah_rm_mem_parcel *parcel, u64 gfn,
+ u64 nr);
+int gunyah_vm_reclaim_parcel(struct gunyah_vm *ghvm,
+ struct gunyah_rm_mem_parcel *parcel, u64 gfn);
+int gunyah_vm_provide_folio(struct gunyah_vm *ghvm, struct folio *folio,
+ u64 gfn, bool share, bool write);
+int gunyah_vm_reclaim_folio(struct gunyah_vm *ghvm, u64 gfn);
+int gunyah_vm_reclaim_range(struct gunyah_vm *ghvm, u64 gfn, u64 nr);
+
#endif
new file mode 100644
@@ -0,0 +1,309 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2023-2024 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#define pr_fmt(fmt) "gunyah_vm_mgr: " fmt
+
+#include <asm/gunyah.h>
+#include <linux/mm.h>
+#include <linux/pagemap.h>
+
+#include "vm_mgr.h"
+
+#define WRITE_TAG (1 << 0)
+#define SHARE_TAG (1 << 1)
+
+static inline struct gunyah_resource *
+__first_resource(struct gunyah_vm_resource_ticket *ticket)
+{
+ return list_first_entry_or_null(&ticket->resources,
+ struct gunyah_resource, list);
+}
+
+int gunyah_vm_parcel_to_paged(struct gunyah_vm *ghvm,
+ struct gunyah_rm_mem_parcel *parcel, u64 gfn,
+ u64 nr)
+{
+ struct gunyah_rm_mem_entry *entry;
+ unsigned long i, entry_size, tag = 0;
+ struct folio *folio;
+ pgoff_t off = 0;
+ int ret;
+
+ if (parcel->n_acl_entries > 1)
+ tag |= SHARE_TAG;
+ if (parcel->acl_entries[0].perms & GUNYAH_RM_ACL_W)
+ tag |= WRITE_TAG;
+
+ for (i = 0; i < parcel->n_mem_entries; i++) {
+ entry = &parcel->mem_entries[i];
+ entry_size = PHYS_PFN(le64_to_cpu(entry->size));
+
+ folio = pfn_folio(PHYS_PFN(le64_to_cpu(entry->phys_addr)));
+ ret = mtree_insert_range(&ghvm->mm, gfn + off, gfn + off + folio_nr_pages(folio) - 1, xa_tag_pointer(folio, tag), GFP_KERNEL);
+ if (ret == -ENOMEM)
+ return ret;
+ BUG_ON(ret);
+ off += folio_nr_pages(folio);
+ }
+
+ BUG_ON(off != nr);
+
+ return 0;
+}
+
+static inline u32 donate_flags(bool share)
+{
+ if (share)
+ return FIELD_PREP_CONST(GUNYAH_MEMEXTENT_OPTION_TYPE_MASK,
+ GUNYAH_MEMEXTENT_DONATE_TO_SIBLING);
+ else
+ return FIELD_PREP_CONST(GUNYAH_MEMEXTENT_OPTION_TYPE_MASK,
+ GUNYAH_MEMEXTENT_DONATE_TO_PROTECTED);
+}
+
+static inline u32 reclaim_flags(bool share)
+{
+ if (share)
+ return FIELD_PREP_CONST(GUNYAH_MEMEXTENT_OPTION_TYPE_MASK,
+ GUNYAH_MEMEXTENT_DONATE_TO_SIBLING);
+ else
+ return FIELD_PREP_CONST(GUNYAH_MEMEXTENT_OPTION_TYPE_MASK,
+ GUNYAH_MEMEXTENT_DONATE_FROM_PROTECTED);
+}
+
+int gunyah_vm_provide_folio(struct gunyah_vm *ghvm, struct folio *folio,
+ u64 gfn, bool share, bool write)
+{
+ struct gunyah_resource *guest_extent, *host_extent, *addrspace;
+ u32 map_flags = BIT(GUNYAH_ADDRSPACE_MAP_FLAG_PARTIAL);
+ u64 extent_attrs, gpa = gunyah_gfn_to_gpa(gfn);
+ phys_addr_t pa = PFN_PHYS(folio_pfn(folio));
+ enum gunyah_pagetable_access access;
+ size_t size = folio_size(folio);
+ enum gunyah_error gunyah_error;
+ unsigned long tag = 0;
+ int ret;
+
+ /* clang-format off */
+ if (share) {
+ guest_extent = __first_resource(&ghvm->guest_shared_extent_ticket);
+ host_extent = __first_resource(&ghvm->host_shared_extent_ticket);
+ } else {
+ guest_extent = __first_resource(&ghvm->guest_private_extent_ticket);
+ host_extent = __first_resource(&ghvm->host_private_extent_ticket);
+ }
+ /* clang-format on */
+ addrspace = __first_resource(&ghvm->addrspace_ticket);
+
+ if (!addrspace || !guest_extent || !host_extent)
+ return -ENODEV;
+
+ if (share) {
+ map_flags |= BIT(GUNYAH_ADDRSPACE_MAP_FLAG_VMMIO);
+ tag |= SHARE_TAG;
+ } else {
+ map_flags |= BIT(GUNYAH_ADDRSPACE_MAP_FLAG_PRIVATE);
+ }
+
+ if (write)
+ tag |= WRITE_TAG;
+
+ ret = mtree_insert_range(&ghvm->mm, gfn,
+ gfn + folio_nr_pages(folio) - 1,
+ xa_tag_pointer(folio, tag), GFP_KERNEL);
+ if (ret == -EEXIST)
+ return -EAGAIN;
+ if (ret)
+ return ret;
+
+ if (share && write)
+ access = GUNYAH_PAGETABLE_ACCESS_RW;
+ else if (share && !write)
+ access = GUNYAH_PAGETABLE_ACCESS_R;
+ else if (!share && write)
+ access = GUNYAH_PAGETABLE_ACCESS_RWX;
+ else /* !share && !write */
+ access = GUNYAH_PAGETABLE_ACCESS_RX;
+
+ gunyah_error = gunyah_hypercall_memextent_donate(donate_flags(share),
+ host_extent->capid,
+ guest_extent->capid,
+ pa, size);
+ if (gunyah_error != GUNYAH_ERROR_OK) {
+ pr_err("Failed to donate memory for guest address 0x%016llx: %d\n",
+ gpa, gunyah_error);
+ ret = gunyah_error_remap(gunyah_error);
+ goto remove;
+ }
+
+ extent_attrs =
+ FIELD_PREP_CONST(GUNYAH_MEMEXTENT_MAPPING_TYPE,
+ ARCH_GUNYAH_DEFAULT_MEMTYPE) |
+ FIELD_PREP(GUNYAH_MEMEXTENT_MAPPING_USER_ACCESS, access) |
+ FIELD_PREP(GUNYAH_MEMEXTENT_MAPPING_KERNEL_ACCESS, access);
+ gunyah_error = gunyah_hypercall_addrspace_map(addrspace->capid,
+ guest_extent->capid, gpa,
+ extent_attrs, map_flags,
+ pa, size);
+ if (gunyah_error != GUNYAH_ERROR_OK) {
+ pr_err("Failed to map guest address 0x%016llx: %d\n", gpa,
+ gunyah_error);
+ ret = gunyah_error_remap(gunyah_error);
+ goto memextent_reclaim;
+ }
+
+ folio_get(folio);
+ if (!share)
+ folio_set_private(folio);
+ return 0;
+memextent_reclaim:
+ gunyah_error = gunyah_hypercall_memextent_donate(reclaim_flags(share),
+ guest_extent->capid,
+ host_extent->capid, pa,
+ size);
+ if (gunyah_error != GUNYAH_ERROR_OK)
+ pr_err("Failed to reclaim memory donation for guest address 0x%016llx: %d\n",
+ gpa, gunyah_error);
+remove:
+ mtree_erase(&ghvm->mm, gfn);
+ return ret;
+}
+
+static int __gunyah_vm_reclaim_folio_locked(struct gunyah_vm *ghvm, void *entry,
+ u64 gfn, const bool sync)
+{
+ u32 map_flags = BIT(GUNYAH_ADDRSPACE_MAP_FLAG_PARTIAL);
+ struct gunyah_resource *guest_extent, *host_extent, *addrspace;
+ enum gunyah_pagetable_access access;
+ enum gunyah_error gunyah_error;
+ struct folio *folio;
+ bool write, share;
+ phys_addr_t pa;
+ size_t size;
+ int ret;
+
+ addrspace = __first_resource(&ghvm->addrspace_ticket);
+ if (!addrspace)
+ return -ENODEV;
+
+ share = !!(xa_pointer_tag(entry) & SHARE_TAG);
+ write = !!(xa_pointer_tag(entry) & WRITE_TAG);
+ folio = xa_untag_pointer(entry);
+
+ if (!sync)
+ map_flags |= BIT(GUNYAH_ADDRSPACE_MAP_FLAG_NOSYNC);
+
+ /* clang-format off */
+ if (share) {
+ guest_extent = __first_resource(&ghvm->guest_shared_extent_ticket);
+ host_extent = __first_resource(&ghvm->host_shared_extent_ticket);
+ map_flags |= BIT(GUNYAH_ADDRSPACE_MAP_FLAG_VMMIO);
+ } else {
+ guest_extent = __first_resource(&ghvm->guest_private_extent_ticket);
+ host_extent = __first_resource(&ghvm->host_private_extent_ticket);
+ map_flags |= BIT(GUNYAH_ADDRSPACE_MAP_FLAG_PRIVATE);
+ }
+ /* clang-format on */
+
+ pa = PFN_PHYS(folio_pfn(folio));
+ size = folio_size(folio);
+
+ gunyah_error = gunyah_hypercall_addrspace_unmap(addrspace->capid,
+ guest_extent->capid,
+ gunyah_gfn_to_gpa(gfn),
+ map_flags, pa, size);
+ if (gunyah_error != GUNYAH_ERROR_OK) {
+ pr_err_ratelimited(
+ "Failed to unmap guest address 0x%016llx: %d\n",
+ gunyah_gfn_to_gpa(gfn), gunyah_error);
+ ret = gunyah_error_remap(gunyah_error);
+ goto err;
+ }
+
+ gunyah_error = gunyah_hypercall_memextent_donate(reclaim_flags(share),
+ guest_extent->capid,
+ host_extent->capid, pa,
+ size);
+ if (gunyah_error != GUNYAH_ERROR_OK) {
+ pr_err_ratelimited(
+ "Failed to reclaim memory donation for guest address 0x%016llx: %d\n",
+ gunyah_gfn_to_gpa(gfn), gunyah_error);
+ ret = gunyah_error_remap(gunyah_error);
+ goto err;
+ }
+
+ if (share && write)
+ access = GUNYAH_PAGETABLE_ACCESS_RW;
+ else if (share && !write)
+ access = GUNYAH_PAGETABLE_ACCESS_R;
+ else if (!share && write)
+ access = GUNYAH_PAGETABLE_ACCESS_RWX;
+ else /* !share && !write */
+ access = GUNYAH_PAGETABLE_ACCESS_RX;
+
+ gunyah_error = gunyah_hypercall_memextent_donate(donate_flags(share),
+ guest_extent->capid,
+ host_extent->capid, pa,
+ size);
+ if (gunyah_error != GUNYAH_ERROR_OK) {
+ pr_err("Failed to reclaim memory donation for guest address 0x%016llx: %d\n",
+ gfn << PAGE_SHIFT, gunyah_error);
+ ret = gunyah_error_remap(gunyah_error);
+ goto err;
+ }
+
+ BUG_ON(mtree_erase(&ghvm->mm, gfn) != entry);
+
+ folio_clear_private(folio);
+ folio_put(folio);
+ return 0;
+err:
+ return ret;
+}
+
+int gunyah_vm_reclaim_folio(struct gunyah_vm *ghvm, u64 gfn)
+{
+ struct folio *folio;
+ void *entry;
+
+ entry = mtree_load(&ghvm->mm, gfn);
+ if (!entry)
+ return 0;
+
+ folio = xa_untag_pointer(entry);
+ if (mtree_load(&ghvm->mm, gfn) != entry)
+ return -EAGAIN;
+
+ return __gunyah_vm_reclaim_folio_locked(ghvm, entry, gfn, true);
+}
+
+int gunyah_vm_reclaim_range(struct gunyah_vm *ghvm, u64 gfn, u64 nr)
+{
+ unsigned long next = gfn, g;
+ struct folio *folio;
+ int ret, ret2 = 0;
+ void *entry;
+ bool sync;
+
+ mt_for_each(&ghvm->mm, entry, next, gfn + nr) {
+ folio = xa_untag_pointer(entry);
+ g = next;
+ sync = !!mt_find_after(&ghvm->mm, &g, gfn + nr);
+
+ g = next - folio_nr_pages(folio);
+ folio_get(folio);
+ folio_lock(folio);
+ if (mtree_load(&ghvm->mm, g) == entry)
+ ret = __gunyah_vm_reclaim_folio_locked(ghvm, entry, g, sync);
+ else
+ ret = -EAGAIN;
+ folio_unlock(folio);
+ folio_put(folio);
+ if (ret && ret2 != -EAGAIN)
+ ret2 = ret;
+ }
+
+ return ret2;
+}