Enable support for creating irqfds which can raise an interrupt on a
Gunyah virtual machine. irqfds are exposed to userspace as a Gunyah VM
function with the name "irqfd". If the VM devicetree is not configured
to create a doorbell with the corresponding label, userspace will still
be able to assert the eventfd but no interrupt will be raised on the
guest.
Co-developed-by: Prakruthi Deepak Heragu <quic_pheragu@quicinc.com>
Signed-off-by: Prakruthi Deepak Heragu <quic_pheragu@quicinc.com>
Signed-off-by: Elliot Berman <quic_eberman@quicinc.com>
---
Documentation/virt/gunyah/vm-manager.rst | 22 +++
drivers/virt/gunyah/Kconfig | 10 ++
drivers/virt/gunyah/Makefile | 1 +
drivers/virt/gunyah/gunyah_irqfd.c | 180 +++++++++++++++++++++++
include/linux/gunyah.h | 5 +
include/uapi/linux/gunyah.h | 11 +-
6 files changed, 228 insertions(+), 1 deletion(-)
create mode 100644 drivers/virt/gunyah/gunyah_irqfd.c
Hi Elliot,
I love your patch! Perhaps something to improve:
[auto build test WARNING on 830b3c68c1fb1e9176028d02ef86f3cf76aa2476]
url: https://github.com/intel-lab-lkp/linux/commits/Elliot-Berman/Drivers-for-gunyah-hypervisor/20221220-073053
base: 830b3c68c1fb1e9176028d02ef86f3cf76aa2476
patch link: https://lore.kernel.org/r/20221219225850.2397345-28-quic_eberman%40quicinc.com
patch subject: [PATCH v8 27/28] virt: gunyah: Add irqfd interface
config: arm64-allyesconfig
compiler: aarch64-linux-gcc (GCC) 12.1.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/intel-lab-lkp/linux/commit/158304cc06a0a3021e5103a3a84fb2b1dfc6f703
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Elliot-Berman/Drivers-for-gunyah-hypervisor/20221220-073053
git checkout 158304cc06a0a3021e5103a3a84fb2b1dfc6f703
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=arm64 olddefconfig
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=arm64 SHELL=/bin/bash drivers/virt/gunyah/
If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <lkp@intel.com>
All warnings (new ones prefixed by >>):
>> drivers/virt/gunyah/gunyah_vcpu.c:58: warning: This comment starts with '/**', but isn't a kernel-doc comment. Refer Documentation/doc-guide/kernel-doc.rst
* When hypervisor wants us to schedule vCPU again, it gives us an interrupt
vim +58 drivers/virt/gunyah/gunyah_vcpu.c
b4c9ba3b6fa553a Elliot Berman 2022-12-19 56
b4c9ba3b6fa553a Elliot Berman 2022-12-19 57 /**
b4c9ba3b6fa553a Elliot Berman 2022-12-19 @58 * When hypervisor wants us to schedule vCPU again, it gives us an interrupt
b4c9ba3b6fa553a Elliot Berman 2022-12-19 59 */
b4c9ba3b6fa553a Elliot Berman 2022-12-19 60 static irqreturn_t gh_vcpu_irq_handler(int irq, void *data)
b4c9ba3b6fa553a Elliot Berman 2022-12-19 61 {
b4c9ba3b6fa553a Elliot Berman 2022-12-19 62 struct gunyah_vcpu *vcpu = data;
b4c9ba3b6fa553a Elliot Berman 2022-12-19 63
b4c9ba3b6fa553a Elliot Berman 2022-12-19 64 complete(&vcpu->ready);
b4c9ba3b6fa553a Elliot Berman 2022-12-19 65 return IRQ_HANDLED;
b4c9ba3b6fa553a Elliot Berman 2022-12-19 66 }
b4c9ba3b6fa553a Elliot Berman 2022-12-19 67
@@ -142,3 +142,25 @@ The vcpu type will register with the VM Manager to expect to control
vCPU number `vcpu_id`. It returns a file descriptor allowing interaction with
the vCPU. See the Gunyah vCPU API description sections for interacting with
the Gunyah vCPU file descriptors.
+
+Type: "irqfd"
+^^^^^^^^^^^^^
+
+::
+
+ struct gh_fn_irqfd_arg {
+ __u32 fd;
+ __u32 label;
+ #define GH_IRQFD_LEVEL (1UL << 0)
+ #define GH_IRQFD_DEASSIGN (1UL << 1)
+ __u32 flags;
+ };
+
+Allows setting an eventfd to directly trigger a guest interrupt.
+irqfd.fd specifies the file descriptor to use as the eventfd.
+irqfd.label corresponds to the doorbell label used in the guest VM's devicetree.
+The irqfd is removed using the GH_IRQFD_DEASSIGN flag and specifying at least
+the irqfd.label.
+
+GH_IRQFD_LEVEL configures the corresponding doorbell to behave like a level
+triggered interrupt.
@@ -49,5 +49,15 @@ config GUNYAH_VCPU
can schedule the guest VM's vCPUs instead of using Gunyah's scheduler.
VMMs can also handle stage 2 faults of the vCPUs.
+ Say Y/M here if unsure and you want to support Gunyah VMMs.
+
+config GUNYAH_IRQFD
+ tristate "Gunyah irqfd interface"
+ depends on GUNYAH_RESOURCE_MANAGER
+ depends on GUNYAH_VM_MANAGER
+ help
+ Enable kernel support for creating irqfds which can raise an interrupt
+ on Gunyah virtual machine.
+
Say Y/M here if unsure and you want to support Gunyah VMMs.
endif
@@ -6,3 +6,4 @@ gunyah_rsc_mgr-$(CONFIG_GUNYAH_VM_MANAGER) += vm_mgr.o vm_mgr_mm.o
obj-$(CONFIG_GUNYAH_RESOURCE_MANAGER) += gunyah_rsc_mgr.o
obj-$(CONFIG_GUNYAH_VCPU) += gunyah_vcpu.o
+obj-$(CONFIG_GUNYAH_IRQFD) += gunyah_irqfd.o
new file mode 100644
@@ -0,0 +1,180 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
+ */
+
+#include <linux/eventfd.h>
+#include <linux/file.h>
+#include <linux/fs.h>
+#include <linux/gunyah.h>
+#include <linux/gunyah_vm_mgr.h>
+#include <linux/kref.h>
+#include <linux/module.h>
+#include <linux/poll.h>
+#include <linux/printk.h>
+
+#include <uapi/linux/gunyah.h>
+
+struct gunyah_irqfd {
+ struct gunyah_resource *ghrsc;
+ struct gunyah_vm_resource_ticket ticket;
+ struct gunyah_vm_function *f;
+
+ struct kref kref;
+ bool level;
+
+ struct eventfd_ctx *ctx;
+ wait_queue_entry_t wait;
+ poll_table pt;
+ struct fd fd;
+ struct work_struct shutdown_work;
+};
+
+static void gh_irqfd_cleanup(struct kref *kref)
+{
+ struct gunyah_irqfd *irqfd = container_of(kref, struct gunyah_irqfd, kref);
+
+ kfree(irqfd);
+}
+
+
+static void irqfd_shutdown(struct work_struct *work)
+{
+ struct gunyah_irqfd *irqfd = container_of(work, struct gunyah_irqfd, shutdown_work);
+ u64 isr;
+
+ if (irqfd->ctx) {
+ eventfd_ctx_remove_wait_queue(irqfd->ctx, &irqfd->wait, &isr);
+ eventfd_ctx_put(irqfd->ctx);
+ fdput(irqfd->fd);
+ irqfd->ctx = NULL;
+ irqfd->fd.file = NULL;
+ }
+
+ kref_put(&irqfd->kref, gh_irqfd_cleanup);
+}
+
+static int irqfd_wakeup(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key)
+{
+ struct gunyah_irqfd *irqfd = container_of(wait, struct gunyah_irqfd, wait);
+ __poll_t flags = key_to_poll(key);
+ u64 enable_mask = GH_DBL_NONBLOCK;
+ u64 old_flags;
+ int ret = 0;
+
+ if (flags & EPOLLIN) {
+ if (irqfd->ghrsc) {
+ ret = gh_hypercall_dbl_send(irqfd->ghrsc->capid, enable_mask, &old_flags);
+ if (ret)
+ pr_err("Failed to assert irq %d\n", irqfd->f->fn.irqfd.label);
+ }
+ }
+
+ return 0;
+}
+
+static void irqfd_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, poll_table *pt)
+{
+ struct gunyah_irqfd *irq_ctx = container_of(pt, struct gunyah_irqfd, pt);
+
+ add_wait_queue(wqh, &irq_ctx->wait);
+}
+
+static int gunyah_irqfd_populate(struct gunyah_vm_resource_ticket *ticket,
+ struct gunyah_resource *ghrsc)
+{
+ struct gunyah_irqfd *irqfd = container_of(ticket, struct gunyah_irqfd, ticket);
+ u64 enable_mask = GH_DBL_NONBLOCK;
+ u64 ack_mask = ~0;
+ int ret = 0;
+
+ irqfd->ghrsc = ghrsc;
+ if (irqfd->level) {
+ ret = gh_hypercall_dbl_set_mask(irqfd->ghrsc->capid, enable_mask, ack_mask);
+ if (ret)
+ pr_warn("irq %d couldn't be set as level triggered. Might cause IRQ storm if asserted\n",
+ irqfd->f->fn.irqfd.label);
+ }
+ kref_get(&irqfd->kref);
+
+ return 0;
+}
+
+static void gunyah_irqfd_unpopulate(struct gunyah_vm_resource_ticket *ticket,
+ struct gunyah_resource *ghrsc)
+{
+ struct gunyah_irqfd *irqfd = container_of(ticket, struct gunyah_irqfd, ticket);
+
+ queue_work(system_wq, &irqfd->shutdown_work);
+ irqfd->ghrsc = NULL;
+ kref_put(&irqfd->kref, gh_irqfd_cleanup);
+}
+
+static long gunyah_irqfd_bind(struct gunyah_vm_function *f)
+{
+ __poll_t events;
+ struct gunyah_irqfd *irqfd;
+ long r;
+
+ irqfd = kzalloc(sizeof(*irqfd), GFP_KERNEL);
+ if (!irqfd)
+ return -ENOMEM;
+
+ irqfd->f = f;
+ f->data = irqfd;
+
+ irqfd->fd = fdget(f->fn.irqfd.fd);
+ if (!irqfd->fd.file) {
+ r = -EBADF;
+ goto err_free;
+ }
+
+ irqfd->ctx = eventfd_ctx_fileget(irqfd->fd.file);
+ if (IS_ERR(irqfd->ctx)) {
+ r = PTR_ERR(irqfd->ctx);
+ goto err_fdput;
+ }
+
+ if (f->fn.irqfd.flags & GH_IRQFD_LEVEL)
+ irqfd->level = true;
+
+ irqfd->ticket.resource_type = GUNYAH_RESOURCE_TYPE_BELL_TX;
+ irqfd->ticket.label = f->fn.irqfd.label;
+ irqfd->ticket.owner = THIS_MODULE;
+ irqfd->ticket.populate = gunyah_irqfd_populate;
+ irqfd->ticket.unpopulate = gunyah_irqfd_unpopulate;
+
+ r = ghvm_add_resource_ticket(f->ghvm, &irqfd->ticket);
+ if (r)
+ goto err_ctx;
+
+ init_waitqueue_func_entry(&irqfd->wait, irqfd_wakeup);
+ INIT_WORK(&irqfd->shutdown_work, irqfd_shutdown);
+ init_poll_funcptr(&irqfd->pt, irqfd_ptable_queue_proc);
+ kref_init(&irqfd->kref);
+
+ events = vfs_poll(irqfd->fd.file, &irqfd->pt);
+ if (events & EPOLLIN)
+ pr_warn("Premature injection of interrupt\n");
+
+ return 0;
+err_ctx:
+ eventfd_ctx_put(irqfd->ctx);
+err_fdput:
+ fdput(irqfd->fd);
+err_free:
+ kfree(irqfd);
+ return r;
+}
+
+static void gunyah_irqfd_release(struct gunyah_vm_function *f)
+{
+ struct gunyah_irqfd *irqfd = f->data;
+
+ /* unpopulate will trigger clean up of the eventfd */
+ ghvm_remove_resource_ticket(irqfd->f->ghvm, &irqfd->ticket);
+}
+
+DECLARE_GUNYAH_VM_FUNCTION_INIT(irqfd, gunyah_irqfd_bind, gunyah_irqfd_release);
+MODULE_DESCRIPTION("Gunyah irqfds");
+MODULE_LICENSE("GPL");
@@ -32,6 +32,11 @@ struct gunyah_resource {
u32 rm_label;
};
+/**
+ * Gunyah Doorbells
+ */
+#define GH_DBL_NONBLOCK BIT(32)
+
/**
* Gunyah Message Queues
*/
@@ -57,10 +57,19 @@ struct gh_fn_vcpu_arg {
__u32 vcpu_id;
};
+struct gh_fn_irqfd_arg {
+ __u32 fd;
+ __u32 label;
+#define GH_IRQFD_LEVEL (1UL << 0)
+#define GH_IRQFD_DEASSIGN (1UL << 1)
+ __u32 flags;
+};
+
struct gh_vm_function {
char name[GUNYAH_FUNCTION_NAME_SIZE];
union {
- struct gh_device_vcpu_arg vcpu;
+ struct gh_fn_vcpu_arg vcpu;
+ struct gh_fn_irqfd_arg irqfd;
char data[GUNYAH_FUNCTION_MAX_ARG_SIZE];
};
};