@@ -22,6 +22,21 @@
#include "vfio_pci_priv.h"
+/*
+ * Interrupt Message Store (IMS) private interrupt context data
+ * @vdev: Virtual device. Used for name of device in
+ * request_irq().
+ * @pdev: PCI device owning the IMS domain from where
+ * interrupts are allocated.
+ * @default_cookie: Default cookie used for IMS interrupts without unique
+ * cookie.
+ */
+struct vfio_pci_ims {
+ struct vfio_device *vdev;
+ struct pci_dev *pdev;
+ union msi_instance_cookie default_cookie;
+};
+
struct vfio_pci_irq_ctx {
struct eventfd_ctx *trigger;
struct virqfd *unmask;
@@ -29,6 +44,9 @@ struct vfio_pci_irq_ctx {
char *name;
bool masked;
struct irq_bypass_producer producer;
+ int ims_id;
+ int virq;
+ union msi_instance_cookie icookie;
};
static bool irq_is(struct vfio_pci_core_device *vdev, int type)
@@ -72,6 +90,12 @@ vfio_irq_ctx_alloc(struct vfio_pci_intr_ctx *intr_ctx, unsigned long index)
if (!ctx)
return NULL;
+ if (intr_ctx->ims_backed_irq) {
+ struct vfio_pci_ims *ims = intr_ctx->priv;
+
+ ctx->icookie = ims->default_cookie;
+ }
+
ret = xa_insert(&intr_ctx->ctx, index, ctx, GFP_KERNEL_ACCOUNT);
if (ret) {
kfree(ctx);
@@ -822,6 +846,7 @@ void vfio_pci_init_intr_ctx(struct vfio_pci_core_device *vdev,
_vfio_pci_init_intr_ctx(intr_ctx);
intr_ctx->ops = &vfio_pci_intr_ops;
intr_ctx->priv = vdev;
+ intr_ctx->ims_backed_irq = false;
}
EXPORT_SYMBOL_GPL(vfio_pci_init_intr_ctx);
@@ -831,6 +856,256 @@ void vfio_pci_release_intr_ctx(struct vfio_pci_intr_ctx *intr_ctx)
}
EXPORT_SYMBOL_GPL(vfio_pci_release_intr_ctx);
+/* Guest MSI-X interrupts backed by IMS host interrupts */
+
+/*
+ * Free the IMS interrupt associated with @ctx.
+ *
+ * For an IMS interrupt the interrupt is freed from the underlying
+ * PCI device's IMS domain.
+ */
+static void vfio_pci_ims_irq_free(struct vfio_pci_intr_ctx *intr_ctx,
+ struct vfio_pci_irq_ctx *ctx)
+{
+ struct vfio_pci_ims *ims = intr_ctx->priv;
+ struct msi_map irq_map = {};
+
+ irq_map.index = ctx->ims_id;
+ irq_map.virq = ctx->virq;
+ pci_ims_free_irq(ims->pdev, irq_map);
+ ctx->ims_id = -EINVAL;
+ ctx->virq = 0;
+}
+
+/*
+ * Allocate a host IMS interrupt for @ctx.
+ *
+ * For an IMS interrupt the interrupt is allocated from the underlying
+ * PCI device's IMS domain.
+ */
+static int vfio_pci_ims_irq_alloc(struct vfio_pci_intr_ctx *intr_ctx,
+ struct vfio_pci_irq_ctx *ctx)
+{
+ struct vfio_pci_ims *ims = intr_ctx->priv;
+ struct msi_map irq_map = {};
+
+ irq_map = pci_ims_alloc_irq(ims->pdev, &ctx->icookie, NULL);
+ if (irq_map.index < 0)
+ return irq_map.index;
+
+ ctx->ims_id = irq_map.index;
+ ctx->virq = irq_map.virq;
+
+ return 0;
+}
+
+static int vfio_pci_ims_set_vector_signal(struct vfio_pci_intr_ctx *intr_ctx,
+ unsigned int vector, int fd)
+{
+ struct vfio_pci_ims *ims = intr_ctx->priv;
+ struct device *dev = &ims->vdev->device;
+ struct vfio_pci_irq_ctx *ctx;
+ struct eventfd_ctx *trigger;
+ int ret;
+
+ ctx = vfio_irq_ctx_get(intr_ctx, vector);
+
+ if (ctx && ctx->trigger) {
+ irq_bypass_unregister_producer(&ctx->producer);
+ free_irq(ctx->virq, ctx->trigger);
+ vfio_pci_ims_irq_free(intr_ctx, ctx);
+ kfree(ctx->name);
+ ctx->name = NULL;
+ eventfd_ctx_put(ctx->trigger);
+ ctx->trigger = NULL;
+ }
+
+ if (fd < 0)
+ return 0;
+
+ /* Interrupt contexts remain allocated until shutdown. */
+ if (!ctx) {
+ ctx = vfio_irq_ctx_alloc(intr_ctx, vector);
+ if (!ctx)
+ return -ENOMEM;
+ }
+
+ ctx->name = kasprintf(GFP_KERNEL, "vfio-ims[%d](%s)", vector,
+ dev_name(dev));
+ if (!ctx->name)
+ return -ENOMEM;
+
+ trigger = eventfd_ctx_fdget(fd);
+ if (IS_ERR(trigger)) {
+ ret = PTR_ERR(trigger);
+ goto out_free_name;
+ }
+
+ ctx->trigger = trigger;
+
+ ret = vfio_pci_ims_irq_alloc(intr_ctx, ctx);
+ if (ret < 0)
+ goto out_put_eventfd_ctx;
+
+ ret = request_irq(ctx->virq, vfio_msihandler, 0, ctx->name,
+ ctx->trigger);
+ if (ret < 0)
+ goto out_free_irq;
+
+ ctx->producer.token = ctx->trigger;
+ ctx->producer.irq = ctx->virq;
+ ret = irq_bypass_register_producer(&ctx->producer);
+ if (unlikely(ret)) {
+ dev_info(&ims->vdev->device,
+ "irq bypass producer (token %p) registration fails: %d\n",
+ &ctx->producer.token, ret);
+ ctx->producer.token = NULL;
+ }
+
+ return 0;
+
+out_free_irq:
+ vfio_pci_ims_irq_free(intr_ctx, ctx);
+out_put_eventfd_ctx:
+ eventfd_ctx_put(ctx->trigger);
+ ctx->trigger = NULL;
+out_free_name:
+ kfree(ctx->name);
+ ctx->name = NULL;
+ return ret;
+}
+
+static int vfio_pci_ims_set_block(struct vfio_pci_intr_ctx *intr_ctx,
+ unsigned int start, unsigned int count,
+ int *fds)
+{
+ unsigned int i, j;
+ int ret = 0;
+
+ for (i = 0, j = start; i < count && !ret; i++, j++) {
+ int fd = fds ? fds[i] : -1;
+
+ ret = vfio_pci_ims_set_vector_signal(intr_ctx, j, fd);
+ }
+
+ if (ret) {
+ for (i = start; i < j; i++)
+ vfio_pci_ims_set_vector_signal(intr_ctx, i, -1);
+ }
+
+ return ret;
+}
+
+/*
+ * Manage Interrupt Message Store (IMS) or emulated interrupts on the
+ * host that are backing guest MSI-X vectors.
+ *
+ * @intr_ctx: Interrupt context
+ * @index: Type of guest vectors to set up. Must be
+ * VFIO_PCI_MSIX_IRQ_INDEX.
+ * @start: First vector index.
+ * @count: Number of vectors.
+ * @flags: Type of data provided in @data.
+ * @data: Data as specified by @flags.
+ *
+ * Caller is required to validate provided range for @vdev.
+ *
+ * Context: Interrupt context must be initialized via
+ * vfio_pci_ims_init_intr_ctx() before any interrupts can be allocated.
+ *
+ * Return: Error code on failure or 0 on success.
+ */
+static int vfio_pci_set_ims_trigger(struct vfio_pci_intr_ctx *intr_ctx,
+ unsigned int index, unsigned int start,
+ unsigned int count, u32 flags,
+ void *data)
+{
+ struct vfio_pci_irq_ctx *ctx;
+ unsigned long i;
+
+ if (index != VFIO_PCI_MSIX_IRQ_INDEX)
+ return -EINVAL;
+
+ if (!count && (flags & VFIO_IRQ_SET_DATA_NONE)) {
+ xa_for_each(&intr_ctx->ctx, i, ctx)
+ vfio_pci_ims_set_vector_signal(intr_ctx, i, -1);
+ return 0;
+ }
+
+ if (flags & VFIO_IRQ_SET_DATA_EVENTFD)
+ return vfio_pci_ims_set_block(intr_ctx, start, count, (int *)data);
+
+ for (i = start; i < start + count; i++) {
+ ctx = vfio_irq_ctx_get(intr_ctx, i);
+ if (!ctx || !ctx->trigger)
+ continue;
+ if (flags & VFIO_IRQ_SET_DATA_NONE) {
+ eventfd_signal(ctx->trigger, 1);
+ } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
+ uint8_t *bools = data;
+
+ if (bools[i - start])
+ eventfd_signal(ctx->trigger, 1);
+ }
+ }
+
+ return 0;
+}
+
+struct vfio_pci_intr_ops vfio_pci_ims_intr_ops = {
+ .set_msix_trigger = vfio_pci_set_ims_trigger,
+ .set_req_trigger = vfio_pci_set_req_trigger,
+};
+
+int vfio_pci_ims_init_intr_ctx(struct vfio_device *vdev,
+ struct vfio_pci_intr_ctx *intr_ctx,
+ struct pci_dev *pdev,
+ union msi_instance_cookie *default_cookie)
+{
+ struct vfio_pci_ims *ims;
+
+ ims = kzalloc(sizeof(*ims), GFP_KERNEL_ACCOUNT);
+ if (!ims)
+ return -ENOMEM;
+
+ ims->pdev = pdev;
+ ims->default_cookie = *default_cookie;
+ ims->vdev = vdev;
+
+ _vfio_pci_init_intr_ctx(intr_ctx);
+
+ intr_ctx->ops = &vfio_pci_ims_intr_ops;
+ intr_ctx->priv = ims;
+ intr_ctx->ims_backed_irq = true;
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(vfio_pci_ims_init_intr_ctx);
+
+void vfio_pci_ims_release_intr_ctx(struct vfio_pci_intr_ctx *intr_ctx)
+{
+ struct vfio_pci_ims *ims = intr_ctx->priv;
+ struct vfio_pci_irq_ctx *ctx;
+ unsigned long i;
+
+ /*
+ * IMS backed MSI-X keeps interrupt context allocated after
+ * interrupt is freed. Interrupt contexts need to be freed
+ * separately.
+ */
+ mutex_lock(&intr_ctx->igate);
+ xa_for_each(&intr_ctx->ctx, i, ctx) {
+ WARN_ON_ONCE(ctx->trigger);
+ WARN_ON_ONCE(ctx->name);
+ xa_erase(&intr_ctx->ctx, i);
+ kfree(ctx);
+ }
+ mutex_unlock(&intr_ctx->igate);
+ kfree(ims);
+ _vfio_pci_release_intr_ctx(intr_ctx);
+}
+EXPORT_SYMBOL_GPL(vfio_pci_ims_release_intr_ctx);
+
int vfio_pci_set_irqs_ioctl(struct vfio_pci_intr_ctx *intr_ctx, uint32_t flags,
unsigned int index, unsigned int start,
unsigned int count, void *data)
@@ -67,6 +67,7 @@ struct vfio_pci_intr_ctx {
struct eventfd_ctx *req_trigger;
struct xarray ctx;
int irq_type;
+ bool ims_backed_irq:1;
};
struct vfio_pci_intr_ops {
@@ -161,6 +162,11 @@ void vfio_pci_release_intr_ctx(struct vfio_pci_intr_ctx *intr_ctx);
int vfio_pci_set_irqs_ioctl(struct vfio_pci_intr_ctx *intr_ctx, uint32_t flags,
unsigned int index, unsigned int start,
unsigned int count, void *data);
+int vfio_pci_ims_init_intr_ctx(struct vfio_device *vdev,
+ struct vfio_pci_intr_ctx *intr_ctx,
+ struct pci_dev *pdev,
+ union msi_instance_cookie *default_cookie);
+void vfio_pci_ims_release_intr_ctx(struct vfio_pci_intr_ctx *intr_ctx);
int vfio_pci_core_ioctl_feature(struct vfio_device *device, u32 flags,
void __user *arg, size_t argsz);
ssize_t vfio_pci_core_read(struct vfio_device *core_vdev, char __user *buf,