[v9,6/6] iommu/dma: Make flush queue sizes and timeout driver configurable
Commit Message
Flush queues currently use a fixed compile time size of 256 entries.
This being a power of 2 allows the compiler to use shift and mask
instead of more expensive modulo operations. With per-CPU flush queues
larger queue sizes would hit per-CPU allocation limits, with a single
flush queue these limits do not apply however. Also with single queues
being particularly suitable for virtualized environments with expensive
IOTLB flushes these benefit especially from larger queues and thus fewer
flushes.
To this end re-order struct iova_fq so we can use a dynamic array and
introduce the flush queue size and timeouts as new options in the
dma_iommu_options struct. So as not to lose the shift and mask
optimization, check that the variable length is a power of 2 and use
explicit shift and mask instead of letting the compiler optimize this.
In the s390 IOMMU driver a large fixed queue size and timeout is then
set together with single queue mode bringing its performance on s390
paged memory guests on par with the previous s390 specific DMA API
implementation.
Reviewed-by: Matthew Rosato <mjrosato@linux.ibm.com> #s390
Signed-off-by: Niklas Schnelle <schnelle@linux.ibm.com>
---
drivers/iommu/dma-iommu.c | 40 +++++++++++++++++++++++++---------------
drivers/iommu/s390-iommu.c | 8 +++++++-
include/linux/iommu.h | 6 +++++-
3 files changed, 37 insertions(+), 17 deletions(-)
Comments
On Mon, May 15, 2023 at 11:15:56AM +0200, Niklas Schnelle wrote:
> In the s390 IOMMU driver a large fixed queue size and timeout is then
> set together with single queue mode bringing its performance on s390
> paged memory guests on par with the previous s390 specific DMA API
> implementation.
Hmm, the right flush-queue size and timeout settings are more a function
of the endpoint device and device driver than of the iommu driver, no? I
think something like this could also help solving the recently reported
scalability problems in the fq-code, if done right.
Regards,
Joerg
On Mon, 2023-05-22 at 18:10 +0200, Joerg Roedel wrote:
> On Mon, May 15, 2023 at 11:15:56AM +0200, Niklas Schnelle wrote:
> > In the s390 IOMMU driver a large fixed queue size and timeout is then
> > set together with single queue mode bringing its performance on s390
> > paged memory guests on par with the previous s390 specific DMA API
> > implementation.
>
> Hmm, the right flush-queue size and timeout settings are more a function
> of the endpoint device and device driver than of the iommu driver, no? I
> think something like this could also help solving the recently reported
> scalability problems in the fq-code, if done right.
>
> Regards,
>
> Joerg
>
In our case the large flush queue and timeout is needed because the
IOTLB flushes of the virtualized s390 IOMMU are used by KVM and z/VM to
synchronize their IOMMU shadow tables thus making them more expensive.
This then applies to all pass-through PCI devices without their drivers
knowing about the IOMMU being virtualized like that. But yes of course
there could be cases where the device driver knows better.
Thanks,
Niklas
@@ -89,10 +89,10 @@ static int __init iommu_dma_forcedac_setup(char *str)
early_param("iommu.forcedac", iommu_dma_forcedac_setup);
/* Number of entries per flush queue */
-#define IOVA_FQ_SIZE 256
+#define IOVA_DEFAULT_FQ_SIZE 256
/* Timeout (in ms) after which entries are flushed from the queue */
-#define IOVA_FQ_TIMEOUT 10
+#define IOVA_DEFAULT_FQ_TIMEOUT 10
/* Flush queue entry for deferred flushing */
struct iova_fq_entry {
@@ -104,18 +104,19 @@ struct iova_fq_entry {
/* Per-CPU flush queue structure */
struct iova_fq {
- struct iova_fq_entry entries[IOVA_FQ_SIZE];
- unsigned int head, tail;
spinlock_t lock;
+ unsigned int head, tail;
+ unsigned int mod_mask;
+ struct iova_fq_entry entries[];
};
#define fq_ring_for_each(i, fq) \
- for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
+ for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) & (fq)->mod_mask)
static inline bool fq_full(struct iova_fq *fq)
{
assert_spin_locked(&fq->lock);
- return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
+ return (((fq->tail + 1) & fq->mod_mask) == fq->head);
}
static inline unsigned int fq_ring_add(struct iova_fq *fq)
@@ -124,7 +125,7 @@ static inline unsigned int fq_ring_add(struct iova_fq *fq)
assert_spin_locked(&fq->lock);
- fq->tail = (idx + 1) % IOVA_FQ_SIZE;
+ fq->tail = (idx + 1) & fq->mod_mask;
return idx;
}
@@ -146,7 +147,7 @@ static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq)
fq->entries[idx].iova_pfn,
fq->entries[idx].pages);
- fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
+ fq->head = (fq->head + 1) & fq->mod_mask;
}
}
@@ -244,7 +245,7 @@ static void queue_iova(struct iommu_dma_cookie *cookie,
if (!atomic_read(&cookie->fq_timer_on) &&
!atomic_xchg(&cookie->fq_timer_on, 1))
mod_timer(&cookie->fq_timer,
- jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
+ jiffies + msecs_to_jiffies(cookie->options.fq_timeout));
}
static void iommu_dma_free_fq_single(struct iova_fq *fq)
@@ -286,27 +287,29 @@ static void iommu_dma_free_fq(struct iommu_dma_cookie *cookie)
}
-static void iommu_dma_init_one_fq(struct iova_fq *fq)
+static void iommu_dma_init_one_fq(struct iova_fq *fq, size_t fq_size)
{
int i;
fq->head = 0;
fq->tail = 0;
+ fq->mod_mask = fq_size - 1;
spin_lock_init(&fq->lock);
- for (i = 0; i < IOVA_FQ_SIZE; i++)
+ for (i = 0; i < fq_size; i++)
INIT_LIST_HEAD(&fq->entries[i].freelist);
}
static int iommu_dma_init_fq_single(struct iommu_dma_cookie *cookie)
{
+ size_t fq_size = cookie->options.fq_size;
struct iova_fq *queue;
- queue = vzalloc(sizeof(*queue));
+ queue = vzalloc(struct_size(queue, entries, fq_size));
if (!queue)
return -ENOMEM;
- iommu_dma_init_one_fq(queue);
+ iommu_dma_init_one_fq(queue, fq_size);
cookie->single_fq = queue;
return 0;
@@ -314,15 +317,17 @@ static int iommu_dma_init_fq_single(struct iommu_dma_cookie *cookie)
static int iommu_dma_init_fq_percpu(struct iommu_dma_cookie *cookie)
{
+ size_t fq_size = cookie->options.fq_size;
struct iova_fq __percpu *queue;
int cpu;
- queue = alloc_percpu(struct iova_fq);
+ queue = __alloc_percpu(struct_size(queue, entries, fq_size),
+ __alignof__(*queue));
if (!queue)
return -ENOMEM;
for_each_possible_cpu(cpu)
- iommu_dma_init_one_fq(per_cpu_ptr(queue, cpu));
+ iommu_dma_init_one_fq(per_cpu_ptr(queue, cpu), fq_size);
cookie->percpu_fq = queue;
return 0;
}
@@ -340,6 +345,9 @@ int iommu_dma_init_fq(struct device *dev, struct iommu_domain *domain)
if (ops->tune_dma_iommu)
ops->tune_dma_iommu(dev, &cookie->options);
+ if (WARN_ON_ONCE(!is_power_of_2(cookie->options.fq_size)))
+ cookie->options.fq_size = IOVA_DEFAULT_FQ_SIZE;
+
atomic64_set(&cookie->fq_flush_start_cnt, 0);
atomic64_set(&cookie->fq_flush_finish_cnt, 0);
@@ -382,6 +390,8 @@ static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type)
INIT_LIST_HEAD(&cookie->msi_page_list);
cookie->type = type;
cookie->options.flags = IOMMU_DMA_OPTS_PER_CPU_QUEUE;
+ cookie->options.fq_size = IOVA_DEFAULT_FQ_SIZE;
+ cookie->options.fq_timeout = IOVA_DEFAULT_FQ_TIMEOUT;
}
return cookie;
}
@@ -451,13 +451,19 @@ static void s390_iommu_get_resv_regions(struct device *dev,
}
}
+#define S390_IOMMU_SINGLE_FQ_SIZE 32768
+#define S390_IOMMU_SINGLE_FQ_TIMEOUT 1000
+
static void s390_iommu_tune_dma_iommu(struct device *dev,
struct dma_iommu_options *options)
{
struct zpci_dev *zdev = to_zpci_dev(dev);
- if (zdev->tlb_refresh)
+ if (zdev->tlb_refresh) {
options->flags |= IOMMU_DMA_OPTS_SINGLE_QUEUE;
+ options->fq_size = S390_IOMMU_SINGLE_FQ_SIZE;
+ options->fq_timeout = S390_IOMMU_SINGLE_FQ_TIMEOUT;
+ }
}
static struct iommu_device *s390_iommu_probe_device(struct device *dev)
@@ -223,6 +223,8 @@ struct iommu_iotlb_gather {
* struct dma_iommu_options - Options for dma-iommu
*
* @flags: Flag bits for enabling/disabling dma-iommu settings
+ * @fq_size: Size of the IOTLB flush queue(s), must be a power of two
+ * @fq_timeout: Timeout used for queued IOTLB flushes
*
* This structure is intended to provide IOMMU drivers a way to influence the
* behavior of the dma-iommu DMA API implementation. This allows optimizing for
@@ -231,7 +233,9 @@ struct iommu_iotlb_gather {
struct dma_iommu_options {
#define IOMMU_DMA_OPTS_PER_CPU_QUEUE (0L << 0)
#define IOMMU_DMA_OPTS_SINGLE_QUEUE (1L << 0)
- u64 flags;
+ u64 flags;
+ size_t fq_size;
+ unsigned int fq_timeout;
};
/**