@@ -37,21 +37,35 @@ struct arm_smmu_bond {
static DEFINE_MUTEX(sva_lock);
+/*
+ * When ssid is 0, update all the CD entries that this domain is attached to.
+ * When ssid is non-zero, write the CD into all the masters where this domain is
+ * the primary domain, with the provided SSID. This is used because SVA still
+ * piggybacks over the primary domain.
+ */
static int arm_smmu_write_ctx_desc_devices(struct arm_smmu_domain *smmu_domain,
int ssid,
struct arm_smmu_ctx_desc *cd)
{
+ struct arm_smmu_attached_domain *attached_domain;
struct arm_smmu_master *master;
unsigned long flags;
int ret;
- spin_lock_irqsave(&smmu_domain->devices_lock, flags);
- list_for_each_entry(master, &smmu_domain->devices, domain_head) {
- ret = arm_smmu_write_ctx_desc(master, ssid, cd);
+ spin_lock_irqsave(&smmu_domain->attached_ssids_lock, flags);
+ list_for_each_entry(attached_domain, &smmu_domain->attached_ssids,
+ list) {
+ master = attached_domain->master;
+ if (ssid && attached_domain->ssid == 0) {
+ ret = arm_smmu_write_ctx_desc(master, ssid, cd);
+ } else {
+ ret = arm_smmu_write_ctx_desc(
+ master, attached_domain->ssid, cd);
+ }
if (ret)
break;
}
- spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+ spin_unlock_irqrestore(&smmu_domain->attached_ssids_lock, flags);
return ret;
}
@@ -222,7 +236,7 @@ static void arm_smmu_mm_invalidate_range(struct mmu_notifier *mn,
if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_BTM))
arm_smmu_tlb_inv_range_asid(start, size, smmu_mn->cd->asid,
PAGE_SIZE, false, smmu_domain);
- arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, start, size);
+ arm_smmu_atc_inv_domain_ssid(smmu_domain, mm->pasid, start, size);
}
static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
@@ -243,7 +257,7 @@ static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
arm_smmu_write_ctx_desc_devices(smmu_domain, mm->pasid, &quiet_cd);
arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_mn->cd->asid);
- arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);
+ arm_smmu_atc_inv_domain_ssid(smmu_domain, mm->pasid, 0, 0);
smmu_mn->cleared = true;
mutex_unlock(&sva_lock);
@@ -333,7 +347,7 @@ static void arm_smmu_mmu_notifier_put(struct arm_smmu_mmu_notifier *smmu_mn)
*/
if (!smmu_mn->cleared) {
arm_smmu_tlb_inv_asid(smmu_domain->smmu, cd->asid);
- arm_smmu_atc_inv_domain(smmu_domain, mm->pasid, 0, 0);
+ arm_smmu_atc_inv_domain_ssid(smmu_domain, mm->pasid, 0, 0);
}
/* Frees smmu_mn */
@@ -1282,7 +1282,7 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_master *master, u32 sid,
};
if (master) {
- smmu_domain = master->domain;
+ smmu_domain = master->non_pasid_domain.domain;
smmu = master->smmu;
}
@@ -1725,7 +1725,14 @@ static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
}
static void
-arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
+arm_smmu_atc_inv_cmd_set_ssid(int ssid, struct arm_smmu_cmdq_ent *cmd)
+{
+ cmd->substream_valid = !!ssid;
+ cmd->atc.ssid = ssid;
+}
+
+static void
+arm_smmu_atc_inv_to_cmd(unsigned long iova, size_t size,
struct arm_smmu_cmdq_ent *cmd)
{
size_t log2_span;
@@ -1750,8 +1757,8 @@ arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
*/
*cmd = (struct arm_smmu_cmdq_ent) {
.opcode = CMDQ_OP_ATC_INV,
- .substream_valid = !!ssid,
- .atc.ssid = ssid,
+ .substream_valid = false,
+ .atc.ssid = 0,
};
if (!size) {
@@ -1797,8 +1804,7 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
struct arm_smmu_cmdq_ent cmd;
struct arm_smmu_cmdq_batch cmds;
- arm_smmu_atc_inv_to_cmd(0, 0, 0, &cmd);
-
+ arm_smmu_atc_inv_to_cmd(0, 0, &cmd);
cmds.num = 0;
for (i = 0; i < master->num_streams; i++) {
cmd.atc.sid = master->streams[i].id;
@@ -1808,13 +1814,19 @@ static int arm_smmu_atc_inv_master(struct arm_smmu_master *master)
return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
}
-int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
- unsigned long iova, size_t size)
+/*
+ * If ssid is non-zero, issue atc invalidations with the given ssid instead of
+ * the one the domain is attached to. This is used by SVA since it's pasid
+ * attachments aren't recorded in smmu_domain yet.
+ */
+int arm_smmu_atc_inv_domain_ssid(struct arm_smmu_domain *smmu_domain, int ssid,
+ unsigned long iova, size_t size)
{
int i;
unsigned long flags;
struct arm_smmu_cmdq_ent cmd;
struct arm_smmu_master *master;
+ struct arm_smmu_attached_domain *attached_domain;
struct arm_smmu_cmdq_batch cmds;
if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
@@ -1837,25 +1849,37 @@ int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
if (!atomic_read(&smmu_domain->nr_ats_masters))
return 0;
- arm_smmu_atc_inv_to_cmd(ssid, iova, size, &cmd);
+ arm_smmu_atc_inv_to_cmd(iova, size, &cmd);
cmds.num = 0;
- spin_lock_irqsave(&smmu_domain->devices_lock, flags);
- list_for_each_entry(master, &smmu_domain->devices, domain_head) {
+ spin_lock_irqsave(&smmu_domain->attached_ssids_lock, flags);
+ list_for_each_entry(attached_domain, &smmu_domain->attached_ssids,
+ list) {
+ master = attached_domain->master;
if (!master->ats_enabled)
continue;
+ if (ssid != 0)
+ arm_smmu_atc_inv_cmd_set_ssid(ssid, &cmd);
+ else
+ arm_smmu_atc_inv_cmd_set_ssid(attached_domain->ssid, &cmd);
for (i = 0; i < master->num_streams; i++) {
cmd.atc.sid = master->streams[i].id;
arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
}
}
- spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+ spin_unlock_irqrestore(&smmu_domain->attached_ssids_lock, flags);
return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
}
+static int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
+ unsigned long iova, size_t size)
+{
+ return arm_smmu_atc_inv_domain_ssid(smmu_domain, 0, iova, size);
+}
+
/* IO_PGTABLE API */
static void arm_smmu_tlb_inv_context(void *cookie)
{
@@ -1877,7 +1901,7 @@ static void arm_smmu_tlb_inv_context(void *cookie)
cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
}
- arm_smmu_atc_inv_domain(smmu_domain, 0, 0, 0);
+ arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
}
static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
@@ -1970,7 +1994,7 @@ static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
* Unfortunately, this can't be leaf-only since we may have
* zapped an entire table.
*/
- arm_smmu_atc_inv_domain(smmu_domain, 0, iova, size);
+ arm_smmu_atc_inv_domain(smmu_domain, iova, size);
}
void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
@@ -2050,8 +2074,8 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
return NULL;
mutex_init(&smmu_domain->init_mutex);
- INIT_LIST_HEAD(&smmu_domain->devices);
- spin_lock_init(&smmu_domain->devices_lock);
+ INIT_LIST_HEAD(&smmu_domain->attached_ssids);
+ spin_lock_init(&smmu_domain->attached_ssids_lock);
INIT_LIST_HEAD(&smmu_domain->mmu_notifiers);
return &smmu_domain->domain;
@@ -2289,12 +2313,12 @@ static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
}
-static void arm_smmu_enable_ats(struct arm_smmu_master *master)
+static void arm_smmu_enable_ats(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain)
{
size_t stu;
struct pci_dev *pdev;
struct arm_smmu_device *smmu = master->smmu;
- struct arm_smmu_domain *smmu_domain = master->domain;
/* Don't enable ATS at the endpoint if it's not enabled in the STE */
if (!master->ats_enabled)
@@ -2310,10 +2334,9 @@ static void arm_smmu_enable_ats(struct arm_smmu_master *master)
dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
}
-static void arm_smmu_disable_ats(struct arm_smmu_master *master)
+static void arm_smmu_disable_ats(struct arm_smmu_master *master,
+ struct arm_smmu_domain *smmu_domain)
{
- struct arm_smmu_domain *smmu_domain = master->domain;
-
if (!master->ats_enabled)
return;
@@ -2377,19 +2400,19 @@ static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
static void arm_smmu_detach_dev(struct arm_smmu_master *master)
{
unsigned long flags;
- struct arm_smmu_domain *smmu_domain = master->domain;
+ struct arm_smmu_domain *smmu_domain = master->non_pasid_domain.domain;
if (!smmu_domain)
return;
- arm_smmu_disable_ats(master);
+ arm_smmu_disable_ats(master, smmu_domain);
- spin_lock_irqsave(&smmu_domain->devices_lock, flags);
- list_del(&master->domain_head);
- spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+ spin_lock_irqsave(&smmu_domain->attached_ssids_lock, flags);
+ list_del(&master->non_pasid_domain.list);
+ spin_unlock_irqrestore(&smmu_domain->attached_ssids_lock, flags);
- master->domain = NULL;
master->ats_enabled = false;
+ master->non_pasid_domain.domain = NULL;
arm_smmu_install_ste_for_dev(master);
}
@@ -2434,8 +2457,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
if (ret)
return ret;
- master->domain = smmu_domain;
-
/*
* The SMMU does not support enabling ATS with bypass. When the STE is
* in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests and
@@ -2449,26 +2470,26 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
if (!master->cd_table.cdtab) {
ret = arm_smmu_alloc_cd_tables(master);
- if (ret) {
- master->domain = NULL;
+ if (ret)
return ret;
- }
}
ret = arm_smmu_write_ctx_desc(master, 0, &smmu_domain->cd);
- if (ret) {
- master->domain = NULL;
+ if (ret)
return ret;
- }
}
+ master->non_pasid_domain.master = master;
+ master->non_pasid_domain.domain = smmu_domain;
+ master->non_pasid_domain.ssid = 0;
arm_smmu_install_ste_for_dev(master);
- spin_lock_irqsave(&smmu_domain->devices_lock, flags);
- list_add(&master->domain_head, &smmu_domain->devices);
- spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
+ spin_lock_irqsave(&smmu_domain->attached_ssids_lock, flags);
+ list_add(&master->non_pasid_domain.list,
+ &smmu_domain->attached_ssids);
+ spin_unlock_irqrestore(&smmu_domain->attached_ssids_lock, flags);
- arm_smmu_enable_ats(master);
+ arm_smmu_enable_ats(master, smmu_domain);
return 0;
}
@@ -689,11 +689,19 @@ struct arm_smmu_stream {
struct rb_node node;
};
+/* List of {masters, ssid} that a domain is attached to */
+struct arm_smmu_attached_domain {
+ struct list_head list;
+ struct arm_smmu_domain *domain;
+ struct arm_smmu_master *master;
+ int ssid;
+};
+
/* SMMU private data for each master */
struct arm_smmu_master {
struct arm_smmu_device *smmu;
struct device *dev;
- struct arm_smmu_domain *domain;
+ struct arm_smmu_attached_domain non_pasid_domain;
struct list_head domain_head;
struct arm_smmu_stream *streams;
/* Locked by the iommu core using the group mutex */
@@ -730,8 +738,12 @@ struct arm_smmu_domain {
struct iommu_domain domain;
- struct list_head devices;
- spinlock_t devices_lock;
+ /*
+ * List of arm_smmu_attached_domain nodes used to track all the
+ * {master, ssid} pairs that this domain is attached to.
+ */
+ struct list_head attached_ssids;
+ spinlock_t attached_ssids_lock;
struct list_head mmu_notifiers;
};
@@ -752,8 +764,8 @@ void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
size_t granule, bool leaf,
struct arm_smmu_domain *smmu_domain);
bool arm_smmu_free_asid(struct arm_smmu_ctx_desc *cd);
-int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain, int ssid,
- unsigned long iova, size_t size);
+int arm_smmu_atc_inv_domain_ssid(struct arm_smmu_domain *smmu_domain, int ssid,
+ unsigned long iova, size_t size);
#ifdef CONFIG_ARM_SMMU_V3_SVA
bool arm_smmu_sva_supported(struct arm_smmu_device *smmu);