@@ -353,6 +353,15 @@ struct kvm_arm_counter_offset {
#define KVM_ARM64_SVE_VLS_WORDS \
((KVM_ARM64_SVE_VQ_MAX - KVM_ARM64_SVE_VQ_MIN) / 64 + 1)
+/* SME registers */
+#define KVM_REG_ARM64_SME (0x17 << KVM_REG_ARM_COPROC_SHIFT)
+
+/* Vector lengths pseudo-register: */
+#define KVM_REG_ARM64_SME_VLS (KVM_REG_ARM64 | KVM_REG_ARM64_SME | \
+ KVM_REG_SIZE_U512 | 0xffff)
+#define KVM_ARM64_SME_VLS_WORDS \
+ ((KVM_ARM64_SVE_VQ_MAX - KVM_ARM64_SVE_VQ_MIN) / 64 + 1)
+
/* Bitmap feature firmware registers */
#define KVM_REG_ARM_FW_FEAT_BMAP (0x0016 << KVM_REG_ARM_COPROC_SHIFT)
#define KVM_REG_ARM_FW_FEAT_BMAP_REG(r) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \
@@ -309,22 +309,20 @@ static int set_core_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
#define vq_mask(vq) ((u64)1 << ((vq) - SVE_VQ_MIN) % 64)
#define vq_present(vqs, vq) (!!((vqs)[vq_word(vq)] & vq_mask(vq)))
-static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+static int get_vec_vls(enum vec_type vec_type, struct kvm_vcpu *vcpu,
+ const struct kvm_one_reg *reg)
{
unsigned int max_vq, vq;
u64 vqs[KVM_ARM64_SVE_VLS_WORDS];
- if (!vcpu_has_sve(vcpu))
- return -ENOENT;
-
- if (WARN_ON(!sve_vl_valid(vcpu->arch.max_vl[ARM64_VEC_SVE])))
+ if (WARN_ON(!sve_vl_valid(vcpu->arch.max_vl[vec_type])))
return -EINVAL;
memset(vqs, 0, sizeof(vqs));
- max_vq = vcpu_sve_max_vq(vcpu);
+ max_vq = vcpu_vec_max_vq(vec_type, vcpu);
for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
- if (sve_vq_available(vq))
+ if (vq_available(vec_type, vq))
vqs[vq_word(vq)] |= vq_mask(vq);
if (copy_to_user((void __user *)reg->addr, vqs, sizeof(vqs)))
@@ -333,18 +331,13 @@ static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
return 0;
}
-static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+static int set_vec_vls(enum vec_type vec_type, struct kvm_vcpu *vcpu,
+ const struct kvm_one_reg *reg)
{
unsigned int max_vq, vq;
u64 vqs[KVM_ARM64_SVE_VLS_WORDS];
- if (!vcpu_has_sve(vcpu))
- return -ENOENT;
-
- if (kvm_arm_vcpu_vec_finalized(vcpu))
- return -EPERM; /* too late! */
-
- if (WARN_ON(vcpu->arch.sve_state))
+ if (WARN_ON(!sve_vl_valid(vcpu->arch.max_vl[vec_type])))
return -EINVAL;
if (copy_from_user(vqs, (const void __user *)reg->addr, sizeof(vqs)))
@@ -355,18 +348,18 @@ static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
if (vq_present(vqs, vq))
max_vq = vq;
- if (max_vq > sve_vq_from_vl(kvm_vec_max_vl[ARM64_VEC_SVE]))
+ if (max_vq > sve_vq_from_vl(kvm_vec_max_vl[vec_type]))
return -EINVAL;
/*
* Vector lengths supported by the host can't currently be
* hidden from the guest individually: instead we can only set a
- * maximum via ZCR_EL2.LEN. So, make sure the available vector
+ * maximum via xCR_EL2.LEN. So, make sure the available vector
* lengths match the set requested exactly up to the requested
* maximum:
*/
for (vq = SVE_VQ_MIN; vq <= max_vq; ++vq)
- if (vq_present(vqs, vq) != sve_vq_available(vq))
+ if (vq_present(vqs, vq) != vq_available(vec_type, vq))
return -EINVAL;
/* Can't run with no vector lengths at all: */
@@ -374,11 +367,33 @@ static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
return -EINVAL;
/* vcpu->arch.sve_state will be alloc'd by kvm_vcpu_finalize_sve() */
- vcpu->arch.max_vl[ARM64_VEC_SVE] = sve_vl_from_vq(max_vq);
+ vcpu->arch.max_vl[vec_type] = sve_vl_from_vq(max_vq);
return 0;
}
+static int get_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+ if (!vcpu_has_sve(vcpu))
+ return -ENOENT;
+
+ return get_vec_vls(ARM64_VEC_SVE, vcpu, reg);
+}
+
+static int set_sve_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+ if (!vcpu_has_sve(vcpu))
+ return -ENOENT;
+
+ if (kvm_arm_vcpu_vec_finalized(vcpu))
+ return -EPERM; /* too late! */
+
+ if (WARN_ON(vcpu->arch.sve_state))
+ return -EINVAL;
+
+ return set_vec_vls(ARM64_VEC_SVE, vcpu, reg);
+}
+
#define SVE_REG_SLICE_SHIFT 0
#define SVE_REG_SLICE_BITS 5
#define SVE_REG_ID_SHIFT (SVE_REG_SLICE_SHIFT + SVE_REG_SLICE_BITS)
@@ -532,6 +547,45 @@ static int set_sve_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
return 0;
}
+static int get_sme_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+ if (!vcpu_has_sme(vcpu))
+ return -ENOENT;
+
+ return get_vec_vls(ARM64_VEC_SME, vcpu, reg);
+}
+
+static int set_sme_vls(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+ if (!vcpu_has_sme(vcpu))
+ return -ENOENT;
+
+ if (kvm_arm_vcpu_vec_finalized(vcpu))
+ return -EPERM; /* too late! */
+
+ if (WARN_ON(vcpu->arch.sme_state))
+ return -EINVAL;
+
+ return set_vec_vls(ARM64_VEC_SME, vcpu, reg);
+}
+
+static int get_sme_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+ /* Handle the KVM_REG_ARM64_SME_VLS pseudo-reg as a special case: */
+ if (reg->id == KVM_REG_ARM64_SME_VLS)
+ return get_sme_vls(vcpu, reg);
+
+ return -EINVAL;
+}
+
+static int set_sme_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
+{
+ /* Handle the KVM_REG_ARM64_SME_VLS pseudo-reg as a special case: */
+ if (reg->id == KVM_REG_ARM64_SME_VLS)
+ return set_sme_vls(vcpu, reg);
+
+ return -EINVAL;
+}
int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
{
return -EINVAL;
@@ -771,6 +825,7 @@ int kvm_arm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
case KVM_REG_ARM_FW_FEAT_BMAP:
return kvm_arm_get_fw_reg(vcpu, reg);
case KVM_REG_ARM64_SVE: return get_sve_reg(vcpu, reg);
+ case KVM_REG_ARM64_SME: return get_sme_reg(vcpu, reg);
}
if (is_timer_reg(reg->id))
@@ -791,6 +846,7 @@ int kvm_arm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg)
case KVM_REG_ARM_FW_FEAT_BMAP:
return kvm_arm_set_fw_reg(vcpu, reg);
case KVM_REG_ARM64_SVE: return set_sve_reg(vcpu, reg);
+ case KVM_REG_ARM64_SME: return set_sme_reg(vcpu, reg);
}
if (is_timer_reg(reg->id))