[2/3] vfio: Add VFIO_DEVICE_PASID_[AT|DE]TACH_IOMMUFD_PT

  This adds ioctls for the userspace to attach a given pasid of a vfio
device to/from an IOAS/HWPT.

Signed-off-by: Yi Liu <yi.l.liu@intel.com>
---
 drivers/vfio/device_cdev.c | 45 +++++++++++++++++++++++++++++++
 drivers/vfio/vfio.h        |  4 +++
 drivers/vfio/vfio_main.c   |  8 ++++++
 include/uapi/linux/vfio.h  | 55 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 112 insertions(+)
  

>-----Original Message-----
>From: Liu, Yi L <yi.l.liu@intel.com>
>Sent: Monday, November 27, 2023 2:39 PM
>Subject: [PATCH 2/3] vfio: Add
>VFIO_DEVICE_PASID_[AT|DE]TACH_IOMMUFD_PT
>
>This adds ioctls for the userspace to attach a given pasid of a vfio
>device to/from an IOAS/HWPT.
>
>Signed-off-by: Yi Liu <yi.l.liu@intel.com>
>---
> drivers/vfio/device_cdev.c | 45 +++++++++++++++++++++++++++++++
> drivers/vfio/vfio.h        |  4 +++
> drivers/vfio/vfio_main.c   |  8 ++++++
> include/uapi/linux/vfio.h  | 55 ++++++++++++++++++++++++++++++++++++++
> 4 files changed, 112 insertions(+)
>
>diff --git a/drivers/vfio/device_cdev.c b/drivers/vfio/device_cdev.c
>index e75da0a70d1f..c2ac7ed44537 100644
>--- a/drivers/vfio/device_cdev.c
>+++ b/drivers/vfio/device_cdev.c
>@@ -210,6 +210,51 @@ int vfio_df_ioctl_detach_pt(struct vfio_device_file *df,
> 	return 0;
> }
>
>+int vfio_df_ioctl_pasid_attach_pt(struct vfio_device_file *df,
>+				  struct vfio_device_pasid_attach_iommufd_pt
>__user *arg)
>+{
>+	struct vfio_device *device = df->device;
>+	struct vfio_device_pasid_attach_iommufd_pt attach;
>+	unsigned long minsz;
>+	int ret;
>+
>+	minsz = offsetofend(struct vfio_device_pasid_attach_iommufd_pt, pt_id);
>+
>+	if (copy_from_user(&attach, arg, minsz))
>+		return -EFAULT;
>+
>+	if (attach.argsz < minsz || attach.flags)
>+		return -EINVAL;
>+
>+	mutex_lock(&device->dev_set->lock);
>+	ret = device->ops->pasid_attach_ioas(device, attach.pasid,
>&attach.pt_id);
>+	mutex_unlock(&device->dev_set->lock);
>+
>+	return ret;
>+}
>+
>+int vfio_df_ioctl_pasid_detach_pt(struct vfio_device_file *df,
>+				  struct vfio_device_pasid_detach_iommufd_pt
>__user *arg)
>+{
>+	struct vfio_device *device = df->device;
>+	struct vfio_device_pasid_detach_iommufd_pt detach;
>+	unsigned long minsz;
>+
>+	minsz = offsetofend(struct vfio_device_pasid_detach_iommufd_pt, flags);

Pasid isn't copied, should use pasid here?

Thanks
Zhenzhong
  

On 2023/11/27 14:50, Duan, Zhenzhong wrote:
> 
> 
>> -----Original Message-----
>> From: Liu, Yi L <yi.l.liu@intel.com>
>> Sent: Monday, November 27, 2023 2:39 PM
>> Subject: [PATCH 2/3] vfio: Add
>> VFIO_DEVICE_PASID_[AT|DE]TACH_IOMMUFD_PT
>>
>> This adds ioctls for the userspace to attach a given pasid of a vfio
>> device to/from an IOAS/HWPT.
>>
>> Signed-off-by: Yi Liu <yi.l.liu@intel.com>
>> ---
>> drivers/vfio/device_cdev.c | 45 +++++++++++++++++++++++++++++++
>> drivers/vfio/vfio.h        |  4 +++
>> drivers/vfio/vfio_main.c   |  8 ++++++
>> include/uapi/linux/vfio.h  | 55 ++++++++++++++++++++++++++++++++++++++
>> 4 files changed, 112 insertions(+)
>>
>> diff --git a/drivers/vfio/device_cdev.c b/drivers/vfio/device_cdev.c
>> index e75da0a70d1f..c2ac7ed44537 100644
>> --- a/drivers/vfio/device_cdev.c
>> +++ b/drivers/vfio/device_cdev.c
>> @@ -210,6 +210,51 @@ int vfio_df_ioctl_detach_pt(struct vfio_device_file *df,
>> 	return 0;
>> }
>>
>> +int vfio_df_ioctl_pasid_attach_pt(struct vfio_device_file *df,
>> +				  struct vfio_device_pasid_attach_iommufd_pt
>> __user *arg)
>> +{
>> +	struct vfio_device *device = df->device;
>> +	struct vfio_device_pasid_attach_iommufd_pt attach;
>> +	unsigned long minsz;
>> +	int ret;
>> +
>> +	minsz = offsetofend(struct vfio_device_pasid_attach_iommufd_pt, pt_id);
>> +
>> +	if (copy_from_user(&attach, arg, minsz))
>> +		return -EFAULT;
>> +
>> +	if (attach.argsz < minsz || attach.flags)
>> +		return -EINVAL;
>> +
>> +	mutex_lock(&device->dev_set->lock);
>> +	ret = device->ops->pasid_attach_ioas(device, attach.pasid,
>> &attach.pt_id);
>> +	mutex_unlock(&device->dev_set->lock);
>> +
>> +	return ret;
>> +}
>> +
>> +int vfio_df_ioctl_pasid_detach_pt(struct vfio_device_file *df,
>> +				  struct vfio_device_pasid_detach_iommufd_pt
>> __user *arg)
>> +{
>> +	struct vfio_device *device = df->device;
>> +	struct vfio_device_pasid_detach_iommufd_pt detach;
>> +	unsigned long minsz;
>> +
>> +	minsz = offsetofend(struct vfio_device_pasid_detach_iommufd_pt, flags);
> 
> Pasid isn't copied, should use pasid here?

good catch! will fix it.
  

On Sun, 26 Nov 2023 22:39:08 -0800
Yi Liu <yi.l.liu@intel.com> wrote:

> This adds ioctls for the userspace to attach a given pasid of a vfio
> device to/from an IOAS/HWPT.
> 
> Signed-off-by: Yi Liu <yi.l.liu@intel.com>
> ---
>  drivers/vfio/device_cdev.c | 45 +++++++++++++++++++++++++++++++
>  drivers/vfio/vfio.h        |  4 +++
>  drivers/vfio/vfio_main.c   |  8 ++++++
>  include/uapi/linux/vfio.h  | 55 ++++++++++++++++++++++++++++++++++++++
>  4 files changed, 112 insertions(+)
> 
> diff --git a/drivers/vfio/device_cdev.c b/drivers/vfio/device_cdev.c
> index e75da0a70d1f..c2ac7ed44537 100644
> --- a/drivers/vfio/device_cdev.c
> +++ b/drivers/vfio/device_cdev.c
> @@ -210,6 +210,51 @@ int vfio_df_ioctl_detach_pt(struct vfio_device_file *df,
>  	return 0;
>  }
>  
> +int vfio_df_ioctl_pasid_attach_pt(struct vfio_device_file *df,
> +				  struct vfio_device_pasid_attach_iommufd_pt __user *arg)
> +{
> +	struct vfio_device *device = df->device;
> +	struct vfio_device_pasid_attach_iommufd_pt attach;
> +	unsigned long minsz;
> +	int ret;
> +
> +	minsz = offsetofend(struct vfio_device_pasid_attach_iommufd_pt, pt_id);
> +
> +	if (copy_from_user(&attach, arg, minsz))
> +		return -EFAULT;
> +
> +	if (attach.argsz < minsz || attach.flags)
> +		return -EINVAL;
> +
> +	mutex_lock(&device->dev_set->lock);
> +	ret = device->ops->pasid_attach_ioas(device, attach.pasid, &attach.pt_id);

These callbacks were only implemented for vfio-pci in the previous
patch but they're called unconditionally.  Thanks,

Alex

> +	mutex_unlock(&device->dev_set->lock);
> +
> +	return ret;
> +}
> +
> +int vfio_df_ioctl_pasid_detach_pt(struct vfio_device_file *df,
> +				  struct vfio_device_pasid_detach_iommufd_pt __user *arg)
> +{
> +	struct vfio_device *device = df->device;
> +	struct vfio_device_pasid_detach_iommufd_pt detach;
> +	unsigned long minsz;
> +
> +	minsz = offsetofend(struct vfio_device_pasid_detach_iommufd_pt, flags);
> +
> +	if (copy_from_user(&detach, arg, minsz))
> +		return -EFAULT;
> +
> +	if (detach.argsz < minsz || detach.flags)
> +		return -EINVAL;
> +
> +	mutex_lock(&device->dev_set->lock);
> +	device->ops->pasid_detach_ioas(device, detach.pasid);
> +	mutex_unlock(&device->dev_set->lock);
> +
> +	return 0;
> +}
> +
>  static char *vfio_device_devnode(const struct device *dev, umode_t *mode)
>  {
>  	return kasprintf(GFP_KERNEL, "vfio/devices/%s", dev_name(dev));
> diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h
> index 307e3f29b527..d228cdb6b345 100644
> --- a/drivers/vfio/vfio.h
> +++ b/drivers/vfio/vfio.h
> @@ -353,6 +353,10 @@ int vfio_df_ioctl_attach_pt(struct vfio_device_file *df,
>  			    struct vfio_device_attach_iommufd_pt __user *arg);
>  int vfio_df_ioctl_detach_pt(struct vfio_device_file *df,
>  			    struct vfio_device_detach_iommufd_pt __user *arg);
> +int vfio_df_ioctl_pasid_attach_pt(struct vfio_device_file *df,
> +				  struct vfio_device_pasid_attach_iommufd_pt __user *arg);
> +int vfio_df_ioctl_pasid_detach_pt(struct vfio_device_file *df,
> +				  struct vfio_device_pasid_detach_iommufd_pt __user *arg);
>  
>  #if IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV)
>  void vfio_init_device_cdev(struct vfio_device *device);
> diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c
> index 8d4995ada74a..ff50c239873d 100644
> --- a/drivers/vfio/vfio_main.c
> +++ b/drivers/vfio/vfio_main.c
> @@ -1240,6 +1240,14 @@ static long vfio_device_fops_unl_ioctl(struct file *filep,
>  		case VFIO_DEVICE_DETACH_IOMMUFD_PT:
>  			ret = vfio_df_ioctl_detach_pt(df, uptr);
>  			goto out;
> +
> +		case VFIO_DEVICE_PASID_ATTACH_IOMMUFD_PT:
> +			ret = vfio_df_ioctl_pasid_attach_pt(df, uptr);
> +			goto out;
> +
> +		case VFIO_DEVICE_PASID_DETACH_IOMMUFD_PT:
> +			ret = vfio_df_ioctl_pasid_detach_pt(df, uptr);
> +			goto out;
>  		}
>  	}
>  
> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
> index 94b3badefde3..495193629029 100644
> --- a/include/uapi/linux/vfio.h
> +++ b/include/uapi/linux/vfio.h
> @@ -977,6 +977,61 @@ struct vfio_device_detach_iommufd_pt {
>  
>  #define VFIO_DEVICE_DETACH_IOMMUFD_PT		_IO(VFIO_TYPE, VFIO_BASE + 20)
>  
> +/*
> + * VFIO_DEVICE_PASID_ATTACH_IOMMUFD_PT - _IOW(VFIO_TYPE, VFIO_BASE + 21,
> + *					      struct vfio_device_pasid_attach_iommufd_pt)
> + * @argsz:	User filled size of this data.
> + * @flags:	Must be 0.
> + * @pasid:	The pasid to be attached.
> + * @pt_id:	Input the target id which can represent an ioas or a hwpt
> + *		allocated via iommufd subsystem.
> + *		Output the input ioas id or the attached hwpt id which could
> + *		be the specified hwpt itself or a hwpt automatically created
> + *		for the specified ioas by kernel during the attachment.
> + *
> + * Associate a pasid (of a cdev device) with an address space within the
> + * bound iommufd. Undo by VFIO_DEVICE_PASID_DETACH_IOMMUFD_PT or device fd
> + * close. This is only allowed on cdev fds.
> + *
> + * If a pasid is currently attached to a valid hw_pagetable (hwpt), without
> + * doing a VFIO_DEVICE_PASID_DETACH_IOMMUFD_PT, a second
> + * VFIO_DEVICE_PASID_ATTACH_IOMMUFD_PT ioctl passing in another hwpt id is
> + * allowed. This action, also known as a hwpt replacement, will replace the
> + * pasid's currently attached hwpt with a new hwpt corresponding to the given
> + * @pt_id.
> + *
> + * Return: 0 on success, -errno on failure.
> + */
> +struct vfio_device_pasid_attach_iommufd_pt {
> +	__u32	argsz;
> +	__u32	flags;
> +	__u32	pasid;
> +	__u32	pt_id;
> +};
> +
> +#define VFIO_DEVICE_PASID_ATTACH_IOMMUFD_PT	_IO(VFIO_TYPE, VFIO_BASE + 21)
> +
> +/*
> + * VFIO_DEVICE_PASID_DETACH_IOMMUFD_PT - _IOW(VFIO_TYPE, VFIO_BASE + 22,
> + *					      struct vfio_device_pasid_detach_iommufd_pt)
> + * @argsz:	User filled size of this data.
> + * @flags:	Must be 0.
> + * @pasid:	The pasid to be detached.
> + *
> + * Remove the association of a pasid (of a cdev device) and its current
> + * associated address space.  After it, the pasid of the device should be in
> + * a blocking DMA state.  This is only allowed on cdev fds.
> + *
> + * Return: 0 on success, -errno on failure.
> + */
> +struct vfio_device_pasid_detach_iommufd_pt {
> +	__u32	argsz;
> +	__u32	flags;
> +	__u32	pasid;
> +};
> +
> +#define VFIO_DEVICE_PASID_DETACH_IOMMUFD_PT	_IO(VFIO_TYPE, VFIO_BASE + 22)
> +
>  /*
>   * Provide support for setting a PCI VF Token, which is used as a shared
>   * secret between PF and VF drivers.  This feature may only be set on a
  

On 2023/12/12 01:05, Alex Williamson wrote:
> On Sun, 26 Nov 2023 22:39:08 -0800
> Yi Liu <yi.l.liu@intel.com> wrote:
> 
>> This adds ioctls for the userspace to attach a given pasid of a vfio
>> device to/from an IOAS/HWPT.
>>
>> Signed-off-by: Yi Liu <yi.l.liu@intel.com>
>> ---
>>   drivers/vfio/device_cdev.c | 45 +++++++++++++++++++++++++++++++
>>   drivers/vfio/vfio.h        |  4 +++
>>   drivers/vfio/vfio_main.c   |  8 ++++++
>>   include/uapi/linux/vfio.h  | 55 ++++++++++++++++++++++++++++++++++++++
>>   4 files changed, 112 insertions(+)
>>
>> diff --git a/drivers/vfio/device_cdev.c b/drivers/vfio/device_cdev.c
>> index e75da0a70d1f..c2ac7ed44537 100644
>> --- a/drivers/vfio/device_cdev.c
>> +++ b/drivers/vfio/device_cdev.c
>> @@ -210,6 +210,51 @@ int vfio_df_ioctl_detach_pt(struct vfio_device_file *df,
>>   	return 0;
>>   }
>>   
>> +int vfio_df_ioctl_pasid_attach_pt(struct vfio_device_file *df,
>> +				  struct vfio_device_pasid_attach_iommufd_pt __user *arg)
>> +{
>> +	struct vfio_device *device = df->device;
>> +	struct vfio_device_pasid_attach_iommufd_pt attach;
>> +	unsigned long minsz;
>> +	int ret;
>> +
>> +	minsz = offsetofend(struct vfio_device_pasid_attach_iommufd_pt, pt_id);
>> +
>> +	if (copy_from_user(&attach, arg, minsz))
>> +		return -EFAULT;
>> +
>> +	if (attach.argsz < minsz || attach.flags)
>> +		return -EINVAL;
>> +
>> +	mutex_lock(&device->dev_set->lock);
>> +	ret = device->ops->pasid_attach_ioas(device, attach.pasid, &attach.pt_id);
> 
> These callbacks were only implemented for vfio-pci in the previous
> patch but they're called unconditionally.  Thanks,

yes, will correct it and below. thanks for catching it.

> 
> Alex
> 
>> +	mutex_unlock(&device->dev_set->lock);
>> +
>> +	return ret;
>> +}
>> +
>> +int vfio_df_ioctl_pasid_detach_pt(struct vfio_device_file *df,
>> +				  struct vfio_device_pasid_detach_iommufd_pt __user *arg)
>> +{
>> +	struct vfio_device *device = df->device;
>> +	struct vfio_device_pasid_detach_iommufd_pt detach;
>> +	unsigned long minsz;
>> +
>> +	minsz = offsetofend(struct vfio_device_pasid_detach_iommufd_pt, flags);
>> +
>> +	if (copy_from_user(&detach, arg, minsz))
>> +		return -EFAULT;
>> +
>> +	if (detach.argsz < minsz || detach.flags)
>> +		return -EINVAL;
>> +
>> +	mutex_lock(&device->dev_set->lock);
>> +	device->ops->pasid_detach_ioas(device, detach.pasid);
>> +	mutex_unlock(&device->dev_set->lock);
>> +
>> +	return 0;
>> +}
>> +
>>   static char *vfio_device_devnode(const struct device *dev, umode_t *mode)
>>   {
>>   	return kasprintf(GFP_KERNEL, "vfio/devices/%s", dev_name(dev));
>> diff --git a/drivers/vfio/vfio.h b/drivers/vfio/vfio.h
>> index 307e3f29b527..d228cdb6b345 100644
>> --- a/drivers/vfio/vfio.h
>> +++ b/drivers/vfio/vfio.h
>> @@ -353,6 +353,10 @@ int vfio_df_ioctl_attach_pt(struct vfio_device_file *df,
>>   			    struct vfio_device_attach_iommufd_pt __user *arg);
>>   int vfio_df_ioctl_detach_pt(struct vfio_device_file *df,
>>   			    struct vfio_device_detach_iommufd_pt __user *arg);
>> +int vfio_df_ioctl_pasid_attach_pt(struct vfio_device_file *df,
>> +				  struct vfio_device_pasid_attach_iommufd_pt __user *arg);
>> +int vfio_df_ioctl_pasid_detach_pt(struct vfio_device_file *df,
>> +				  struct vfio_device_pasid_detach_iommufd_pt __user *arg);
>>   
>>   #if IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV)
>>   void vfio_init_device_cdev(struct vfio_device *device);
>> diff --git a/drivers/vfio/vfio_main.c b/drivers/vfio/vfio_main.c
>> index 8d4995ada74a..ff50c239873d 100644
>> --- a/drivers/vfio/vfio_main.c
>> +++ b/drivers/vfio/vfio_main.c
>> @@ -1240,6 +1240,14 @@ static long vfio_device_fops_unl_ioctl(struct file *filep,
>>   		case VFIO_DEVICE_DETACH_IOMMUFD_PT:
>>   			ret = vfio_df_ioctl_detach_pt(df, uptr);
>>   			goto out;
>> +
>> +		case VFIO_DEVICE_PASID_ATTACH_IOMMUFD_PT:
>> +			ret = vfio_df_ioctl_pasid_attach_pt(df, uptr);
>> +			goto out;
>> +
>> +		case VFIO_DEVICE_PASID_DETACH_IOMMUFD_PT:
>> +			ret = vfio_df_ioctl_pasid_detach_pt(df, uptr);
>> +			goto out;
>>   		}
>>   	}
>>   
>> diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h
>> index 94b3badefde3..495193629029 100644
>> --- a/include/uapi/linux/vfio.h
>> +++ b/include/uapi/linux/vfio.h
>> @@ -977,6 +977,61 @@ struct vfio_device_detach_iommufd_pt {
>>   
>>   #define VFIO_DEVICE_DETACH_IOMMUFD_PT		_IO(VFIO_TYPE, VFIO_BASE + 20)
>>   
>> +/*
>> + * VFIO_DEVICE_PASID_ATTACH_IOMMUFD_PT - _IOW(VFIO_TYPE, VFIO_BASE + 21,
>> + *					      struct vfio_device_pasid_attach_iommufd_pt)
>> + * @argsz:	User filled size of this data.
>> + * @flags:	Must be 0.
>> + * @pasid:	The pasid to be attached.
>> + * @pt_id:	Input the target id which can represent an ioas or a hwpt
>> + *		allocated via iommufd subsystem.
>> + *		Output the input ioas id or the attached hwpt id which could
>> + *		be the specified hwpt itself or a hwpt automatically created
>> + *		for the specified ioas by kernel during the attachment.
>> + *
>> + * Associate a pasid (of a cdev device) with an address space within the
>> + * bound iommufd. Undo by VFIO_DEVICE_PASID_DETACH_IOMMUFD_PT or device fd
>> + * close. This is only allowed on cdev fds.
>> + *
>> + * If a pasid is currently attached to a valid hw_pagetable (hwpt), without
>> + * doing a VFIO_DEVICE_PASID_DETACH_IOMMUFD_PT, a second
>> + * VFIO_DEVICE_PASID_ATTACH_IOMMUFD_PT ioctl passing in another hwpt id is
>> + * allowed. This action, also known as a hwpt replacement, will replace the
>> + * pasid's currently attached hwpt with a new hwpt corresponding to the given
>> + * @pt_id.
>> + *
>> + * Return: 0 on success, -errno on failure.
>> + */
>> +struct vfio_device_pasid_attach_iommufd_pt {
>> +	__u32	argsz;
>> +	__u32	flags;
>> +	__u32	pasid;
>> +	__u32	pt_id;
>> +};
>> +
>> +#define VFIO_DEVICE_PASID_ATTACH_IOMMUFD_PT	_IO(VFIO_TYPE, VFIO_BASE + 21)
>> +
>> +/*
>> + * VFIO_DEVICE_PASID_DETACH_IOMMUFD_PT - _IOW(VFIO_TYPE, VFIO_BASE + 22,
>> + *					      struct vfio_device_pasid_detach_iommufd_pt)
>> + * @argsz:	User filled size of this data.
>> + * @flags:	Must be 0.
>> + * @pasid:	The pasid to be detached.
>> + *
>> + * Remove the association of a pasid (of a cdev device) and its current
>> + * associated address space.  After it, the pasid of the device should be in
>> + * a blocking DMA state.  This is only allowed on cdev fds.
>> + *
>> + * Return: 0 on success, -errno on failure.
>> + */
>> +struct vfio_device_pasid_detach_iommufd_pt {
>> +	__u32	argsz;
>> +	__u32	flags;
>> +	__u32	pasid;
>> +};
>> +
>> +#define VFIO_DEVICE_PASID_DETACH_IOMMUFD_PT	_IO(VFIO_TYPE, VFIO_BASE + 22)
>> +
>>   /*
>>    * Provide support for setting a PCI VF Token, which is used as a shared
>>    * secret between PF and VF drivers.  This feature may only be set on a
>