[v5,4/4] vdpa_sim: Implement resume vdpa op

  From: Sebastien Boeuf <sebastien.boeuf@intel.com>

Implement resume operation for vdpa_sim devices, so vhost-vdpa will
offer that backend feature and userspace can effectively resume the
device.

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
---
 drivers/vdpa/vdpa_sim/vdpa_sim.c | 28 ++++++++++++++++++++++++++++
 drivers/vdpa/vdpa_sim/vdpa_sim.h |  1 +
 2 files changed, 29 insertions(+)
  

On Wed, Oct 26, 2022 at 11:09 PM <sebastien.boeuf@intel.com> wrote:
>
> From: Sebastien Boeuf <sebastien.boeuf@intel.com>
>
> Implement resume operation for vdpa_sim devices, so vhost-vdpa will
> offer that backend feature and userspace can effectively resume the
> device.
>
> Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
> ---
>  drivers/vdpa/vdpa_sim/vdpa_sim.c | 28 ++++++++++++++++++++++++++++
>  drivers/vdpa/vdpa_sim/vdpa_sim.h |  1 +
>  2 files changed, 29 insertions(+)
>
> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> index b071f0d842fb..84fee8bb2929 100644
> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> @@ -357,6 +357,11 @@ static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
>         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
>         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
>
> +       if (!vdpasim->running) {
> +               vdpasim->pending_kick = true;
> +               return;

I think we may hit here when the driver kicks vq before DRIVER_OK. Do
we need to check device status in this case and resume?

Thanks

> +       }
> +
>         if (vq->ready)
>                 schedule_work(&vdpasim->work);
>  }
> @@ -527,6 +532,27 @@ static int vdpasim_suspend(struct vdpa_device *vdpa)
>         return 0;
>  }
>
> +static int vdpasim_resume(struct vdpa_device *vdpa)
> +{
> +       struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> +       int i;
> +
> +       spin_lock(&vdpasim->lock);
> +       vdpasim->running = true;
> +
> +       if (vdpasim->pending_kick) {
> +               /* Process pending descriptors */
> +               for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)
> +                       vdpasim_kick_vq(vdpa, i);
> +
> +               vdpasim->pending_kick = false;
> +       }
> +
> +       spin_unlock(&vdpasim->lock);
> +
> +       return 0;
> +}
> +
>  static size_t vdpasim_get_config_size(struct vdpa_device *vdpa)
>  {
>         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> @@ -717,6 +743,7 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
>         .set_status             = vdpasim_set_status,
>         .reset                  = vdpasim_reset,
>         .suspend                = vdpasim_suspend,
> +       .resume                 = vdpasim_resume,
>         .get_config_size        = vdpasim_get_config_size,
>         .get_config             = vdpasim_get_config,
>         .set_config             = vdpasim_set_config,
> @@ -750,6 +777,7 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
>         .set_status             = vdpasim_set_status,
>         .reset                  = vdpasim_reset,
>         .suspend                = vdpasim_suspend,
> +       .resume                 = vdpasim_resume,
>         .get_config_size        = vdpasim_get_config_size,
>         .get_config             = vdpasim_get_config,
>         .set_config             = vdpasim_set_config,
> diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> index 0e78737dcc16..a745605589e2 100644
> --- a/drivers/vdpa/vdpa_sim/vdpa_sim.h
> +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> @@ -67,6 +67,7 @@ struct vdpasim {
>         u64 features;
>         u32 groups;
>         bool running;
> +       bool pending_kick;
>         /* spinlock to synchronize iommu table */
>         spinlock_t iommu_lock;
>  };
> --
> 2.34.1
>
> ---------------------------------------------------------------------
> Intel Corporation SAS (French simplified joint stock company)
> Registered headquarters: "Les Montalets"- 2, rue de Paris,
> 92196 Meudon Cedex, France
> Registration Number:  302 456 199 R.C.S. NANTERRE
> Capital: 5 208 026.16 Euros
>
> This e-mail and any attachments may contain confidential material for
> the sole use of the intended recipient(s). Any review or distribution
> by others is strictly prohibited. If you are not the intended
> recipient, please contact the sender and delete all copies.
>
  

On Mon, Nov 07, 2022 at 03:43:54PM +0800, Jason Wang wrote:
> On Wed, Oct 26, 2022 at 11:09 PM <sebastien.boeuf@intel.com> wrote:
> >
> > From: Sebastien Boeuf <sebastien.boeuf@intel.com>
> >
> > Implement resume operation for vdpa_sim devices, so vhost-vdpa will
> > offer that backend feature and userspace can effectively resume the
> > device.
> >
> > Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
> > ---
> >  drivers/vdpa/vdpa_sim/vdpa_sim.c | 28 ++++++++++++++++++++++++++++
> >  drivers/vdpa/vdpa_sim/vdpa_sim.h |  1 +
> >  2 files changed, 29 insertions(+)
> >
> > diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > index b071f0d842fb..84fee8bb2929 100644
> > --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > @@ -357,6 +357,11 @@ static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
> >         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> >         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
> >
> > +       if (!vdpasim->running) {
> > +               vdpasim->pending_kick = true;
> > +               return;
> 
> I think we may hit here when the driver kicks vq before DRIVER_OK. Do
> we need to check device status in this case and resume?
> 
> Thanks

Sebastien did you forget to reply here?

> > +       }
> > +
> >         if (vq->ready)
> >                 schedule_work(&vdpasim->work);
> >  }
> > @@ -527,6 +532,27 @@ static int vdpasim_suspend(struct vdpa_device *vdpa)
> >         return 0;
> >  }
> >
> > +static int vdpasim_resume(struct vdpa_device *vdpa)
> > +{
> > +       struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> > +       int i;
> > +
> > +       spin_lock(&vdpasim->lock);
> > +       vdpasim->running = true;
> > +
> > +       if (vdpasim->pending_kick) {
> > +               /* Process pending descriptors */
> > +               for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)
> > +                       vdpasim_kick_vq(vdpa, i);
> > +
> > +               vdpasim->pending_kick = false;
> > +       }
> > +
> > +       spin_unlock(&vdpasim->lock);
> > +
> > +       return 0;
> > +}
> > +
> >  static size_t vdpasim_get_config_size(struct vdpa_device *vdpa)
> >  {
> >         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> > @@ -717,6 +743,7 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
> >         .set_status             = vdpasim_set_status,
> >         .reset                  = vdpasim_reset,
> >         .suspend                = vdpasim_suspend,
> > +       .resume                 = vdpasim_resume,
> >         .get_config_size        = vdpasim_get_config_size,
> >         .get_config             = vdpasim_get_config,
> >         .set_config             = vdpasim_set_config,
> > @@ -750,6 +777,7 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
> >         .set_status             = vdpasim_set_status,
> >         .reset                  = vdpasim_reset,
> >         .suspend                = vdpasim_suspend,
> > +       .resume                 = vdpasim_resume,
> >         .get_config_size        = vdpasim_get_config_size,
> >         .get_config             = vdpasim_get_config,
> >         .set_config             = vdpasim_set_config,
> > diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> > index 0e78737dcc16..a745605589e2 100644
> > --- a/drivers/vdpa/vdpa_sim/vdpa_sim.h
> > +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> > @@ -67,6 +67,7 @@ struct vdpasim {
> >         u64 features;
> >         u32 groups;
> >         bool running;
> > +       bool pending_kick;
> >         /* spinlock to synchronize iommu table */
> >         spinlock_t iommu_lock;
> >  };
> > --
> > 2.34.1
> >
> > ---------------------------------------------------------------------
> > Intel Corporation SAS (French simplified joint stock company)
> > Registered headquarters: "Les Montalets"- 2, rue de Paris,
> > 92196 Meudon Cedex, France
> > Registration Number:  302 456 199 R.C.S. NANTERRE
> > Capital: 5 208 026.16 Euros
> >
> > This e-mail and any attachments may contain confidential material for
> > the sole use of the intended recipient(s). Any review or distribution
> > by others is strictly prohibited. If you are not the intended
> > recipient, please contact the sender and delete all copies.
> >
  

On Mon, 2022-12-19 at 01:23 -0500, Michael S. Tsirkin wrote:
> On Mon, Nov 07, 2022 at 03:43:54PM +0800, Jason Wang wrote:
> > On Wed, Oct 26, 2022 at 11:09 PM <sebastien.boeuf@intel.com> wrote:
> > > 
> > > From: Sebastien Boeuf <sebastien.boeuf@intel.com>
> > > 
> > > Implement resume operation for vdpa_sim devices, so vhost-vdpa
> > > will
> > > offer that backend feature and userspace can effectively resume
> > > the
> > > device.
> > > 
> > > Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
> > > ---
> > >  drivers/vdpa/vdpa_sim/vdpa_sim.c | 28
> > > ++++++++++++++++++++++++++++
> > >  drivers/vdpa/vdpa_sim/vdpa_sim.h |  1 +
> > >  2 files changed, 29 insertions(+)
> > > 
> > > diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > > b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > > index b071f0d842fb..84fee8bb2929 100644
> > > --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > > +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > > @@ -357,6 +357,11 @@ static void vdpasim_kick_vq(struct
> > > vdpa_device *vdpa, u16 idx)
> > >         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> > >         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
> > > 
> > > +       if (!vdpasim->running) {
> > > +               vdpasim->pending_kick = true;
> > > +               return;
> > 
> > I think we may hit here when the driver kicks vq before DRIVER_OK.
> > Do
> > we need to check device status in this case and resume?
> > 
> > Thanks
> 
> Sebastien did you forget to reply here?

Ah yes sorry I got carried away with other things and forgot about
this. And then I was on vacation.

I'll look at it this week.

Thanks,
Sebastien

> 
> > > +       }
> > > +
> > >         if (vq->ready)
> > >                 schedule_work(&vdpasim->work);
> > >  }
> > > @@ -527,6 +532,27 @@ static int vdpasim_suspend(struct
> > > vdpa_device *vdpa)
> > >         return 0;
> > >  }
> > > 
> > > +static int vdpasim_resume(struct vdpa_device *vdpa)
> > > +{
> > > +       struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> > > +       int i;
> > > +
> > > +       spin_lock(&vdpasim->lock);
> > > +       vdpasim->running = true;
> > > +
> > > +       if (vdpasim->pending_kick) {
> > > +               /* Process pending descriptors */
> > > +               for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)
> > > +                       vdpasim_kick_vq(vdpa, i);
> > > +
> > > +               vdpasim->pending_kick = false;
> > > +       }
> > > +
> > > +       spin_unlock(&vdpasim->lock);
> > > +
> > > +       return 0;
> > > +}
> > > +
> > >  static size_t vdpasim_get_config_size(struct vdpa_device *vdpa)
> > >  {
> > >         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> > > @@ -717,6 +743,7 @@ static const struct vdpa_config_ops
> > > vdpasim_config_ops = {
> > >         .set_status             = vdpasim_set_status,
> > >         .reset                  = vdpasim_reset,
> > >         .suspend                = vdpasim_suspend,
> > > +       .resume                 = vdpasim_resume,
> > >         .get_config_size        = vdpasim_get_config_size,
> > >         .get_config             = vdpasim_get_config,
> > >         .set_config             = vdpasim_set_config,
> > > @@ -750,6 +777,7 @@ static const struct vdpa_config_ops
> > > vdpasim_batch_config_ops = {
> > >         .set_status             = vdpasim_set_status,
> > >         .reset                  = vdpasim_reset,
> > >         .suspend                = vdpasim_suspend,
> > > +       .resume                 = vdpasim_resume,
> > >         .get_config_size        = vdpasim_get_config_size,
> > >         .get_config             = vdpasim_get_config,
> > >         .set_config             = vdpasim_set_config,
> > > diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h
> > > b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> > > index 0e78737dcc16..a745605589e2 100644
> > > --- a/drivers/vdpa/vdpa_sim/vdpa_sim.h
> > > +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> > > @@ -67,6 +67,7 @@ struct vdpasim {
> > >         u64 features;
> > >         u32 groups;
> > >         bool running;
> > > +       bool pending_kick;
> > >         /* spinlock to synchronize iommu table */
> > >         spinlock_t iommu_lock;
> > >  };
> > > --
> > > 2.34.1
> > > 
> > > -----------------------------------------------------------------
> > > ----
> > > Intel Corporation SAS (French simplified joint stock company)
> > > Registered headquarters: "Les Montalets"- 2, rue de Paris,
> > > 92196 Meudon Cedex, France
> > > Registration Number:  302 456 199 R.C.S. NANTERRE
> > > Capital: 5 208 026.16 Euros
> > > 
> > > This e-mail and any attachments may contain confidential material
> > > for
> > > the sole use of the intended recipient(s). Any review or
> > > distribution
> > > by others is strictly prohibited. If you are not the intended
> > > recipient, please contact the sender and delete all copies.
> > > 
> 

---------------------------------------------------------------------
Intel Corporation SAS (French simplified joint stock company)
Registered headquarters: "Les Montalets"- 2, rue de Paris, 
92196 Meudon Cedex, France
Registration Number:  302 456 199 R.C.S. NANTERRE
Capital: 5 208 026.16 Euros

This e-mail and any attachments may contain confidential material for
the sole use of the intended recipient(s). Any review or distribution
by others is strictly prohibited. If you are not the intended
recipient, please contact the sender and delete all copies.
  

On Tue, Jan 3, 2023 at 12:52 AM Boeuf, Sebastien
<sebastien.boeuf@intel.com> wrote:
>
> Hi Jason,
>
> Are you suggesting we do something like the following:
>
> if ((!vdpasim->running) && (vdpasim->status == DRIVER_OK)) {
>       vdpasim->pending_kick = true;
>       return;
> }
>
> ?

Yes.

Thanks

>
> Thanks,
> Sebastien
> ________________________________
> From: Jason Wang <jasowang@redhat.com>
> Sent: Monday, November 7, 2022 8:43 AM
> To: Boeuf, Sebastien <sebastien.boeuf@intel.com>
> Cc: linux-kernel@vger.kernel.org <linux-kernel@vger.kernel.org>; virtualization@lists.linux-foundation.org <virtualization@lists.linux-foundation.org>; mst@redhat.com <mst@redhat.com>; eperezma@redhat.com <eperezma@redhat.com>
> Subject: Re: [PATCH v5 4/4] vdpa_sim: Implement resume vdpa op
>
> On Wed, Oct 26, 2022 at 11:09 PM <sebastien.boeuf@intel.com> wrote:
> >
> > From: Sebastien Boeuf <sebastien.boeuf@intel.com>
> >
> > Implement resume operation for vdpa_sim devices, so vhost-vdpa will
> > offer that backend feature and userspace can effectively resume the
> > device.
> >
> > Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
> > ---
> >  drivers/vdpa/vdpa_sim/vdpa_sim.c | 28 ++++++++++++++++++++++++++++
> >  drivers/vdpa/vdpa_sim/vdpa_sim.h |  1 +
> >  2 files changed, 29 insertions(+)
> >
> > diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > index b071f0d842fb..84fee8bb2929 100644
> > --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c
> > @@ -357,6 +357,11 @@ static void vdpasim_kick_vq(struct vdpa_device *vdpa, u16 idx)
> >         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> >         struct vdpasim_virtqueue *vq = &vdpasim->vqs[idx];
> >
> > +       if (!vdpasim->running) {
> > +               vdpasim->pending_kick = true;
> > +               return;
>
> I think we may hit here when the driver kicks vq before DRIVER_OK. Do
> we need to check device status in this case and resume?
>
> Thanks
>
> > +       }
> > +
> >         if (vq->ready)
> >                 schedule_work(&vdpasim->work);
> >  }
> > @@ -527,6 +532,27 @@ static int vdpasim_suspend(struct vdpa_device *vdpa)
> >         return 0;
> >  }
> >
> > +static int vdpasim_resume(struct vdpa_device *vdpa)
> > +{
> > +       struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> > +       int i;
> > +
> > +       spin_lock(&vdpasim->lock);
> > +       vdpasim->running = true;
> > +
> > +       if (vdpasim->pending_kick) {
> > +               /* Process pending descriptors */
> > +               for (i = 0; i < vdpasim->dev_attr.nvqs; ++i)
> > +                       vdpasim_kick_vq(vdpa, i);
> > +
> > +               vdpasim->pending_kick = false;
> > +       }
> > +
> > +       spin_unlock(&vdpasim->lock);
> > +
> > +       return 0;
> > +}
> > +
> >  static size_t vdpasim_get_config_size(struct vdpa_device *vdpa)
> >  {
> >         struct vdpasim *vdpasim = vdpa_to_sim(vdpa);
> > @@ -717,6 +743,7 @@ static const struct vdpa_config_ops vdpasim_config_ops = {
> >         .set_status             = vdpasim_set_status,
> >         .reset                  = vdpasim_reset,
> >         .suspend                = vdpasim_suspend,
> > +       .resume                 = vdpasim_resume,
> >         .get_config_size        = vdpasim_get_config_size,
> >         .get_config             = vdpasim_get_config,
> >         .set_config             = vdpasim_set_config,
> > @@ -750,6 +777,7 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = {
> >         .set_status             = vdpasim_set_status,
> >         .reset                  = vdpasim_reset,
> >         .suspend                = vdpasim_suspend,
> > +       .resume                 = vdpasim_resume,
> >         .get_config_size        = vdpasim_get_config_size,
> >         .get_config             = vdpasim_get_config,
> >         .set_config             = vdpasim_set_config,
> > diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> > index 0e78737dcc16..a745605589e2 100644
> > --- a/drivers/vdpa/vdpa_sim/vdpa_sim.h
> > +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h
> > @@ -67,6 +67,7 @@ struct vdpasim {
> >         u64 features;
> >         u32 groups;
> >         bool running;
> > +       bool pending_kick;
> >         /* spinlock to synchronize iommu table */
> >         spinlock_t iommu_lock;
> >  };
> > --
> > 2.34.1
> >
> > ---------------------------------------------------------------------
> > Intel Corporation SAS (French simplified joint stock company)
> > Registered headquarters: "Les Montalets"- 2, rue de Paris,
> > 92196 Meudon Cedex, France
> > Registration Number:  302 456 199 R.C.S. NANTERRE
> > Capital: 5 208 026.16 Euros
> >
> > This e-mail and any attachments may contain confidential material for
> > the sole use of the intended recipient(s). Any review or distribution
> > by others is strictly prohibited. If you are not the intended
> > recipient, please contact the sender and delete all copies.
> >
>
> ---------------------------------------------------------------------
> Intel Corporation SAS (French simplified joint stock company)
> Registered headquarters: "Les Montalets"- 2, rue de Paris,
> 92196 Meudon Cedex, France
> Registration Number:  302 456 199 R.C.S. NANTERRE
> Capital: 5 208 026.16 Euros
>
> This e-mail and any attachments may contain confidential material for
> the sole use of the intended recipient(s). Any review or distribution
> by others is strictly prohibited. If you are not the intended
> recipient, please contact the sender and delete all copies.