[V7,net] net: mana: Fix MANA VF unload when hardware is

  When unloading the MANA driver, mana_dealloc_queues() waits for the MANA
hardware to complete any inflight packets and set the pending send count
to zero. But if the hardware has failed, mana_dealloc_queues()
could wait forever.

Fix this by adding a timeout to the wait. Set the timeout to 120 seconds,
which is a somewhat arbitrary value that is more than long enough for
functional hardware to complete any sends.

Cc: stable@vger.kernel.org
Fixes: ca9c54d2d6a5 ("net: mana: Add a driver for Microsoft Azure Network Adapter (MANA)")

Signed-off-by: Souradeep Chakrabarti <schakrabarti@linux.microsoft.com>
---
V6 -> V7:
* Optimized the while loop for freeing skb.

V5 -> V6:
* Added pcie_flr to reset the pci after timeout.
* Fixed the position of changelog.
* Removed unused variable like cq.

V4 -> V5:
* Added fixes tag
* Changed the usleep_range from static to incremental value.
* Initialized timeout in the begining.

V3 -> V4:
* Removed the unnecessary braces from mana_dealloc_queues().

V2 -> V3:
* Removed the unnecessary braces from mana_dealloc_queues().

V1 -> V2:
* Added net branch
* Removed the typecasting to (struct mana_context*) of void pointer
* Repositioned timeout variable in mana_dealloc_queues()
* Repositioned vf_unload_timeout in mana_context struct, to utilise the
 6 bytes hole
---
 drivers/net/ethernet/microsoft/mana/mana_en.c | 37 +++++++++++++++++--
 1 file changed, 33 insertions(+), 4 deletions(-)
  

On Tue, Aug 01, 2023 at 05:29:13AM -0700, Souradeep Chakrabarti wrote:

...

Hi Souradeep,


> +	for (i = 0; i < apc->num_queues; i++) {
> +		txq = &apc->tx_qp[i].txq;
> +		while (skb = skb_dequeue(&txq->pending_skbs)) {

W=1 builds with both clang-16 and gcc-12 complain that
they would like an extra set of parentheses around
an assignment used as a truth value.

> +			mana_unmap_skb(skb, apc);
> +			dev_consume_skb_any(skb);
> +		}
> +		atomic_set(&txq->pending_sends, 0);
> +	}
>  	/* We're 100% sure the queues can no longer be woken up, because
>  	 * we're sure now mana_poll_tx_cq() can't be running.
>  	 */
> -- 
> 2.34.1
> 
>
  

On 8/1/2023 5:29 AM, Souradeep Chakrabarti wrote:
> When unloading the MANA driver, mana_dealloc_queues() waits for the MANA
> hardware to complete any inflight packets and set the pending send count
> to zero. But if the hardware has failed, mana_dealloc_queues()
> could wait forever.
> 
> Fix this by adding a timeout to the wait. Set the timeout to 120 seconds,

tx timeout in stack defaults to 5 seconds, do you not have that on? What
happens when you start getting resets while unloading?

> which is a somewhat arbitrary value that is more than long enough for
> functional hardware to complete any sends.

I'd say 2 or 5 seconds is probably plenty of time to hang up a driver
unload.

> 
> Cc: stable@vger.kernel.org
> Fixes: ca9c54d2d6a5 ("net: mana: Add a driver for Microsoft Azure Network Adapter (MANA)")
> 
> Signed-off-by: Souradeep Chakrabarti <schakrabarti@linux.microsoft.com>

keep s-o-b and other trailers together please, no spaces, it messes up
git and doesn't conform to kernel standards.


> ---
> V6 -> V7:
> * Optimized the while loop for freeing skb.
> 
> V5 -> V6:
> * Added pcie_flr to reset the pci after timeout.
> * Fixed the position of changelog.
> * Removed unused variable like cq.
> 
> V4 -> V5:
> * Added fixes tag
> * Changed the usleep_range from static to incremental value.
> * Initialized timeout in the begining.
> 
> V3 -> V4:
> * Removed the unnecessary braces from mana_dealloc_queues().
> 
> V2 -> V3:
> * Removed the unnecessary braces from mana_dealloc_queues().
> 
> V1 -> V2:
> * Added net branch
> * Removed the typecasting to (struct mana_context*) of void pointer
> * Repositioned timeout variable in mana_dealloc_queues()
> * Repositioned vf_unload_timeout in mana_context struct, to utilise the
>  6 bytes hole
> ---
>  drivers/net/ethernet/microsoft/mana/mana_en.c | 37 +++++++++++++++++--
>  1 file changed, 33 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c
> index a499e460594b..3c5552a176d0 100644
> --- a/drivers/net/ethernet/microsoft/mana/mana_en.c
> +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
> @@ -8,6 +8,7 @@
>  #include <linux/ethtool.h>
>  #include <linux/filter.h>
>  #include <linux/mm.h>
> +#include <linux/pci.h>
>  
>  #include <net/checksum.h>
>  #include <net/ip6_checksum.h>
> @@ -2345,9 +2346,12 @@ int mana_attach(struct net_device *ndev)
>  static int mana_dealloc_queues(struct net_device *ndev)
>  {
>  	struct mana_port_context *apc = netdev_priv(ndev);
> +	unsigned long timeout = jiffies + 120 * HZ;
>  	struct gdma_dev *gd = apc->ac->gdma_dev;
>  	struct mana_txq *txq;
> +	struct sk_buff *skb;
>  	int i, err;
> +	u32 tsleep;
>  
>  	if (apc->port_is_up)
>  		return -EINVAL;
> @@ -2363,15 +2367,40 @@ static int mana_dealloc_queues(struct net_device *ndev)
>  	 * to false, but it doesn't matter since mana_start_xmit() drops any
>  	 * new packets due to apc->port_is_up being false.
>  	 *
> -	 * Drain all the in-flight TX packets
> +	 * Drain all the in-flight TX packets.
> +	 * A timeout of 120 seconds for all the queues is used.
> +	 * This will break the while loop when h/w is not responding.
> +	 * This value of 120 has been decided here considering max
> +	 * number of queues.
>  	 */
> +
>  	for (i = 0; i < apc->num_queues; i++) {
>  		txq = &apc->tx_qp[i].txq;
> -
> -		while (atomic_read(&txq->pending_sends) > 0)
> -			usleep_range(1000, 2000);
> +		tsleep = 1000;
> +		while (atomic_read(&txq->pending_sends) > 0 &&
> +		       time_before(jiffies, timeout)) {
> +			usleep_range(tsleep, tsleep + 1000);
> +			tsleep <<= 1;
> +		}
> +		if (atomic_read(&txq->pending_sends)) {
> +			err = pcie_flr(to_pci_dev(gd->gdma_context->dev));
> +			if (err) {
> +				netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n",
> +					   err, atomic_read(&txq->pending_sends),
> +					   txq->gdma_txq_id);
> +			}
> +			break;
> +		}
>  	}
>  
> +	for (i = 0; i < apc->num_queues; i++) {
> +		txq = &apc->tx_qp[i].txq;
> +		while (skb = skb_dequeue(&txq->pending_skbs)) {
> +			mana_unmap_skb(skb, apc);
> +			dev_consume_skb_any(skb);

dev_kfree_skb_any() would be more correct here since this is an error
path and the transmit is presumed dropped, isn't it?


> +		}
> +		atomic_set(&txq->pending_sends, 0);
> +	}
>  	/* We're 100% sure the queues can no longer be woken up, because
>  	 * we're sure now mana_poll_tx_cq() can't be running.
>  	 */
  

>-----Original Message-----
>From: Jesse Brandeburg <jesse.brandeburg@intel.com>
>Sent: Tuesday, August 1, 2023 11:34 PM
>To: Souradeep Chakrabarti <schakrabarti@linux.microsoft.com>; KY Srinivasan
><kys@microsoft.com>; Haiyang Zhang <haiyangz@microsoft.com>;
>wei.liu@kernel.org; Dexuan Cui <decui@microsoft.com>;
>davem@davemloft.net; edumazet@google.com; kuba@kernel.org;
>pabeni@redhat.com; Long Li <longli@microsoft.com>; Ajay Sharma
><sharmaajay@microsoft.com>; leon@kernel.org; cai.huoqing@linux.dev;
>ssengar@linux.microsoft.com; vkuznets <vkuznets@redhat.com>;
>tglx@linutronix.de; linux-hyperv@vger.kernel.org; netdev@vger.kernel.org;
>linux-kernel@vger.kernel.org; linux-rdma@vger.kernel.org
>Cc: Souradeep Chakrabarti <schakrabarti@microsoft.com>;
>stable@vger.kernel.org
>Subject: [EXTERNAL] Re: [PATCH V7 net] net: mana: Fix MANA VF unload when
>hardware is
>
>On 8/1/2023 5:29 AM, Souradeep Chakrabarti wrote:
>> When unloading the MANA driver, mana_dealloc_queues() waits for the
>> MANA hardware to complete any inflight packets and set the pending
>> send count to zero. But if the hardware has failed,
>> mana_dealloc_queues() could wait forever.
>>
>> Fix this by adding a timeout to the wait. Set the timeout to 120
>> seconds,
>
>tx timeout in stack defaults to 5 seconds, do you not have that on? What
>happens when you start getting resets while unloading?
>
>> which is a somewhat arbitrary value that is more than long enough for
>> functional hardware to complete any sends.
>
>I'd say 2 or 5 seconds is probably plenty of time to hang up a driver unload.
>
Thank you for the comment. This was already discussed in V4.
I am just sharing the summary here:
This waiting time is usually much shorter than 120 sec. 
The long wait only happens in rare and unexpected NIC HW non-responding cases. 
At that point, we don't actually care if the pending packets are sent or not. 
But if we free the queues too soon, and the HW is slow for unexpected reasons, 
a delayed completion notice will DMA into the freed memory and cause corruption. 
That's why we have a longer waiting time. 
>>
>> Cc: stable@vger.kernel.org
>> Fixes: ca9c54d2d6a5 ("net: mana: Add a driver for Microsoft Azure
>> Network Adapter (MANA)")
>>
>> Signed-off-by: Souradeep Chakrabarti
>> <schakrabarti@linux.microsoft.com>
>
>keep s-o-b and other trailers together please, no spaces, it messes up git and
>doesn't conform to kernel standards.
>
>
>> ---
>> V6 -> V7:
>> * Optimized the while loop for freeing skb.
>>
>> V5 -> V6:
>> * Added pcie_flr to reset the pci after timeout.
>> * Fixed the position of changelog.
>> * Removed unused variable like cq.
>>
>> V4 -> V5:
>> * Added fixes tag
>> * Changed the usleep_range from static to incremental value.
>> * Initialized timeout in the begining.
>>
>> V3 -> V4:
>> * Removed the unnecessary braces from mana_dealloc_queues().
>>
>> V2 -> V3:
>> * Removed the unnecessary braces from mana_dealloc_queues().
>>
>> V1 -> V2:
>> * Added net branch
>> * Removed the typecasting to (struct mana_context*) of void pointer
>> * Repositioned timeout variable in mana_dealloc_queues()
>> * Repositioned vf_unload_timeout in mana_context struct, to utilise
>> the
>>  6 bytes hole
>> ---
>>  drivers/net/ethernet/microsoft/mana/mana_en.c | 37
>> +++++++++++++++++--
>>  1 file changed, 33 insertions(+), 4 deletions(-)
>>
>> diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c
>> b/drivers/net/ethernet/microsoft/mana/mana_en.c
>> index a499e460594b..3c5552a176d0 100644
>> --- a/drivers/net/ethernet/microsoft/mana/mana_en.c
>> +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c
>> @@ -8,6 +8,7 @@
>>  #include <linux/ethtool.h>
>>  #include <linux/filter.h>
>>  #include <linux/mm.h>
>> +#include <linux/pci.h>
>>
>>  #include <net/checksum.h>
>>  #include <net/ip6_checksum.h>
>> @@ -2345,9 +2346,12 @@ int mana_attach(struct net_device *ndev)
>> static int mana_dealloc_queues(struct net_device *ndev)  {
>>  	struct mana_port_context *apc = netdev_priv(ndev);
>> +	unsigned long timeout = jiffies + 120 * HZ;
>>  	struct gdma_dev *gd = apc->ac->gdma_dev;
>>  	struct mana_txq *txq;
>> +	struct sk_buff *skb;
>>  	int i, err;
>> +	u32 tsleep;
>>
>>  	if (apc->port_is_up)
>>  		return -EINVAL;
>> @@ -2363,15 +2367,40 @@ static int mana_dealloc_queues(struct
>net_device *ndev)
>>  	 * to false, but it doesn't matter since mana_start_xmit() drops any
>>  	 * new packets due to apc->port_is_up being false.
>>  	 *
>> -	 * Drain all the in-flight TX packets
>> +	 * Drain all the in-flight TX packets.
>> +	 * A timeout of 120 seconds for all the queues is used.
>> +	 * This will break the while loop when h/w is not responding.
>> +	 * This value of 120 has been decided here considering max
>> +	 * number of queues.
>>  	 */
>> +
>>  	for (i = 0; i < apc->num_queues; i++) {
>>  		txq = &apc->tx_qp[i].txq;
>> -
>> -		while (atomic_read(&txq->pending_sends) > 0)
>> -			usleep_range(1000, 2000);
>> +		tsleep = 1000;
>> +		while (atomic_read(&txq->pending_sends) > 0 &&
>> +		       time_before(jiffies, timeout)) {
>> +			usleep_range(tsleep, tsleep + 1000);
>> +			tsleep <<= 1;
>> +		}
>> +		if (atomic_read(&txq->pending_sends)) {
>> +			err = pcie_flr(to_pci_dev(gd->gdma_context->dev));
>> +			if (err) {
>> +				netdev_err(ndev, "flr failed %d with %d pkts
>pending in txq %u\n",
>> +					   err, atomic_read(&txq-
>>pending_sends),
>> +					   txq->gdma_txq_id);
>> +			}
>> +			break;
>> +		}
>>  	}
>>
>> +	for (i = 0; i < apc->num_queues; i++) {
>> +		txq = &apc->tx_qp[i].txq;
>> +		while (skb = skb_dequeue(&txq->pending_skbs)) {
>> +			mana_unmap_skb(skb, apc);
>> +			dev_consume_skb_any(skb);
>
>dev_kfree_skb_any() would be more correct here since this is an error path and
>the transmit is presumed dropped, isn't it?
Yes, dev_kfree_skb_any() will be a better approach in this scenario. Will change it in next version.
>
>
>> +		}
>> +		atomic_set(&txq->pending_sends, 0);
>> +	}
>>  	/* We're 100% sure the queues can no longer be woken up, because
>>  	 * we're sure now mana_poll_tx_cq() can't be running.
>>  	 */
  

>-----Original Message-----
>From: Simon Horman <horms@kernel.org>
>Sent: Tuesday, August 1, 2023 9:01 PM
>To: Souradeep Chakrabarti <schakrabarti@linux.microsoft.com>
>Cc: KY Srinivasan <kys@microsoft.com>; Haiyang Zhang
><haiyangz@microsoft.com>; wei.liu@kernel.org; Dexuan Cui
><decui@microsoft.com>; davem@davemloft.net; edumazet@google.com;
>kuba@kernel.org; pabeni@redhat.com; Long Li <longli@microsoft.com>; Ajay
>Sharma <sharmaajay@microsoft.com>; leon@kernel.org;
>cai.huoqing@linux.dev; ssengar@linux.microsoft.com; vkuznets
><vkuznets@redhat.com>; tglx@linutronix.de; linux-hyperv@vger.kernel.org;
>netdev@vger.kernel.org; linux-kernel@vger.kernel.org; linux-
>rdma@vger.kernel.org; Souradeep Chakrabarti
><schakrabarti@microsoft.com>; stable@vger.kernel.org
>Subject: [EXTERNAL] Re: [PATCH V7 net] net: mana: Fix MANA VF unload when
>hardware is
>
>On Tue, Aug 01, 2023 at 05:29:13AM -0700, Souradeep Chakrabarti wrote:
>
>...
>
>Hi Souradeep,
>
>
>> +	for (i = 0; i < apc->num_queues; i++) {
>> +		txq = &apc->tx_qp[i].txq;
>> +		while (skb = skb_dequeue(&txq->pending_skbs)) {
>
>W=1 builds with both clang-16 and gcc-12 complain that they would like an
>extra set of parentheses around an assignment used as a truth value.
Thanks for letting me know. I will fix it in next version.
>
>> +			mana_unmap_skb(skb, apc);
>> +			dev_consume_skb_any(skb);
>> +		}
>> +		atomic_set(&txq->pending_sends, 0);
>> +	}
>>  	/* We're 100% sure the queues can no longer be woken up, because
>>  	 * we're sure now mana_poll_tx_cq() can't be running.
>>  	 */
>> --
>> 2.34.1
>>
>>
  

On Wed, Aug 02, 2023 at 10:57:52AM +0530, Kalesh Anakkur Purayil wrote:
> Hi Souradeep,
> 
> It looks like the subject line is not complete. I could see "net: mana: Fix
> MANA VF unload when hardware is".
> 
> Is that correct?
> 
> Regards,
> Kalesh
>
Yes, it got truncated. Will fix it in next version. 
> On Wed, Aug 2, 2023 at 12:29 AM Souradeep Chakrabarti <
> schakrabarti@microsoft.com> wrote:
> 
> >
> >
> > >-----Original Message-----
> > >From: Simon Horman <horms@kernel.org>
> > >Sent: Tuesday, August 1, 2023 9:01 PM
> > >To: Souradeep Chakrabarti <schakrabarti@linux.microsoft.com>
> > >Cc: KY Srinivasan <kys@microsoft.com>; Haiyang Zhang
> > ><haiyangz@microsoft.com>; wei.liu@kernel.org; Dexuan Cui
> > ><decui@microsoft.com>; davem@davemloft.net; edumazet@google.com;
> > >kuba@kernel.org; pabeni@redhat.com; Long Li <longli@microsoft.com>; Ajay
> > >Sharma <sharmaajay@microsoft.com>; leon@kernel.org;
> > >cai.huoqing@linux.dev; ssengar@linux.microsoft.com; vkuznets
> > ><vkuznets@redhat.com>; tglx@linutronix.de; linux-hyperv@vger.kernel.org;
> > >netdev@vger.kernel.org; linux-kernel@vger.kernel.org; linux-
> > >rdma@vger.kernel.org; Souradeep Chakrabarti
> > ><schakrabarti@microsoft.com>; stable@vger.kernel.org
> > >Subject: [EXTERNAL] Re: [PATCH V7 net] net: mana: Fix MANA VF unload when
> > >hardware is
> > >
> > >On Tue, Aug 01, 2023 at 05:29:13AM -0700, Souradeep Chakrabarti wrote:
> > >
> > >...
> > >
> > >Hi Souradeep,
> > >
> > >
> > >> +    for (i = 0; i < apc->num_queues; i++) {
> > >> +            txq = &apc->tx_qp[i].txq;
> > >> +            while (skb = skb_dequeue(&txq->pending_skbs)) {
> > >
> > >W=1 builds with both clang-16 and gcc-12 complain that they would like an
> > >extra set of parentheses around an assignment used as a truth value.
> > Thanks for letting me know. I will fix it in next version.
> > >
> > >> +                    mana_unmap_skb(skb, apc);
> > >> +                    dev_consume_skb_any(skb);
> > >> +            }
> > >> +            atomic_set(&txq->pending_sends, 0);
> > >> +    }
> > >>      /* We're 100% sure the queues can no longer be woken up, because
> > >>       * we're sure now mana_poll_tx_cq() can't be running.
> > >>       */
> > >> --
> > >> 2.34.1
> > >>
> > >>
> >
> >
> 
> -- 
> Regards,
> Kalesh A P
  

Hi Souradeep,

kernel test robot noticed the following build warnings:

[auto build test WARNING on net/main]

url:    https://github.com/intel-lab-lkp/linux/commits/Souradeep-Chakrabarti/net-mana-Fix-MANA-VF-unload-when-hardware-is/20230801-203141
base:   net/main
patch link:    https://lore.kernel.org/r/1690892953-25201-1-git-send-email-schakrabarti%40linux.microsoft.com
patch subject: [PATCH V7 net] net: mana: Fix MANA VF unload when hardware is
config: x86_64-allyesconfig (https://download.01.org/0day-ci/archive/20230802/202308021532.8iYkExDh-lkp@intel.com/config)
compiler: gcc-12 (Debian 12.2.0-14) 12.2.0
reproduce: (https://download.01.org/0day-ci/archive/20230802/202308021532.8iYkExDh-lkp@intel.com/reproduce)

If you fix the issue in a separate patch/commit (i.e. not just a new version of
the same patch/commit), kindly add following tags
| Reported-by: kernel test robot <lkp@intel.com>
| Closes: https://lore.kernel.org/oe-kbuild-all/202308021532.8iYkExDh-lkp@intel.com/

All warnings (new ones prefixed by >>):

   drivers/net/ethernet/microsoft/mana/mana_en.c: In function 'mana_dealloc_queues':
>> drivers/net/ethernet/microsoft/mana/mana_en.c:2398:24: warning: suggest parentheses around assignment used as truth value [-Wparentheses]
    2398 |                 while (skb = skb_dequeue(&txq->pending_skbs)) {
         |                        ^~~


vim +2398 drivers/net/ethernet/microsoft/mana/mana_en.c

  2345	
  2346	static int mana_dealloc_queues(struct net_device *ndev)
  2347	{
  2348		struct mana_port_context *apc = netdev_priv(ndev);
  2349		unsigned long timeout = jiffies + 120 * HZ;
  2350		struct gdma_dev *gd = apc->ac->gdma_dev;
  2351		struct mana_txq *txq;
  2352		struct sk_buff *skb;
  2353		int i, err;
  2354		u32 tsleep;
  2355	
  2356		if (apc->port_is_up)
  2357			return -EINVAL;
  2358	
  2359		mana_chn_setxdp(apc, NULL);
  2360	
  2361		if (gd->gdma_context->is_pf)
  2362			mana_pf_deregister_filter(apc);
  2363	
  2364		/* No packet can be transmitted now since apc->port_is_up is false.
  2365		 * There is still a tiny chance that mana_poll_tx_cq() can re-enable
  2366		 * a txq because it may not timely see apc->port_is_up being cleared
  2367		 * to false, but it doesn't matter since mana_start_xmit() drops any
  2368		 * new packets due to apc->port_is_up being false.
  2369		 *
  2370		 * Drain all the in-flight TX packets.
  2371		 * A timeout of 120 seconds for all the queues is used.
  2372		 * This will break the while loop when h/w is not responding.
  2373		 * This value of 120 has been decided here considering max
  2374		 * number of queues.
  2375		 */
  2376	
  2377		for (i = 0; i < apc->num_queues; i++) {
  2378			txq = &apc->tx_qp[i].txq;
  2379			tsleep = 1000;
  2380			while (atomic_read(&txq->pending_sends) > 0 &&
  2381			       time_before(jiffies, timeout)) {
  2382				usleep_range(tsleep, tsleep + 1000);
  2383				tsleep <<= 1;
  2384			}
  2385			if (atomic_read(&txq->pending_sends)) {
  2386				err = pcie_flr(to_pci_dev(gd->gdma_context->dev));
  2387				if (err) {
  2388					netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n",
  2389						   err, atomic_read(&txq->pending_sends),
  2390						   txq->gdma_txq_id);
  2391				}
  2392				break;
  2393			}
  2394		}
  2395	
  2396		for (i = 0; i < apc->num_queues; i++) {
  2397			txq = &apc->tx_qp[i].txq;
> 2398			while (skb = skb_dequeue(&txq->pending_skbs)) {
  2399				mana_unmap_skb(skb, apc);
  2400				dev_consume_skb_any(skb);
  2401			}
  2402			atomic_set(&txq->pending_sends, 0);
  2403		}
  2404		/* We're 100% sure the queues can no longer be woken up, because
  2405		 * we're sure now mana_poll_tx_cq() can't be running.
  2406		 */
  2407	
  2408		apc->rss_state = TRI_STATE_FALSE;
  2409		err = mana_config_rss(apc, TRI_STATE_FALSE, false, false);
  2410		if (err) {
  2411			netdev_err(ndev, "Failed to disable vPort: %d\n", err);
  2412			return err;
  2413		}
  2414	
  2415		mana_destroy_vport(apc);
  2416	
  2417		return 0;
  2418	}
  2419