[v2,2/6] RDMA/bnxt_re: Use auxiliary driver interface

Message ID 20221025173110.33192-3-ajit.khaparde@broadcom.com
State New
Headers
Series Add Auxiliary driver support |

Commit Message

Ajit Khaparde Oct. 25, 2022, 5:31 p.m. UTC
  Use auxiliary driver interface for driver load, unload ROCE driver.
The driver does not need to register the interface using the netdev
notifier anymore. Removed the bnxt_re_dev_list which is not needed.
Currently probe, remove and shutdown ops have been implemented for
the auxiliary device.

Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
Reviewed-by: Andy Gospodarek <andrew.gospodarek@broadcom.com>
---
 drivers/infiniband/hw/bnxt_re/bnxt_re.h       |   9 +-
 drivers/infiniband/hw/bnxt_re/main.c          | 387 +++++++-----------
 drivers/net/ethernet/broadcom/bnxt/bnxt.c     |  64 ---
 drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c |  67 ++-
 drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h |   3 +
 5 files changed, 214 insertions(+), 316 deletions(-)
  

Comments

Leon Romanovsky Oct. 26, 2022, 9:24 a.m. UTC | #1
On Tue, Oct 25, 2022 at 10:31:06AM -0700, Ajit Khaparde wrote:
> Use auxiliary driver interface for driver load, unload ROCE driver.
> The driver does not need to register the interface using the netdev
> notifier anymore. Removed the bnxt_re_dev_list which is not needed.
> Currently probe, remove and shutdown ops have been implemented for
> the auxiliary device.
> 
> Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
> Reviewed-by: Andy Gospodarek <andrew.gospodarek@broadcom.com>
> ---
>  drivers/infiniband/hw/bnxt_re/bnxt_re.h       |   9 +-
>  drivers/infiniband/hw/bnxt_re/main.c          | 387 +++++++-----------
>  drivers/net/ethernet/broadcom/bnxt/bnxt.c     |  64 ---
>  drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c |  67 ++-
>  drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h |   3 +
>  5 files changed, 214 insertions(+), 316 deletions(-)

<...>

> -static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev)
> +static struct bnxt_en_dev *bnxt_re_dev_probe(struct auxiliary_device *adev)
>  {
> -	struct bnxt_en_dev *en_dev;
> +	struct bnxt_aux_dev *aux_dev =
> +		container_of(adev, struct bnxt_aux_dev, aux_dev);
> +	struct bnxt_en_dev *en_dev = NULL;
>  	struct pci_dev *pdev;
>  
> -	en_dev = ((struct bnxt*)netdev_priv(netdev))->edev;
> -	if (IS_ERR(en_dev))
> -		return en_dev;
> +	if (aux_dev)
> +		en_dev = aux_dev->edev;
> +
> +	if (!en_dev)
> +		return NULL;

Thank you for working to convert this driver to auxiliary bus. I'm
confident that it will be ready soon.

In order to effectively review this series, you need to structure
patches in such way that you don't remove in patch X+1 code that you
added in patch X.

Also you should remove maze of redundant functions that do nothing, but
just call to another function with useless checks.

Auxiliary devices shouldn't be created if en_dev == NULL.

Thanks
  
Ajit Khaparde Oct. 26, 2022, 3:58 p.m. UTC | #2
On Wed, Oct 26, 2022 at 2:24 AM Leon Romanovsky <leon@kernel.org> wrote:
>
> On Tue, Oct 25, 2022 at 10:31:06AM -0700, Ajit Khaparde wrote:
> > Use auxiliary driver interface for driver load, unload ROCE driver.
> > The driver does not need to register the interface using the netdev
> > notifier anymore. Removed the bnxt_re_dev_list which is not needed.
> > Currently probe, remove and shutdown ops have been implemented for
> > the auxiliary device.
> >
> > Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
> > Reviewed-by: Andy Gospodarek <andrew.gospodarek@broadcom.com>
> > ---
> >  drivers/infiniband/hw/bnxt_re/bnxt_re.h       |   9 +-
> >  drivers/infiniband/hw/bnxt_re/main.c          | 387 +++++++-----------
> >  drivers/net/ethernet/broadcom/bnxt/bnxt.c     |  64 ---
> >  drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c |  67 ++-
> >  drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h |   3 +
> >  5 files changed, 214 insertions(+), 316 deletions(-)
>
> <...>
>
> > -static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev)
> > +static struct bnxt_en_dev *bnxt_re_dev_probe(struct auxiliary_device *adev)
> >  {
> > -     struct bnxt_en_dev *en_dev;
> > +     struct bnxt_aux_dev *aux_dev =
> > +             container_of(adev, struct bnxt_aux_dev, aux_dev);
> > +     struct bnxt_en_dev *en_dev = NULL;
> >       struct pci_dev *pdev;
> >
> > -     en_dev = ((struct bnxt*)netdev_priv(netdev))->edev;
> > -     if (IS_ERR(en_dev))
> > -             return en_dev;
> > +     if (aux_dev)
> > +             en_dev = aux_dev->edev;
> > +
> > +     if (!en_dev)
> > +             return NULL;
>
> Thank you for working to convert this driver to auxiliary bus. I'm
> confident that it will be ready soon.
Thanks

>
> In order to effectively review this series, you need to structure
> patches in such way that you don't remove in patch X+1 code that you
> added in patch X.
Sure. Moving from v1 to v2, I surely ran into the situation and cleaned them.
I will clean up the rest in case I missed something.

>
> Also you should remove maze of redundant functions that do nothing, but
> just call to another function with useless checks.
ACK

>
> Auxiliary devices shouldn't be created if en_dev == NULL.
ACK

& Thanks

>
> Thanks
  
Leon Romanovsky Nov. 7, 2022, 6:37 a.m. UTC | #3
On Fri, Nov 04, 2022 at 09:27:27AM -0700, Ajit Khaparde wrote:
> Add auxiliary device driver for Broadcom devices.
> The bnxt_en driver will register and initialize an aux device
> if RDMA is enabled in the underlying device.
> The bnxt_re driver will then probe and initialize the
> RoCE interfaces with the infiniband stack.
> 
> v1->v2:
> - Incorporated review comments including usage of ulp_id &
>   complex function indirections.
> - Used function calls provided by the auxiliary bus interface
>   instead of proprietary calls.
> - Refactor code to remove ROCE driver's access to bnxt structure.
> 
> v2->v3:
> - Addressed review comments including cleanup of some unnecessary wrappers
> - Fixed warnings seen during cross compilation
> 
> Please apply. Thanks.

Please send this series as standalone one and not as a reply
to previous discussion. Reply-to messes review flow, especially
for series.

Jakub, I'll review it once Ajit will send it properly.

Thanks
  
Jakub Kicinski Nov. 7, 2022, 4:06 p.m. UTC | #4
On Mon, 7 Nov 2022 08:37:13 +0200 Leon Romanovsky wrote:
> Please send this series as standalone one and not as a reply
> to previous discussion. Reply-to messes review flow, especially
> for series.
> 
> Jakub, I'll review it once Ajit will send it properly.

IIUC we wait for you or Jason to review any of the RoCE bifurcation
patches before considering them for inclusion.
  
Leon Romanovsky Nov. 7, 2022, 5:26 p.m. UTC | #5
On Mon, Nov 07, 2022 at 08:06:05AM -0800, Jakub Kicinski wrote:
> On Mon, 7 Nov 2022 08:37:13 +0200 Leon Romanovsky wrote:
> > Please send this series as standalone one and not as a reply
> > to previous discussion. Reply-to messes review flow, especially
> > for series.
> > 
> > Jakub, I'll review it once Ajit will send it properly.
> 
> IIUC we wait for you or Jason to review any of the RoCE bifurcation
> patches before considering them for inclusion.

Thanks
  
Ajit Khaparde Nov. 7, 2022, 6:21 p.m. UTC | #6
On Sun, Nov 6, 2022 at 10:37 PM Leon Romanovsky <leon@kernel.org> wrote:
>
> On Fri, Nov 04, 2022 at 09:27:27AM -0700, Ajit Khaparde wrote:
> > Add auxiliary device driver for Broadcom devices.
> > The bnxt_en driver will register and initialize an aux device
> > if RDMA is enabled in the underlying device.
> > The bnxt_re driver will then probe and initialize the
> > RoCE interfaces with the infiniband stack.
> >
> > v1->v2:
> > - Incorporated review comments including usage of ulp_id &
> >   complex function indirections.
> > - Used function calls provided by the auxiliary bus interface
> >   instead of proprietary calls.
> > - Refactor code to remove ROCE driver's access to bnxt structure.
> >
> > v2->v3:
> > - Addressed review comments including cleanup of some unnecessary wrappers
> > - Fixed warnings seen during cross compilation
> >
> > Please apply. Thanks.
>
> Please send this series as standalone one and not as a reply
> to previous discussion. Reply-to messes review flow, especially
> for series.
ACK.

>
> Jakub, I'll review it once Ajit will send it properly.
>
> Thanks
  

Patch

diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
index 785c37cae3c0..b0465c8d229a 100644
--- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
+++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
@@ -89,13 +89,6 @@  struct bnxt_re_ring_attr {
 	u8		mode;
 };
 
-struct bnxt_re_work {
-	struct work_struct	work;
-	unsigned long		event;
-	struct bnxt_re_dev      *rdev;
-	struct net_device	*vlan_dev;
-};
-
 struct bnxt_re_sqp_entries {
 	struct bnxt_qplib_sge sge;
 	u64 wrid;
@@ -132,6 +125,7 @@  struct bnxt_re_dev {
 #define BNXT_RE_FLAG_ERR_DEVICE_DETACHED       17
 #define BNXT_RE_FLAG_ISSUE_ROCE_STATS          29
 	struct net_device		*netdev;
+	struct notifier_block		nb;
 	unsigned int			version, major, minor;
 	struct bnxt_qplib_chip_ctx	*chip_ctx;
 	struct bnxt_en_dev		*en_dev;
@@ -194,5 +188,4 @@  static inline struct device *rdev_to_dev(struct bnxt_re_dev *rdev)
 		return  &rdev->ibdev.dev;
 	return NULL;
 }
-
 #endif
diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
index f868b285d2a1..0bbd2b3095d6 100644
--- a/drivers/infiniband/hw/bnxt_re/main.c
+++ b/drivers/infiniband/hw/bnxt_re/main.c
@@ -48,6 +48,7 @@ 
 #include <net/ipv6.h>
 #include <net/addrconf.h>
 #include <linux/if_ether.h>
+#include <linux/auxiliary_bus.h>
 
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_user_verbs.h>
@@ -74,14 +75,14 @@  MODULE_DESCRIPTION(BNXT_RE_DESC " Driver");
 MODULE_LICENSE("Dual BSD/GPL");
 
 /* globals */
-static struct list_head bnxt_re_dev_list = LIST_HEAD_INIT(bnxt_re_dev_list);
-/* Mutex to protect the list of bnxt_re devices added */
-static DEFINE_MUTEX(bnxt_re_dev_lock);
-static struct workqueue_struct *bnxt_re_wq;
-static void bnxt_re_remove_device(struct bnxt_re_dev *rdev);
-static void bnxt_re_dealloc_driver(struct ib_device *ib_dev);
+static DEFINE_MUTEX(bnxt_re_mutex);
+
 static void bnxt_re_stop_irq(void *handle);
 static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev);
+static int bnxt_re_netdev_event(struct notifier_block *notifier,
+				unsigned long event, void *ptr);
+static void bnxt_re_remove_device(struct bnxt_re_dev *rdev);
+static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev);
 
 static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode)
 {
@@ -233,7 +234,6 @@  static void bnxt_re_stop(void *p)
 
 	if (!rdev)
 		return;
-	ASSERT_RTNL();
 
 	/* L2 driver invokes this callback during device error/crash or device
 	 * reset. Current RoCE driver doesn't recover the device in case of
@@ -282,16 +282,14 @@  static void bnxt_re_sriov_config(void *p, int num_vfs)
 	}
 }
 
-static void bnxt_re_shutdown(void *p)
+static void bnxt_re_shutdown(struct auxiliary_device *adev)
 {
-	struct bnxt_re_dev *rdev = p;
+	struct bnxt_re_dev *rdev = auxiliary_get_drvdata(adev);
 
 	if (!rdev)
 		return;
-	ASSERT_RTNL();
-	/* Release the MSIx vectors before queuing unregister */
-	bnxt_re_stop_irq(rdev);
-	ib_unregister_device_queued(&rdev->ibdev);
+	ib_unregister_device(&rdev->ibdev);
+	bnxt_re_remove_device(rdev);
 }
 
 static void bnxt_re_stop_irq(void *handle)
@@ -346,11 +344,9 @@  static void bnxt_re_start_irq(void *handle, struct bnxt_msix_entry *ent)
 }
 
 static struct bnxt_ulp_ops bnxt_re_ulp_ops = {
-	.ulp_async_notifier = NULL,
 	.ulp_stop = bnxt_re_stop,
 	.ulp_start = bnxt_re_start,
 	.ulp_sriov_config = bnxt_re_sriov_config,
-	.ulp_shutdown = bnxt_re_shutdown,
 	.ulp_irq_stop = bnxt_re_stop_irq,
 	.ulp_irq_restart = bnxt_re_start_irq
 };
@@ -401,7 +397,6 @@  static int bnxt_re_free_msix(struct bnxt_re_dev *rdev)
 
 	en_dev = rdev->en_dev;
 
-
 	rc = en_dev->en_ops->bnxt_free_msix(rdev->en_dev, BNXT_ROCE_ULP);
 
 	return rc;
@@ -458,12 +453,17 @@  static void bnxt_re_fill_fw_msg(struct bnxt_fw_msg *fw_msg, void *msg,
 static int bnxt_re_net_ring_free(struct bnxt_re_dev *rdev,
 				 u16 fw_ring_id, int type)
 {
-	struct bnxt_en_dev *en_dev = rdev->en_dev;
+	struct bnxt_en_dev *en_dev;
 	struct hwrm_ring_free_input req = {0};
 	struct hwrm_ring_free_output resp;
 	struct bnxt_fw_msg fw_msg;
 	int rc = -EINVAL;
 
+	if (!rdev)
+		return rc;
+
+	en_dev = rdev->en_dev;
+
 	if (!en_dev)
 		return rc;
 
@@ -584,21 +584,6 @@  static int bnxt_re_net_stats_ctx_alloc(struct bnxt_re_dev *rdev,
 
 /* Device */
 
-static bool is_bnxt_re_dev(struct net_device *netdev)
-{
-	struct ethtool_drvinfo drvinfo;
-
-	if (netdev->ethtool_ops && netdev->ethtool_ops->get_drvinfo) {
-		memset(&drvinfo, 0, sizeof(drvinfo));
-		netdev->ethtool_ops->get_drvinfo(netdev, &drvinfo);
-
-		if (strcmp(drvinfo.driver, "bnxt_en"))
-			return false;
-		return true;
-	}
-	return false;
-}
-
 static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev)
 {
 	struct ib_device *ibdev =
@@ -609,27 +594,22 @@  static struct bnxt_re_dev *bnxt_re_from_netdev(struct net_device *netdev)
 	return container_of(ibdev, struct bnxt_re_dev, ibdev);
 }
 
-static struct bnxt_en_dev *bnxt_re_dev_probe(struct net_device *netdev)
+static struct bnxt_en_dev *bnxt_re_dev_probe(struct auxiliary_device *adev)
 {
-	struct bnxt_en_dev *en_dev;
+	struct bnxt_aux_dev *aux_dev =
+		container_of(adev, struct bnxt_aux_dev, aux_dev);
+	struct bnxt_en_dev *en_dev = NULL;
 	struct pci_dev *pdev;
 
-	en_dev = ((struct bnxt*)netdev_priv(netdev))->edev;
-	if (IS_ERR(en_dev))
-		return en_dev;
+	if (aux_dev)
+		en_dev = aux_dev->edev;
+
+	if (!en_dev)
+		return NULL;
 
 	pdev = en_dev->pdev;
 	if (!pdev)
-		return ERR_PTR(-EINVAL);
-
-	if (!(en_dev->flags & BNXT_EN_FLAG_ROCE_CAP)) {
-		dev_info(&pdev->dev,
-			"%s: probe error: RoCE is not supported on this device",
-			ROCE_DRV_MODULE_NAME);
-		return ERR_PTR(-ENODEV);
-	}
-
-	dev_hold(netdev);
+		return NULL;
 
 	return en_dev;
 }
@@ -679,7 +659,6 @@  static const struct ib_device_ops bnxt_re_dev_ops = {
 	.create_qp = bnxt_re_create_qp,
 	.create_srq = bnxt_re_create_srq,
 	.create_user_ah = bnxt_re_create_ah,
-	.dealloc_driver = bnxt_re_dealloc_driver,
 	.dealloc_pd = bnxt_re_dealloc_pd,
 	.dealloc_ucontext = bnxt_re_dealloc_ucontext,
 	.del_gid = bnxt_re_del_gid,
@@ -744,18 +723,7 @@  static int bnxt_re_register_ib(struct bnxt_re_dev *rdev)
 	return ib_register_device(ibdev, "bnxt_re%d", &rdev->en_dev->pdev->dev);
 }
 
-static void bnxt_re_dev_remove(struct bnxt_re_dev *rdev)
-{
-	dev_put(rdev->netdev);
-	rdev->netdev = NULL;
-	mutex_lock(&bnxt_re_dev_lock);
-	list_del_rcu(&rdev->list);
-	mutex_unlock(&bnxt_re_dev_lock);
-
-	synchronize_rcu();
-}
-
-static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
+static struct bnxt_re_dev *bnxt_re_dev_add(struct bnxt_aux_dev *aux_dev,
 					   struct bnxt_en_dev *en_dev)
 {
 	struct bnxt_re_dev *rdev;
@@ -768,8 +736,8 @@  static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
 		return NULL;
 	}
 	/* Default values */
-	rdev->netdev = netdev;
-	dev_hold(rdev->netdev);
+	rdev->nb.notifier_call = NULL;
+	rdev->netdev = en_dev->net;
 	rdev->en_dev = en_dev;
 	rdev->id = rdev->en_dev->pdev->devfn;
 	INIT_LIST_HEAD(&rdev->qp_list);
@@ -784,9 +752,6 @@  static struct bnxt_re_dev *bnxt_re_dev_add(struct net_device *netdev,
 	rdev->cosq[0] = 0xFFFF;
 	rdev->cosq[1] = 0xFFFF;
 
-	mutex_lock(&bnxt_re_dev_lock);
-	list_add_tail_rcu(&rdev->list, &bnxt_re_dev_list);
-	mutex_unlock(&bnxt_re_dev_lock);
 	return rdev;
 }
 
@@ -1147,6 +1112,9 @@  static void bnxt_re_dev_stop(struct bnxt_re_dev *rdev)
 	struct ib_qp_attr qp_attr;
 	struct bnxt_re_qp *qp;
 
+	if (!rdev)
+		return;
+
 	qp_attr.qp_state = IB_QPS_ERR;
 	mutex_lock(&rdev->qp_lock);
 	list_for_each_entry(qp, &rdev->qp_list, list) {
@@ -1323,7 +1291,7 @@  static int bnxt_re_ib_init(struct bnxt_re_dev *rdev)
 		pr_err("Failed to register with IB: %#x\n", rc);
 		return rc;
 	}
-	dev_info(rdev_to_dev(rdev), "Device registered successfully");
+	dev_info(rdev_to_dev(rdev), "Device registered with IB successfully");
 	ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed,
 			 &rdev->active_width);
 	set_bit(BNXT_RE_FLAG_ISSUE_ROCE_STATS, &rdev->flags);
@@ -1541,135 +1509,49 @@  static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
 	return rc;
 }
 
-static void bnxt_re_dev_unreg(struct bnxt_re_dev *rdev)
+static void bnxt_re_remove_device(struct bnxt_re_dev *rdev)
 {
-	struct net_device *netdev = rdev->netdev;
-
-	bnxt_re_dev_remove(rdev);
-
-	if (netdev)
-		dev_put(netdev);
+	bnxt_re_dev_uninit(rdev);
 }
 
-static int bnxt_re_dev_reg(struct bnxt_re_dev **rdev, struct net_device *netdev)
+static int bnxt_re_add_device(struct auxiliary_device *adev, u8 wqe_mode)
 {
+	struct bnxt_aux_dev *aux_dev =
+		container_of(adev, struct bnxt_aux_dev, aux_dev);
 	struct bnxt_en_dev *en_dev;
+	struct bnxt_re_dev *rdev;
 	int rc = 0;
 
-	if (!is_bnxt_re_dev(netdev))
+	en_dev = bnxt_re_dev_probe(adev);
+	if (!en_dev)
 		return -ENODEV;
 
-	en_dev = bnxt_re_dev_probe(netdev);
-	if (IS_ERR(en_dev)) {
-		if (en_dev != ERR_PTR(-ENODEV))
-			ibdev_err(&(*rdev)->ibdev, "%s: Failed to probe\n",
-				  ROCE_DRV_MODULE_NAME);
-		rc = PTR_ERR(en_dev);
-		goto exit;
-	}
-	*rdev = bnxt_re_dev_add(netdev, en_dev);
-	if (!*rdev) {
+	rdev = bnxt_re_dev_add(aux_dev, en_dev);
+	if (!rdev || !rdev_to_dev(rdev)) {
 		rc = -ENOMEM;
-		dev_put(netdev);
 		goto exit;
 	}
-exit:
-	return rc;
-}
-
-static void bnxt_re_remove_device(struct bnxt_re_dev *rdev)
-{
-	bnxt_re_dev_uninit(rdev);
-	pci_dev_put(rdev->en_dev->pdev);
-	bnxt_re_dev_unreg(rdev);
-}
-
-static int bnxt_re_add_device(struct bnxt_re_dev **rdev,
-			      struct net_device *netdev, u8 wqe_mode)
-{
-	int rc;
 
-	rc = bnxt_re_dev_reg(rdev, netdev);
-	if (rc == -ENODEV)
-		return rc;
-	if (rc) {
-		pr_err("Failed to register with the device %s: %#x\n",
-		       netdev->name, rc);
-		return rc;
-	}
+	rc = bnxt_re_dev_init(rdev, wqe_mode);
+	if (rc)
+		goto re_dev_dealloc;
 
-	pci_dev_get((*rdev)->en_dev->pdev);
-	rc = bnxt_re_dev_init(*rdev, wqe_mode);
+	rc = bnxt_re_ib_init(rdev);
 	if (rc) {
-		pci_dev_put((*rdev)->en_dev->pdev);
-		bnxt_re_dev_unreg(*rdev);
+		pr_err("Failed to register with IB: %s",
+			aux_dev->aux_dev.name);
+		goto re_dev_uninit;
 	}
+	auxiliary_set_drvdata(adev, rdev);
 
-	return rc;
-}
-
-static void bnxt_re_dealloc_driver(struct ib_device *ib_dev)
-{
-	struct bnxt_re_dev *rdev =
-		container_of(ib_dev, struct bnxt_re_dev, ibdev);
-
-	dev_info(rdev_to_dev(rdev), "Unregistering Device");
-
-	rtnl_lock();
-	bnxt_re_remove_device(rdev);
-	rtnl_unlock();
-}
-
-/* Handle all deferred netevents tasks */
-static void bnxt_re_task(struct work_struct *work)
-{
-	struct bnxt_re_work *re_work;
-	struct bnxt_re_dev *rdev;
-	int rc = 0;
-
-	re_work = container_of(work, struct bnxt_re_work, work);
-	rdev = re_work->rdev;
-
-	if (re_work->event == NETDEV_REGISTER) {
-		rc = bnxt_re_ib_init(rdev);
-		if (rc) {
-			ibdev_err(&rdev->ibdev,
-				  "Failed to register with IB: %#x", rc);
-			rtnl_lock();
-			bnxt_re_remove_device(rdev);
-			rtnl_unlock();
-			goto exit;
-		}
-		goto exit;
-	}
-
-	if (!ib_device_try_get(&rdev->ibdev))
-		goto exit;
+	return 0;
 
-	switch (re_work->event) {
-	case NETDEV_UP:
-		bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
-				       IB_EVENT_PORT_ACTIVE);
-		break;
-	case NETDEV_DOWN:
-		bnxt_re_dev_stop(rdev);
-		break;
-	case NETDEV_CHANGE:
-		if (!netif_carrier_ok(rdev->netdev))
-			bnxt_re_dev_stop(rdev);
-		else if (netif_carrier_ok(rdev->netdev))
-			bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
-					       IB_EVENT_PORT_ACTIVE);
-		ib_get_eth_speed(&rdev->ibdev, 1, &rdev->active_speed,
-				 &rdev->active_width);
-		break;
-	default:
-		break;
-	}
-	ib_device_put(&rdev->ibdev);
+re_dev_uninit:
+	bnxt_re_dev_uninit(rdev);
+re_dev_dealloc:
+	ib_dealloc_device(&rdev->ibdev);
 exit:
-	put_device(&rdev->ibdev.dev);
-	kfree(re_work);
+	return rc;
 }
 
 /*
@@ -1690,109 +1572,130 @@  static int bnxt_re_netdev_event(struct notifier_block *notifier,
 				unsigned long event, void *ptr)
 {
 	struct net_device *real_dev, *netdev = netdev_notifier_info_to_dev(ptr);
-	struct bnxt_re_work *re_work;
 	struct bnxt_re_dev *rdev;
-	int rc = 0;
-	bool sch_work = false;
-	bool release = true;
 
 	real_dev = rdma_vlan_dev_real_dev(netdev);
 	if (!real_dev)
 		real_dev = netdev;
 
-	rdev = bnxt_re_from_netdev(real_dev);
-	if (!rdev && event != NETDEV_REGISTER)
-		return NOTIFY_OK;
-
 	if (real_dev != netdev)
 		goto exit;
 
-	switch (event) {
-	case NETDEV_REGISTER:
-		if (rdev)
-			break;
-		rc = bnxt_re_add_device(&rdev, real_dev,
-					BNXT_QPLIB_WQE_MODE_STATIC);
-		if (!rc)
-			sch_work = true;
-		release = false;
-		break;
+	rdev = bnxt_re_from_netdev(real_dev);
+	if (!rdev)
+		return NOTIFY_DONE;
 
-	case NETDEV_UNREGISTER:
-		ib_unregister_device_queued(&rdev->ibdev);
-		break;
 
+	switch (event) {
+	case NETDEV_UP:
+	case NETDEV_DOWN:
+	case NETDEV_CHANGE:
+		bnxt_re_dispatch_event(&rdev->ibdev, NULL, 1,
+					netif_carrier_ok(real_dev) ?
+					IB_EVENT_PORT_ACTIVE :
+					IB_EVENT_PORT_ERR);
+		break;
 	default:
-		sch_work = true;
 		break;
 	}
-	if (sch_work) {
-		/* Allocate for the deferred task */
-		re_work = kzalloc(sizeof(*re_work), GFP_KERNEL);
-		if (re_work) {
-			get_device(&rdev->ibdev.dev);
-			re_work->rdev = rdev;
-			re_work->event = event;
-			re_work->vlan_dev = (real_dev == netdev ?
-					     NULL : netdev);
-			INIT_WORK(&re_work->work, bnxt_re_task);
-			queue_work(bnxt_re_wq, &re_work->work);
-		}
-	}
-
+	ib_device_put(&rdev->ibdev);
 exit:
-	if (rdev && release)
-		ib_device_put(&rdev->ibdev);
 	return NOTIFY_DONE;
 }
 
-static struct notifier_block bnxt_re_netdev_notifier = {
-	.notifier_call = bnxt_re_netdev_event
-};
+#define BNXT_ADEV_NAME "bnxt_en"
 
-static int __init bnxt_re_mod_init(void)
+static void bnxt_re_remove(struct auxiliary_device *adev)
 {
-	int rc = 0;
+	struct bnxt_re_dev *rdev = auxiliary_get_drvdata(adev);
 
-	pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version);
+	if (!rdev)
+		return;
 
-	bnxt_re_wq = create_singlethread_workqueue("bnxt_re");
-	if (!bnxt_re_wq)
-		return -ENOMEM;
+	mutex_lock(&bnxt_re_mutex);
+	if (rdev->nb.notifier_call) {
+		unregister_netdevice_notifier(&rdev->nb);
+		rdev->nb.notifier_call = NULL;
+	} else {
+		/* If notifier is null, we should have already done a
+		 * clean up before coming here.
+		 */
+		goto skip_remove;
+	}
+
+	ib_unregister_device(&rdev->ibdev);
+	ib_dealloc_device(&rdev->ibdev);
+	bnxt_re_remove_device(rdev);
+skip_remove:
+	mutex_unlock(&bnxt_re_mutex);
+}
+
+static int bnxt_re_probe(struct auxiliary_device *adev,
+			 const struct auxiliary_device_id *id)
+{
+	struct bnxt_re_dev *rdev;
+	int rc;
 
-	INIT_LIST_HEAD(&bnxt_re_dev_list);
+	mutex_lock(&bnxt_re_mutex);
+	rc = bnxt_re_add_device(adev, BNXT_QPLIB_WQE_MODE_STATIC);
+	if (rc) {
+		mutex_unlock(&bnxt_re_mutex);
+		return rc;
+	}
 
-	rc = register_netdevice_notifier(&bnxt_re_netdev_notifier);
+	rdev = auxiliary_get_drvdata(adev);
+
+	rdev->nb.notifier_call = bnxt_re_netdev_event;
+	rc = register_netdevice_notifier(&rdev->nb);
 	if (rc) {
+		rdev->nb.notifier_call = NULL;
 		pr_err("%s: Cannot register to netdevice_notifier",
 		       ROCE_DRV_MODULE_NAME);
-		goto err_netdev;
+		goto err;
 	}
+
+	mutex_unlock(&bnxt_re_mutex);
 	return 0;
 
-err_netdev:
-	destroy_workqueue(bnxt_re_wq);
+err:
+	mutex_unlock(&bnxt_re_mutex);
+	bnxt_re_remove(adev);
 
 	return rc;
 }
 
-static void __exit bnxt_re_mod_exit(void)
+static const struct auxiliary_device_id bnxt_re_id_table[] = {
+	{ .name = BNXT_ADEV_NAME ".rdma", },
+	{},
+};
+
+MODULE_DEVICE_TABLE(auxiliary, bnxt_re_id_table);
+
+static struct auxiliary_driver bnxt_re_driver = {
+	.name = "rdma",
+	.probe = bnxt_re_probe,
+	.remove = bnxt_re_remove,
+	.shutdown = bnxt_re_shutdown,
+	.id_table = bnxt_re_id_table,
+};
+
+static int __init bnxt_re_mod_init(void)
 {
-	struct bnxt_re_dev *rdev;
+	int rc = 0;
 
-	unregister_netdevice_notifier(&bnxt_re_netdev_notifier);
-	if (bnxt_re_wq)
-		destroy_workqueue(bnxt_re_wq);
-	list_for_each_entry(rdev, &bnxt_re_dev_list, list) {
-		/* VF device removal should be called before the removal
-		 * of PF device. Queue VFs unregister first, so that VFs
-		 * shall be removed before the PF during the call of
-		 * ib_unregister_driver.
-		 */
-		if (rdev->is_virtfn)
-			ib_unregister_device(&rdev->ibdev);
+	pr_info("%s: %s", ROCE_DRV_MODULE_NAME, version);
+	rc = auxiliary_driver_register(&bnxt_re_driver);
+	if (rc) {
+		pr_err("%s: Failed to register auxiliary driver\n",
+			ROCE_DRV_MODULE_NAME);
+		return rc;
 	}
-	ib_unregister_driver(RDMA_DRIVER_BNXT_RE);
+	return 0;
+}
+
+static void __exit bnxt_re_mod_exit(void)
+{
+	auxiliary_driver_unregister(&bnxt_re_driver);
 }
 
 module_init(bnxt_re_mod_init);
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
index 0c8786ea2c1c..3cbedbecf3b3 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c
@@ -79,8 +79,6 @@ 
 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("Broadcom BCM573xx network driver");
 
-static DEFINE_IDA(bnxt_aux_dev_ids);
-
 #define BNXT_RX_OFFSET (NET_SKB_PAD + NET_IP_ALIGN)
 #define BNXT_RX_DMA_OFFSET NET_SKB_PAD
 #define BNXT_RX_COPY_THRESH 256
@@ -213,68 +211,6 @@  static const struct pci_device_id bnxt_pci_tbl[] = {
 
 MODULE_DEVICE_TABLE(pci, bnxt_pci_tbl);
 
-static void bnxt_aux_dev_free(struct bnxt *bp)
-{
-	kfree(bp->aux_dev);
-	bp->aux_dev = NULL;
-}
-
-static struct bnxt_aux_dev *bnxt_aux_dev_alloc(struct bnxt *bp)
-{
-	struct bnxt_aux_dev *bnxt_adev;
-
-	bnxt_adev =  kzalloc(sizeof(*bnxt_adev), GFP_KERNEL);
-	if (!bnxt_adev)
-		return ERR_PTR(-ENOMEM);
-
-	return bnxt_adev;
-}
-
-static void bnxt_rdma_aux_device_uninit(struct bnxt_aux_dev *bnxt_adev)
-{
-	struct auxiliary_device *adev;
-
-	if (IS_ERR_OR_NULL(bnxt_adev))
-		return;
-
-	adev = &bnxt_adev->aux_dev;
-	auxiliary_device_delete(adev);
-	auxiliary_device_uninit(adev);
-	if (bnxt_adev->id >= 0)
-		ida_free(&bnxt_aux_dev_ids, bnxt_adev->id);
-}
-
-static void bnxt_rdma_aux_device_init(struct bnxt *bp)
-{
-	struct net_device *dev = bp->dev;
-	int rc;
-
-	if (bp->flags & BNXT_FLAG_ROCE_CAP) {
-		bp->aux_dev = bnxt_aux_dev_alloc(bp);
-		if (IS_ERR_OR_NULL(bp->aux_dev)) {
-			netdev_warn(dev, "Failed to init auxiliary device for ROCE\n");
-			goto skip_aux_init;
-		}
-
-		bp->aux_dev->id = ida_alloc(&bnxt_aux_dev_ids, GFP_KERNEL);
-		if (bp->aux_dev->id < 0) {
-			netdev_warn(dev, "ida alloc failed for ROCE auxiliary device\n");
-			bnxt_aux_dev_free(bp);
-			goto skip_aux_init;
-		}
-
-		/* If aux bus init fails, continue with netdev init. */
-		rc = bnxt_rdma_aux_device_add(bp);
-		if (rc) {
-			netdev_warn(dev, "Failed to add auxiliary device for ROCE\n");
-			ida_free(&bnxt_aux_dev_ids, bp->aux_dev->id);
-			bnxt_aux_dev_free(bp);
-		}
-	}
-skip_aux_init:
-	return;
-}
-
 static const u16 bnxt_vf_req_snif[] = {
 	HWRM_FUNC_CFG,
 	HWRM_FUNC_VF_CFG,
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
index 44a0922663cf..1d2d30f97ed9 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c
@@ -19,12 +19,15 @@ 
 #include <linux/irq.h>
 #include <asm/byteorder.h>
 #include <linux/bitmap.h>
+#include <linux/auxiliary_bus.h>
 
 #include "bnxt_hsi.h"
 #include "bnxt.h"
 #include "bnxt_hwrm.h"
 #include "bnxt_ulp.h"
 
+static DEFINE_IDA(bnxt_aux_dev_ids);
+
 static int bnxt_register_dev(struct bnxt_en_dev *edev, unsigned int ulp_id,
 			     struct bnxt_ulp_ops *ulp_ops, void *handle)
 {
@@ -76,8 +79,6 @@  static int bnxt_unregister_dev(struct bnxt_en_dev *edev, unsigned int ulp_id)
 	if (ulp_id >= BNXT_MAX_ULP)
 		return -EINVAL;
 
-	edev->flags |= BNXT_EN_FLAG_ULP_STOPPED;
-
 	ulp = &edev->ulp_tbl[ulp_id];
 	if (!rcu_access_pointer(ulp->ulp_ops)) {
 		netdev_err(bp->dev, "ulp id %d not registered\n", ulp_id);
@@ -465,6 +466,68 @@  static const struct bnxt_en_ops bnxt_en_ops_tbl = {
 	.bnxt_register_fw_async_events	= bnxt_register_async_events,
 };
 
+void bnxt_aux_dev_free(struct bnxt *bp)
+{
+	kfree(bp->aux_dev);
+	bp->aux_dev = NULL;
+}
+
+static struct bnxt_aux_dev *bnxt_aux_dev_alloc(struct bnxt *bp)
+{
+	struct bnxt_aux_dev *bnxt_adev;
+
+	bnxt_adev =  kzalloc(sizeof(*bnxt_adev), GFP_KERNEL);
+	if (!bnxt_adev)
+		return ERR_PTR(-ENOMEM);
+
+	return bnxt_adev;
+}
+
+void bnxt_rdma_aux_device_uninit(struct bnxt_aux_dev *bnxt_adev)
+{
+	struct auxiliary_device *adev;
+
+	if (IS_ERR_OR_NULL(bnxt_adev))
+		return;
+
+	adev = &bnxt_adev->aux_dev;
+	auxiliary_device_delete(adev);
+	auxiliary_device_uninit(adev);
+	if (bnxt_adev->id >= 0)
+		ida_free(&bnxt_aux_dev_ids, bnxt_adev->id);
+}
+
+void bnxt_rdma_aux_device_init(struct bnxt *bp)
+{
+	struct net_device *dev = bp->dev;
+	int rc;
+
+	if (bp->flags & BNXT_FLAG_ROCE_CAP) {
+		bp->aux_dev = bnxt_aux_dev_alloc(bp);
+		if (IS_ERR_OR_NULL(bp->aux_dev)) {
+			netdev_warn(dev, "Failed to init auxiliary device for ROCE\n");
+			goto skip_aux_init;
+		}
+
+		bp->aux_dev->id = ida_alloc(&bnxt_aux_dev_ids, GFP_KERNEL);
+		if (bp->aux_dev->id < 0) {
+			netdev_warn(dev, "ida alloc failed for ROCE auxiliary device\n");
+			bnxt_aux_dev_free(bp);
+			goto skip_aux_init;
+		}
+
+		/* If aux bus init fails, continue with netdev init. */
+		rc = bnxt_rdma_aux_device_add(bp);
+		if (rc) {
+			netdev_warn(dev, "Failed to add auxiliary device for ROCE\n");
+			ida_free(&bnxt_aux_dev_ids, bp->aux_dev->id);
+			bnxt_aux_dev_free(bp);
+		}
+	}
+skip_aux_init:
+	return;
+}
+
 void bnxt_aux_dev_release(struct device *dev)
 {
 	struct bnxt_aux_dev *bnxt_adev =
diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h
index f26f1075bfee..6cac4c0b708b 100644
--- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h
+++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.h
@@ -109,4 +109,7 @@  void bnxt_ulp_async_events(struct bnxt *bp, struct hwrm_async_event_cmpl *cmpl);
 void bnxt_aux_dev_release(struct device *dev);
 int bnxt_rdma_aux_device_add(struct bnxt *bp);
 
+void bnxt_rdma_aux_device_uninit(struct bnxt_aux_dev *bnxt_adev);
+void bnxt_rdma_aux_device_init(struct bnxt *bp);
+void bnxt_aux_dev_free(struct bnxt *bp);
 #endif