@@ -4,9 +4,11 @@ MODULE_NAME := mtk_tmi
mtk_tmi-y = \
pcie/mtk_pci.o \
- mtk_dev.o \
- mtk_bm.o \
- mtk_ctrl_plane.o
+ mtk_dev.o \
+ mtk_bm.o \
+ mtk_ctrl_plane.o \
+ mtk_cldma.o \
+ pcie/mtk_cldma_drv_t800.o
ccflags-y += -I$(srctree)/$(src)/
ccflags-y += -I$(srctree)/$(src)/pcie/
new file mode 100644
@@ -0,0 +1,258 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2022, MediaTek Inc.
+ */
+
+#include <linux/device.h>
+#include <linux/mutex.h>
+#include <linux/pm_runtime.h>
+#include <linux/skbuff.h>
+
+#include "mtk_cldma.h"
+#include "mtk_cldma_drv_t800.h"
+
+/* cldma_init() - Initialize CLDMA
+ *
+ * @trans: pointer to transaction structure
+ *
+ * Return:
+ * 0 - OK
+ * -ENOMEM - out of memory
+ */
+static int mtk_cldma_init(struct mtk_ctrl_trans *trans)
+{
+ struct cldma_dev *cd;
+
+ cd = devm_kzalloc(trans->mdev->dev, sizeof(*cd), GFP_KERNEL);
+ if (!cd)
+ return -ENOMEM;
+
+ cd->trans = trans;
+ cd->hw_ops.init = mtk_cldma_hw_init_t800;
+ cd->hw_ops.exit = mtk_cldma_hw_exit_t800;
+ cd->hw_ops.txq_alloc = mtk_cldma_txq_alloc_t800;
+ cd->hw_ops.rxq_alloc = mtk_cldma_rxq_alloc_t800;
+ cd->hw_ops.txq_free = mtk_cldma_txq_free_t800;
+ cd->hw_ops.rxq_free = mtk_cldma_rxq_free_t800;
+ cd->hw_ops.start_xfer = mtk_cldma_start_xfer_t800;
+
+ trans->dev[CLDMA_CLASS_ID] = cd;
+
+ return 0;
+}
+
+/* cldma_exit() - De-Initialize CLDMA
+ *
+ * @trans: pointer to transaction structure
+ *
+ * Return:
+ * 0 - OK
+ */
+static int mtk_cldma_exit(struct mtk_ctrl_trans *trans)
+{
+ struct cldma_dev *cd;
+
+ cd = trans->dev[CLDMA_CLASS_ID];
+ if (!cd)
+ return 0;
+
+ devm_kfree(trans->mdev->dev, cd);
+
+ return 0;
+}
+
+/* cldma_open() - Initialize CLDMA hardware queue
+ *
+ * @cd: pointer to CLDMA device
+ * @skb: pointer to socket buffer
+ *
+ * Return:
+ * 0 - OK
+ * -EBUSY - hardware queue is busy
+ * -EIO - failed to initialize hardware queue
+ * -EINVAL - invalid input parameters
+ */
+static int mtk_cldma_open(struct cldma_dev *cd, struct sk_buff *skb)
+{
+ struct trb_open_priv *trb_open_priv = (struct trb_open_priv *)skb->data;
+ struct trb *trb = (struct trb *)skb->cb;
+ struct cldma_hw *hw;
+ struct virtq *vq;
+ struct txq *txq;
+ struct rxq *rxq;
+ int err = 0;
+
+ vq = cd->trans->vq_tbl + trb->vqno;
+ hw = cd->cldma_hw[vq->hif_id & HIF_ID_BITMASK];
+ trb_open_priv->tx_mtu = vq->tx_mtu;
+ trb_open_priv->rx_mtu = vq->rx_mtu;
+ if (unlikely(vq->rxqno < 0 || vq->rxqno >= HW_QUEUE_NUM) ||
+ unlikely(vq->txqno < 0 || vq->txqno >= HW_QUEUE_NUM)) {
+ err = -EINVAL;
+ goto exit;
+ }
+
+ if (hw->txq[vq->txqno] || hw->rxq[vq->rxqno]) {
+ err = -EBUSY;
+ goto exit;
+ }
+
+ txq = cd->hw_ops.txq_alloc(hw, skb);
+ if (!txq) {
+ err = -EIO;
+ goto exit;
+ }
+
+ rxq = cd->hw_ops.rxq_alloc(hw, skb);
+ if (!rxq) {
+ err = -EIO;
+ cd->hw_ops.txq_free(hw, trb->vqno);
+ goto exit;
+ }
+
+exit:
+ trb->status = err;
+ trb->trb_complete(skb);
+
+ return err;
+}
+
+/* cldma_tx() - start CLDMA TX transaction
+ *
+ * @cd: pointer to CLDMA device
+ * @skb: pointer to socket buffer
+ *
+ * Return:
+ * 0 - OK
+ * -EPIPE - hardware queue is broken
+ */
+static int mtk_cldma_tx(struct cldma_dev *cd, struct sk_buff *skb)
+{
+ struct trb *trb = (struct trb *)skb->cb;
+ struct cldma_hw *hw;
+ struct virtq *vq;
+ struct txq *txq;
+
+ vq = cd->trans->vq_tbl + trb->vqno;
+ hw = cd->cldma_hw[vq->hif_id & HIF_ID_BITMASK];
+ txq = hw->txq[vq->txqno];
+ if (txq->is_stopping)
+ return -EPIPE;
+
+ cd->hw_ops.start_xfer(hw, vq->txqno);
+
+ return 0;
+}
+
+/* cldma_close() - De-Initialize CLDMA hardware queue
+ *
+ * @cd: pointer to CLDMA device
+ * @skb: pointer to socket buffer
+ *
+ * Return:
+ * 0 - OK
+ */
+static int mtk_cldma_close(struct cldma_dev *cd, struct sk_buff *skb)
+{
+ struct trb *trb = (struct trb *)skb->cb;
+ struct cldma_hw *hw;
+ struct virtq *vq;
+
+ vq = cd->trans->vq_tbl + trb->vqno;
+ hw = cd->cldma_hw[vq->hif_id & HIF_ID_BITMASK];
+
+ cd->hw_ops.txq_free(hw, trb->vqno);
+ cd->hw_ops.rxq_free(hw, trb->vqno);
+
+ trb->status = 0;
+ trb->trb_complete(skb);
+
+ return 0;
+}
+
+static int mtk_cldma_submit_tx(void *dev, struct sk_buff *skb)
+{
+ struct trb *trb = (struct trb *)skb->cb;
+ struct cldma_dev *cd = dev;
+ dma_addr_t data_dma_addr;
+ struct cldma_hw *hw;
+ struct tx_req *req;
+ struct virtq *vq;
+ struct txq *txq;
+ int err;
+
+ vq = cd->trans->vq_tbl + trb->vqno;
+ hw = cd->cldma_hw[vq->hif_id & HIF_ID_BITMASK];
+ txq = hw->txq[vq->txqno];
+
+ if (!txq->req_budget)
+ return -EAGAIN;
+
+ err = mtk_dma_map_single(hw->mdev, &data_dma_addr, skb->data,
+ skb->len, DMA_TO_DEVICE);
+ if (err)
+ return -EFAULT;
+
+ mutex_lock(&txq->lock);
+ txq->req_budget--;
+ mutex_unlock(&txq->lock);
+
+ req = txq->req_pool + txq->wr_idx;
+ req->gpd->tx_gpd.debug_id = 0x01;
+ req->gpd->tx_gpd.data_buff_ptr_h = cpu_to_le32((u64)(data_dma_addr) >> 32);
+ req->gpd->tx_gpd.data_buff_ptr_l = cpu_to_le32(data_dma_addr);
+ req->gpd->tx_gpd.data_buff_len = cpu_to_le16(skb->len);
+ req->gpd->tx_gpd.gpd_flags = CLDMA_GPD_FLAG_IOC | CLDMA_GPD_FLAG_HWO;
+
+ req->data_vm_addr = skb->data;
+ req->data_dma_addr = data_dma_addr;
+ req->data_len = skb->len;
+ req->skb = skb;
+ txq->wr_idx = (txq->wr_idx + 1) % txq->req_pool_size;
+
+ wmb(); /* ensure GPD setup done before HW start */
+
+ return 0;
+}
+
+/* cldma_trb_process() - Dispatch trb request to low-level CLDMA routine
+ *
+ * @dev: pointer to CLDMA device
+ * @skb: pointer to socket buffer
+ *
+ * Return:
+ * 0 - OK
+ * -EBUSY - hardware queue is busy
+ * -EINVAL - invalid input
+ * -EIO - failed to initialize hardware queue
+ * -EPIPE - hardware queue is broken
+ */
+static int mtk_cldma_trb_process(void *dev, struct sk_buff *skb)
+{
+ struct trb *trb = (struct trb *)skb->cb;
+ struct cldma_dev *cd = dev;
+ int err;
+
+ switch (trb->cmd) {
+ case TRB_CMD_ENABLE:
+ err = mtk_cldma_open(cd, skb);
+ break;
+ case TRB_CMD_TX:
+ err = mtk_cldma_tx(cd, skb);
+ break;
+ case TRB_CMD_DISABLE:
+ err = mtk_cldma_close(cd, skb);
+ break;
+ default:
+ err = -EINVAL;
+ }
+
+ return err;
+}
+
+struct hif_ops cldma_ops = {
+ .init = mtk_cldma_init,
+ .exit = mtk_cldma_exit,
+ .trb_process = mtk_cldma_trb_process,
+ .submit_tx = mtk_cldma_submit_tx,
+};
new file mode 100644
@@ -0,0 +1,158 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear
+ *
+ * Copyright (c) 2022, MediaTek Inc.
+ */
+
+#ifndef __MTK_CLDMA_H__
+#define __MTK_CLDMA_H__
+
+#include <linux/types.h>
+
+#include "mtk_ctrl_plane.h"
+#include "mtk_dev.h"
+
+#define HW_QUEUE_NUM 8
+#define ALLQ (0XFF)
+#define LINK_ERROR_VAL (0XFFFFFFFF)
+
+#define CLDMA_CLASS_ID 0
+
+#define NR_CLDMA 2
+#define CLDMA0 (((CLDMA_CLASS_ID) << HIF_CLASS_SHIFT) + 0)
+#define CLDMA1 (((CLDMA_CLASS_ID) << HIF_CLASS_SHIFT) + 1)
+
+#define TXQ(N) (N)
+#define RXQ(N) (N)
+
+#define CLDMA_GPD_FLAG_HWO BIT(0)
+#define CLDMA_GPD_FLAG_IOC BIT(7)
+
+enum mtk_ip_busy_src {
+ IP_BUSY_TXDONE = 0,
+ IP_BUSY_RXDONE = 24,
+};
+
+enum mtk_intr_type {
+ QUEUE_XFER_DONE = 0,
+ QUEUE_ERROR = 16,
+ INVALID_TYPE
+};
+
+enum mtk_tx_rx {
+ DIR_TX,
+ DIR_RX,
+ INVALID_DIR
+};
+
+union gpd {
+ struct {
+ u8 gpd_flags;
+ u8 non_used1;
+ __le16 data_allow_len;
+ __le32 next_gpd_ptr_h;
+ __le32 next_gpd_ptr_l;
+ __le32 data_buff_ptr_h;
+ __le32 data_buff_ptr_l;
+ __le16 data_recv_len;
+ u8 non_used2;
+ u8 debug_id;
+ } rx_gpd;
+
+ struct {
+ u8 gpd_flags;
+ u8 non_used1;
+ u8 non_used2;
+ u8 debug_id;
+ __le32 next_gpd_ptr_h;
+ __le32 next_gpd_ptr_l;
+ __le32 data_buff_ptr_h;
+ __le32 data_buff_ptr_l;
+ __le16 data_buff_len;
+ __le16 non_used3;
+ } tx_gpd;
+};
+
+struct rx_req {
+ union gpd *gpd;
+ int mtu;
+ struct sk_buff *skb;
+ size_t data_len;
+ dma_addr_t gpd_dma_addr;
+ dma_addr_t data_dma_addr;
+};
+
+struct rxq {
+ struct cldma_hw *hw;
+ int rxqno;
+ int vqno;
+ struct virtq *vq;
+ struct work_struct rx_done_work;
+ struct rx_req *req_pool;
+ int req_pool_size;
+ int free_idx;
+ unsigned short rx_done_cnt;
+ void *arg;
+ int (*rx_done)(struct sk_buff *skb, int len, void *priv);
+};
+
+struct tx_req {
+ union gpd *gpd;
+ int mtu;
+ void *data_vm_addr;
+ size_t data_len;
+ dma_addr_t data_dma_addr;
+ dma_addr_t gpd_dma_addr;
+ struct sk_buff *skb;
+ int (*trb_complete)(struct sk_buff *skb);
+};
+
+struct txq {
+ struct cldma_hw *hw;
+ int txqno;
+ int vqno;
+ struct virtq *vq;
+ struct mutex lock; /* protect structure fields */
+ struct work_struct tx_done_work;
+ struct tx_req *req_pool;
+ int req_pool_size;
+ int req_budget;
+ int wr_idx;
+ int free_idx;
+ bool tx_started;
+ bool is_stopping;
+ unsigned short tx_done_cnt;
+};
+
+struct cldma_dev;
+struct cldma_hw;
+
+struct cldma_hw_ops {
+ int (*init)(struct cldma_dev *cd, int hif_id);
+ int (*exit)(struct cldma_dev *cd, int hif_id);
+ struct txq* (*txq_alloc)(struct cldma_hw *hw, struct sk_buff *skb);
+ struct rxq* (*rxq_alloc)(struct cldma_hw *hw, struct sk_buff *skb);
+ int (*txq_free)(struct cldma_hw *hw, int vqno);
+ int (*rxq_free)(struct cldma_hw *hw, int vqno);
+ int (*start_xfer)(struct cldma_hw *hw, int qno);
+};
+
+struct cldma_hw {
+ int hif_id;
+ int base_addr;
+ int pci_ext_irq_id;
+ struct mtk_md_dev *mdev;
+ struct cldma_dev *cd;
+ struct txq *txq[HW_QUEUE_NUM];
+ struct rxq *rxq[HW_QUEUE_NUM];
+ struct dma_pool *dma_pool;
+ struct workqueue_struct *wq;
+};
+
+struct cldma_dev {
+ struct cldma_hw *cldma_hw[NR_CLDMA];
+ struct mtk_ctrl_trans *trans;
+ struct cldma_hw_ops hw_ops;
+};
+
+extern struct hif_ops cldma_ops;
+#endif
@@ -17,7 +17,55 @@
#define BUFF_3_5K_MAX_CNT (100)
#define BUFF_63K_MAX_CNT (64)
+#define HIF_CLASS_NUM (1)
+#define HIF_CLASS_SHIFT (8)
+#define HIF_ID_BITMASK (0x01)
+
+enum mtk_trb_cmd_type {
+ TRB_CMD_ENABLE = 1,
+ TRB_CMD_TX,
+ TRB_CMD_DISABLE,
+};
+
+struct trb_open_priv {
+ u16 tx_mtu;
+ u16 rx_mtu;
+ int (*rx_done)(struct sk_buff *skb, int len, void *priv);
+};
+
+struct trb {
+ u8 vqno;
+ enum mtk_trb_cmd_type cmd;
+ int status;
+ struct kref kref;
+ void *priv;
+ int (*trb_complete)(struct sk_buff *skb);
+};
+
+struct virtq {
+ int vqno;
+ int hif_id;
+ int txqno;
+ int rxqno;
+ int tx_mtu;
+ int rx_mtu;
+ int tx_req_num;
+ int rx_req_num;
+};
+
+struct mtk_ctrl_trans;
+
+struct hif_ops {
+ int (*init)(struct mtk_ctrl_trans *trans);
+ int (*exit)(struct mtk_ctrl_trans *trans);
+ int (*submit_tx)(void *dev, struct sk_buff *skb);
+ int (*trb_process)(void *dev, struct sk_buff *skb);
+};
+
struct mtk_ctrl_trans {
+ struct virtq *vq_tbl;
+ void *dev[HIF_CLASS_NUM];
+ struct hif_ops *ops[HIF_CLASS_NUM];
struct mtk_ctrl_blk *ctrl_blk;
struct mtk_md_dev *mdev;
};
new file mode 100644
@@ -0,0 +1,948 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+/*
+ * Copyright (c) 2022, MediaTek Inc.
+ */
+
+#include <linux/delay.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/mutex.h>
+#include <linux/pm_runtime.h>
+#include <linux/sched.h>
+#include <linux/workqueue.h>
+
+#include "mtk_bm.h"
+#include "mtk_cldma_drv_t800.h"
+#include "mtk_ctrl_plane.h"
+#include "mtk_dev.h"
+#include "mtk_reg.h"
+
+#define DMA_POOL_NAME_LEN 64
+
+#define CLDMA_STOP_HW_WAIT_TIME_MS (20)
+#define CLDMA_STOP_HW_POLLING_MAX_CNT (10)
+
+#define CLDMA0_BASE_ADDR (0x1021C000)
+#define CLDMA1_BASE_ADDR (0x1021E000)
+
+/* CLDMA IN(Tx) */
+#define REG_CLDMA_UL_START_ADDRL_0 (0x0004)
+#define REG_CLDMA_UL_START_ADDRH_0 (0x0008)
+#define REG_CLDMA_UL_STATUS (0x0084)
+#define REG_CLDMA_UL_START_CMD (0x0088)
+#define REG_CLDMA_UL_RESUME_CMD (0x008C)
+#define REG_CLDMA_UL_STOP_CMD (0x0090)
+#define REG_CLDMA_UL_ERROR (0x0094)
+#define REG_CLDMA_UL_CFG (0x0098)
+#define REG_CLDMA_UL_DUMMY_0 (0x009C)
+
+/* CLDMA OUT(Rx) */
+#define REG_CLDMA_SO_START_CMD (0x0400 + 0x01BC)
+#define REG_CLDMA_SO_RESUME_CMD (0x0400 + 0x01C0)
+#define REG_CLDMA_SO_STOP_CMD (0x0400 + 0x01C4)
+#define REG_CLDMA_SO_DUMMY_0 (0x0400 + 0x0108)
+#define REG_CLDMA_SO_CFG (0x0400 + 0x0004)
+#define REG_CLDMA_SO_START_ADDRL_0 (0x0400 + 0x0078)
+#define REG_CLDMA_SO_START_ADDRH_0 (0x0400 + 0x007C)
+#define REG_CLDMA_SO_CUR_ADDRL_0 (0x0400 + 0x00B8)
+#define REG_CLDMA_SO_CUR_ADDRH_0 (0x0400 + 0x00BC)
+#define REG_CLDMA_SO_STATUS (0x0400 + 0x00F8)
+
+/* CLDMA MISC */
+#define REG_CLDMA_L2TISAR0 (0x0800 + 0x0010)
+#define REG_CLDMA_L2TISAR1 (0x0800 + 0x0014)
+#define REG_CLDMA_L2TIMR0 (0x0800 + 0x0018)
+#define REG_CLDMA_L2TIMR1 (0x0800 + 0x001C)
+#define REG_CLDMA_L2TIMCR0 (0x0800 + 0x0020)
+#define REG_CLDMA_L2TIMCR1 (0x0800 + 0x0024)
+#define REG_CLDMA_L2TIMSR0 (0x0800 + 0x0028)
+#define REG_CLDMA_L2TIMSR1 (0x0800 + 0x002C)
+#define REG_CLDMA_L3TISAR0 (0x0800 + 0x0030)
+#define REG_CLDMA_L3TISAR1 (0x0800 + 0x0034)
+#define REG_CLDMA_L3TIMR0 (0x0800 + 0x0038)
+#define REG_CLDMA_L3TIMR1 (0x0800 + 0x003C)
+#define REG_CLDMA_L3TIMCR0 (0x0800 + 0x0040)
+#define REG_CLDMA_L3TIMCR1 (0x0800 + 0x0044)
+#define REG_CLDMA_L3TIMSR0 (0x0800 + 0x0048)
+#define REG_CLDMA_L3TIMSR1 (0x0800 + 0x004C)
+#define REG_CLDMA_L2RISAR0 (0x0800 + 0x0050)
+#define REG_CLDMA_L2RISAR1 (0x0800 + 0x0054)
+#define REG_CLDMA_L3RISAR0 (0x0800 + 0x0070)
+#define REG_CLDMA_L3RISAR1 (0x0800 + 0x0074)
+#define REG_CLDMA_L3RIMR0 (0x0800 + 0x0078)
+#define REG_CLDMA_L3RIMR1 (0x0800 + 0x007C)
+#define REG_CLDMA_L3RIMCR0 (0x0800 + 0x0080)
+#define REG_CLDMA_L3RIMCR1 (0x0800 + 0x0084)
+#define REG_CLDMA_L3RIMSR0 (0x0800 + 0x0088)
+#define REG_CLDMA_L3RIMSR1 (0x0800 + 0x008C)
+#define REG_CLDMA_IP_BUSY (0x0800 + 0x00B4)
+#define REG_CLDMA_L3TISAR2 (0x0800 + 0x00C0)
+#define REG_CLDMA_L3TIMR2 (0x0800 + 0x00C4)
+#define REG_CLDMA_L3TIMCR2 (0x0800 + 0x00C8)
+#define REG_CLDMA_L3TIMSR2 (0x0800 + 0x00CC)
+
+#define REG_CLDMA_L2RIMR0 (0x0800 + 0x00E8)
+#define REG_CLDMA_L2RIMR1 (0x0800 + 0x00EC)
+#define REG_CLDMA_L2RIMCR0 (0x0800 + 0x00F0)
+#define REG_CLDMA_L2RIMCR1 (0x0800 + 0x00F4)
+#define REG_CLDMA_L2RIMSR0 (0x0800 + 0x00F8)
+#define REG_CLDMA_L2RIMSR1 (0x0800 + 0x00FC)
+
+#define REG_CLDMA_INT_EAP_USIP_MASK (0x0800 + 0x011C)
+#define REG_CLDMA_RQ1_GPD_DONE_CNT (0x0800 + 0x0174)
+#define REG_CLDMA_TQ1_GPD_DONE_CNT (0x0800 + 0x0184)
+
+#define REG_CLDMA_IP_BUSY_TO_PCIE_MASK (0x0800 + 0x0194)
+#define REG_CLDMA_IP_BUSY_TO_PCIE_MASK_SET (0x0800 + 0x0198)
+#define REG_CLDMA_IP_BUSY_TO_PCIE_MASK_CLR (0x0800 + 0x019C)
+
+#define REG_CLDMA_IP_BUSY_TO_AP_MASK (0x0800 + 0x0200)
+#define REG_CLDMA_IP_BUSY_TO_AP_MASK_SET (0x0800 + 0x0204)
+#define REG_CLDMA_IP_BUSY_TO_AP_MASK_CLR (0x0800 + 0x0208)
+
+/* CLDMA RESET */
+#define REG_INFRA_RST0_SET (0x120)
+#define REG_INFRA_RST0_CLR (0x124)
+#define REG_CLDMA0_RST_SET_BIT (8)
+#define REG_CLDMA0_RST_CLR_BIT (8)
+
+static void mtk_cldma_setup_start_addr(struct mtk_md_dev *mdev, int base,
+ enum mtk_tx_rx dir, int qno, dma_addr_t addr)
+{
+ unsigned int addr_l;
+ unsigned int addr_h;
+
+ if (dir == DIR_TX) {
+ addr_l = base + REG_CLDMA_UL_START_ADDRL_0 + qno * HW_QUEUE_NUM;
+ addr_h = base + REG_CLDMA_UL_START_ADDRH_0 + qno * HW_QUEUE_NUM;
+ } else {
+ addr_l = base + REG_CLDMA_SO_START_ADDRL_0 + qno * HW_QUEUE_NUM;
+ addr_h = base + REG_CLDMA_SO_START_ADDRH_0 + qno * HW_QUEUE_NUM;
+ }
+
+ mtk_hw_write32(mdev, addr_l, (u32)addr);
+ mtk_hw_write32(mdev, addr_h, (u32)((u64)addr >> 32));
+}
+
+static void mtk_cldma_mask_intr(struct mtk_md_dev *mdev, int base,
+ enum mtk_tx_rx dir, int qno, enum mtk_intr_type type)
+{
+ u32 addr;
+ u32 val;
+
+ if (unlikely(qno < 0 || qno >= HW_QUEUE_NUM))
+ return;
+
+ if (dir == DIR_TX)
+ addr = base + REG_CLDMA_L2TIMSR0;
+ else
+ addr = base + REG_CLDMA_L2RIMSR0;
+
+ if (qno == ALLQ)
+ val = qno << type;
+ else
+ val = BIT(qno) << type;
+
+ mtk_hw_write32(mdev, addr, val);
+}
+
+static void mtk_cldma_unmask_intr(struct mtk_md_dev *mdev, int base,
+ enum mtk_tx_rx dir, int qno, enum mtk_intr_type type)
+{
+ u32 addr;
+ u32 val;
+
+ if (unlikely(qno < 0 || qno >= HW_QUEUE_NUM))
+ return;
+
+ if (dir == DIR_TX)
+ addr = base + REG_CLDMA_L2TIMCR0;
+ else
+ addr = base + REG_CLDMA_L2RIMCR0;
+
+ if (qno == ALLQ)
+ val = qno << type;
+ else
+ val = BIT(qno) << type;
+
+ mtk_hw_write32(mdev, addr, val);
+}
+
+static void mtk_cldma_clr_intr_status(struct mtk_md_dev *mdev, int base,
+ int dir, int qno, enum mtk_intr_type type)
+{
+ u32 addr;
+ u32 val;
+
+ if (unlikely(qno < 0 || qno >= HW_QUEUE_NUM))
+ return;
+
+ if (type == QUEUE_ERROR) {
+ if (dir == DIR_TX) {
+ val = mtk_hw_read32(mdev, base + REG_CLDMA_L3TISAR0);
+ mtk_hw_write32(mdev, base + REG_CLDMA_L3TISAR0, val);
+ val = mtk_hw_read32(mdev, base + REG_CLDMA_L3TISAR1);
+ mtk_hw_write32(mdev, base + REG_CLDMA_L3TISAR1, val);
+ } else {
+ val = mtk_hw_read32(mdev, base + REG_CLDMA_L3RISAR0);
+ mtk_hw_write32(mdev, base + REG_CLDMA_L3RISAR0, val);
+ val = mtk_hw_read32(mdev, base + REG_CLDMA_L3RISAR1);
+ mtk_hw_write32(mdev, base + REG_CLDMA_L3RISAR1, val);
+ }
+ }
+
+ if (dir == DIR_TX)
+ addr = base + REG_CLDMA_L2TISAR0;
+ else
+ addr = base + REG_CLDMA_L2RISAR0;
+
+ if (qno == ALLQ)
+ val = qno << type;
+ else
+ val = BIT(qno) << type;
+
+ mtk_hw_write32(mdev, addr, val);
+ val = mtk_hw_read32(mdev, addr);
+}
+
+static u32 mtk_cldma_check_intr_status(struct mtk_md_dev *mdev, int base,
+ int dir, int qno, enum mtk_intr_type type)
+{
+ u32 addr;
+ u32 val;
+ u32 sta;
+
+ if (dir == DIR_TX)
+ addr = base + REG_CLDMA_L2TISAR0;
+ else
+ addr = base + REG_CLDMA_L2RISAR0;
+
+ val = mtk_hw_read32(mdev, addr);
+ if (val == LINK_ERROR_VAL)
+ sta = val;
+ else if (qno == ALLQ)
+ sta = (val >> type) & 0xFF;
+ else
+ sta = (val >> type) & BIT(qno);
+ return sta;
+}
+
+static void mtk_cldma_start_queue(struct mtk_md_dev *mdev, int base, enum mtk_tx_rx dir, int qno)
+{
+ u32 val = BIT(qno);
+ u32 addr;
+
+ if (dir == DIR_TX)
+ addr = base + REG_CLDMA_UL_START_CMD;
+ else
+ addr = base + REG_CLDMA_SO_START_CMD;
+
+ mtk_hw_write32(mdev, addr, val);
+}
+
+static void mtk_cldma_resume_queue(struct mtk_md_dev *mdev, int base, enum mtk_tx_rx dir, int qno)
+{
+ u32 val = BIT(qno);
+ u32 addr;
+
+ if (dir == DIR_TX)
+ addr = base + REG_CLDMA_UL_RESUME_CMD;
+ else
+ addr = base + REG_CLDMA_SO_RESUME_CMD;
+
+ mtk_hw_write32(mdev, addr, val);
+}
+
+static u32 mtk_cldma_queue_status(struct mtk_md_dev *mdev, int base, enum mtk_tx_rx dir, int qno)
+{
+ u32 addr;
+ u32 val;
+
+ if (dir == DIR_TX)
+ addr = base + REG_CLDMA_UL_STATUS;
+ else
+ addr = base + REG_CLDMA_SO_STATUS;
+
+ val = mtk_hw_read32(mdev, addr);
+
+ if (qno == ALLQ || val == LINK_ERROR_VAL)
+ return val;
+ else
+ return val & BIT(qno);
+}
+
+static void mtk_cldma_mask_ip_busy_to_pci(struct mtk_md_dev *mdev,
+ int base, int qno, enum mtk_ip_busy_src type)
+{
+ if (qno == ALLQ)
+ mtk_hw_write32(mdev, base + REG_CLDMA_IP_BUSY_TO_PCIE_MASK_SET, qno << type);
+ else
+ mtk_hw_write32(mdev, base + REG_CLDMA_IP_BUSY_TO_PCIE_MASK_SET, BIT(qno) << type);
+}
+
+static void mtk_cldma_unmask_ip_busy_to_pci(struct mtk_md_dev *mdev,
+ int base, int qno, enum mtk_ip_busy_src type)
+{
+ if (qno == ALLQ)
+ mtk_hw_write32(mdev, base + REG_CLDMA_IP_BUSY_TO_PCIE_MASK_CLR, qno << type);
+ else
+ mtk_hw_write32(mdev, base + REG_CLDMA_IP_BUSY_TO_PCIE_MASK_CLR, BIT(qno) << type);
+}
+
+static void mtk_cldma_stop_queue(struct mtk_md_dev *mdev, int base, enum mtk_tx_rx dir, int qno)
+{
+ u32 val = (qno == ALLQ) ? qno : BIT(qno);
+ u32 addr;
+
+ if (dir == DIR_TX)
+ addr = base + REG_CLDMA_UL_STOP_CMD;
+ else
+ addr = base + REG_CLDMA_SO_STOP_CMD;
+
+ mtk_hw_write32(mdev, addr, val);
+}
+
+static void mtk_cldma_clear_ip_busy(struct mtk_md_dev *mdev, int base)
+{
+ mtk_hw_write32(mdev, base + REG_CLDMA_IP_BUSY, 0x01);
+}
+
+static void mtk_cldma_hw_init(struct mtk_md_dev *mdev, int base)
+{
+ u32 val = mtk_hw_read32(mdev, base + REG_CLDMA_UL_CFG);
+
+ val = (val & (~(0x7 << 5))) | ((0x4) << 5);
+ mtk_hw_write32(mdev, base + REG_CLDMA_UL_CFG, val);
+
+ val = mtk_hw_read32(mdev, base + REG_CLDMA_SO_CFG);
+ val = (val & (~(0x7 << 10))) | ((0x4) << 10) | (1 << 2);
+ mtk_hw_write32(mdev, base + REG_CLDMA_SO_CFG, val);
+
+ mtk_hw_write32(mdev, base + REG_CLDMA_IP_BUSY_TO_PCIE_MASK_CLR, 0);
+ mtk_hw_write32(mdev, base + REG_CLDMA_IP_BUSY_TO_AP_MASK_CLR, 0);
+
+ /* enable interrupt to PCIe */
+ mtk_hw_write32(mdev, base + REG_CLDMA_INT_EAP_USIP_MASK, 0);
+
+ /* disable illegal memory check */
+ mtk_hw_write32(mdev, base + REG_CLDMA_UL_DUMMY_0, 1);
+ mtk_hw_write32(mdev, base + REG_CLDMA_SO_DUMMY_0, 1);
+}
+
+static void mtk_cldma_tx_done_work(struct work_struct *work)
+{
+ struct txq *txq = container_of(work, struct txq, tx_done_work);
+ struct mtk_md_dev *mdev = txq->hw->mdev;
+ struct tx_req *req;
+ unsigned int state;
+ struct trb *trb;
+ int i;
+
+again:
+ for (i = 0; i < txq->req_pool_size; i++) {
+ req = txq->req_pool + txq->free_idx;
+ if ((req->gpd->tx_gpd.gpd_flags & CLDMA_GPD_FLAG_HWO) || !req->data_vm_addr)
+ break;
+
+ mtk_dma_unmap_single(mdev, req->data_dma_addr, req->data_len, DMA_TO_DEVICE);
+
+ trb = (struct trb *)req->skb->cb;
+ trb->status = 0;
+ trb->trb_complete(req->skb);
+
+ req->data_vm_addr = NULL;
+ req->data_dma_addr = 0;
+ req->data_len = 0;
+
+ txq->free_idx = (txq->free_idx + 1) % txq->req_pool_size;
+ mutex_lock(&txq->lock);
+ txq->req_budget++;
+ mutex_unlock(&txq->lock);
+ }
+ mtk_cldma_unmask_ip_busy_to_pci(mdev, txq->hw->base_addr, txq->txqno, IP_BUSY_TXDONE);
+ state = mtk_cldma_check_intr_status(mdev, txq->hw->base_addr,
+ DIR_TX, txq->txqno, QUEUE_XFER_DONE);
+ if (state) {
+ if (unlikely(state == LINK_ERROR_VAL))
+ return;
+
+ mtk_cldma_clr_intr_status(mdev, txq->hw->base_addr, DIR_TX,
+ txq->txqno, QUEUE_XFER_DONE);
+
+ if (need_resched()) {
+ mtk_cldma_mask_ip_busy_to_pci(mdev, txq->hw->base_addr,
+ txq->txqno, IP_BUSY_TXDONE);
+ cond_resched();
+ mtk_cldma_unmask_ip_busy_to_pci(mdev, txq->hw->base_addr,
+ txq->txqno, IP_BUSY_TXDONE);
+ }
+
+ goto again;
+ }
+
+ mtk_cldma_unmask_intr(mdev, txq->hw->base_addr, DIR_TX, txq->txqno, QUEUE_XFER_DONE);
+ mtk_cldma_clear_ip_busy(mdev, txq->hw->base_addr);
+}
+
+static void mtk_cldma_rx_done_work(struct work_struct *work)
+{
+ struct rxq *rxq = container_of(work, struct rxq, rx_done_work);
+ struct cldma_hw *hw = rxq->hw;
+ u32 curr_addr_h, curr_addr_l;
+ struct mtk_bm_pool *bm_pool;
+ struct mtk_md_dev *mdev;
+ struct rx_req *req;
+ u64 curr_addr;
+ int i, err;
+ u32 state;
+ u64 addr;
+
+ mdev = hw->mdev;
+ if (rxq->vq->rx_mtu > VQ_MTU_3_5K)
+ bm_pool = rxq->hw->cd->trans->ctrl_blk->bm_pool_63K;
+ else
+ bm_pool = rxq->hw->cd->trans->ctrl_blk->bm_pool;
+
+ do {
+ for (i = 0; i < rxq->req_pool_size; i++) {
+ req = rxq->req_pool + rxq->free_idx;
+ if ((req->gpd->rx_gpd.gpd_flags & CLDMA_GPD_FLAG_HWO)) {
+ addr = hw->base_addr + REG_CLDMA_SO_CUR_ADDRH_0 +
+ (u64)rxq->rxqno * HW_QUEUE_NUM;
+ curr_addr_h = mtk_hw_read32(mdev, addr);
+ addr = hw->base_addr + REG_CLDMA_SO_CUR_ADDRL_0 +
+ (u64)rxq->rxqno * HW_QUEUE_NUM;
+ curr_addr_l = mtk_hw_read32(mdev, addr);
+ curr_addr = ((u64)curr_addr_h << 32) | curr_addr_l;
+
+ if (req->gpd_dma_addr == curr_addr &&
+ (req->gpd->rx_gpd.gpd_flags & CLDMA_GPD_FLAG_HWO))
+ break;
+ }
+
+ mtk_dma_unmap_single(mdev, req->data_dma_addr, req->mtu, DMA_FROM_DEVICE);
+
+ rxq->rx_done(req->skb, le16_to_cpu(req->gpd->rx_gpd.data_recv_len),
+ rxq->arg);
+
+ rxq->free_idx = (rxq->free_idx + 1) % rxq->req_pool_size;
+ req->skb = mtk_bm_alloc(bm_pool);
+ if (!req->skb)
+ break;
+
+ err = mtk_dma_map_single(mdev, &req->data_dma_addr, req->skb->data,
+ req->mtu, DMA_FROM_DEVICE);
+ if (unlikely(err)) {
+ mtk_bm_free(bm_pool, req->skb);
+ break;
+ }
+
+ req->gpd->rx_gpd.data_recv_len = 0;
+ req->gpd->rx_gpd.data_buff_ptr_h =
+ cpu_to_le32((u64)req->data_dma_addr >> 32);
+ req->gpd->rx_gpd.data_buff_ptr_l = cpu_to_le32(req->data_dma_addr);
+ req->gpd->rx_gpd.gpd_flags = CLDMA_GPD_FLAG_IOC | CLDMA_GPD_FLAG_HWO;
+ }
+
+ mtk_cldma_resume_queue(mdev, rxq->hw->base_addr, DIR_RX, rxq->rxqno);
+ state = mtk_cldma_check_intr_status(mdev, rxq->hw->base_addr,
+ DIR_RX, rxq->rxqno, QUEUE_XFER_DONE);
+
+ if (!state)
+ break;
+
+ mtk_cldma_clr_intr_status(mdev, rxq->hw->base_addr, DIR_RX,
+ rxq->rxqno, QUEUE_XFER_DONE);
+
+ if (need_resched())
+ cond_resched();
+ } while (true);
+
+ mtk_cldma_unmask_intr(mdev, rxq->hw->base_addr, DIR_RX, rxq->rxqno, QUEUE_XFER_DONE);
+ mtk_cldma_mask_ip_busy_to_pci(mdev, rxq->hw->base_addr, rxq->rxqno, IP_BUSY_RXDONE);
+ mtk_cldma_clear_ip_busy(mdev, rxq->hw->base_addr);
+}
+
+static int mtk_cldma_isr(int irq_id, void *param)
+{
+ u32 txq_xfer_done, rxq_xfer_done;
+ struct cldma_hw *hw = param;
+ u32 tx_mask, rx_mask;
+ u32 txq_err, rxq_err;
+ u32 tx_sta, rx_sta;
+ struct txq *txq;
+ struct rxq *rxq;
+ int i;
+
+ tx_sta = mtk_hw_read32(hw->mdev, hw->base_addr + REG_CLDMA_L2TISAR0);
+ tx_mask = mtk_hw_read32(hw->mdev, hw->base_addr + REG_CLDMA_L2TIMR0);
+ rx_sta = mtk_hw_read32(hw->mdev, hw->base_addr + REG_CLDMA_L2RISAR0);
+ rx_mask = mtk_hw_read32(hw->mdev, hw->base_addr + REG_CLDMA_L2RIMR0);
+
+ tx_sta = tx_sta & (~tx_mask);
+ rx_sta = rx_sta & (~rx_mask);
+
+ if (tx_sta) {
+ /* TX mask */
+ mtk_hw_write32(hw->mdev, hw->base_addr + REG_CLDMA_L2TIMSR0, tx_sta);
+
+ txq_err = (tx_sta >> QUEUE_ERROR) & 0xFF;
+ if (txq_err) {
+ mtk_cldma_clr_intr_status(hw->mdev, hw->base_addr,
+ DIR_TX, ALLQ, QUEUE_ERROR);
+ mtk_hw_write32(hw->mdev, hw->base_addr + REG_CLDMA_L2TIMCR0,
+ (txq_err << QUEUE_ERROR));
+ }
+
+ /* TX clear */
+ mtk_hw_write32(hw->mdev, hw->base_addr + REG_CLDMA_L2TISAR0, tx_sta);
+
+ txq_xfer_done = (tx_sta >> QUEUE_XFER_DONE) & 0xFF;
+ if (txq_xfer_done) {
+ for (i = 0; i < HW_QUEUE_NUM; i++) {
+ if (txq_xfer_done & (1 << i)) {
+ txq = hw->txq[i];
+ queue_work(hw->wq, &txq->tx_done_work);
+ }
+ }
+ }
+ }
+
+ if (rx_sta) {
+ /* RX mask */
+ mtk_hw_write32(hw->mdev, hw->base_addr + REG_CLDMA_L2RIMSR0, rx_sta);
+
+ rxq_err = (rx_sta >> QUEUE_ERROR) & 0xFF;
+ if (rxq_err) {
+ mtk_cldma_clr_intr_status(hw->mdev, hw->base_addr,
+ DIR_RX, ALLQ, QUEUE_ERROR);
+ mtk_hw_write32(hw->mdev, hw->base_addr + REG_CLDMA_L2RIMCR0,
+ (rxq_err << QUEUE_ERROR));
+ }
+
+ /* RX clear */
+ mtk_hw_write32(hw->mdev, hw->base_addr + REG_CLDMA_L2RISAR0, rx_sta);
+
+ rxq_xfer_done = (rx_sta >> QUEUE_XFER_DONE) & 0xFF;
+ if (rxq_xfer_done) {
+ for (i = 0; i < HW_QUEUE_NUM; i++) {
+ if (rxq_xfer_done & (1 << i)) {
+ rxq = hw->rxq[i];
+ queue_work(hw->wq, &rxq->rx_done_work);
+ }
+ }
+ }
+ }
+
+ mtk_hw_clear_irq(hw->mdev, hw->pci_ext_irq_id);
+ mtk_hw_unmask_irq(hw->mdev, hw->pci_ext_irq_id);
+
+ return IRQ_HANDLED;
+}
+
+int mtk_cldma_hw_init_t800(struct cldma_dev *cd, int hif_id)
+{
+ char pool_name[DMA_POOL_NAME_LEN];
+ struct cldma_hw *hw;
+ unsigned int flag;
+
+ if (cd->cldma_hw[hif_id])
+ return 0;
+
+ hw = devm_kzalloc(cd->trans->mdev->dev, sizeof(*hw), GFP_KERNEL);
+ if (!hw)
+ return -ENOMEM;
+
+ hw->cd = cd;
+ hw->mdev = cd->trans->mdev;
+ hw->hif_id = ((CLDMA_CLASS_ID) << 8) + hif_id;
+ snprintf(pool_name, DMA_POOL_NAME_LEN, "cldma%d_pool_%s", hw->hif_id, hw->mdev->dev_str);
+ hw->dma_pool = mtk_dma_pool_create(hw->mdev, pool_name, sizeof(union gpd), 64, 0);
+ if (!hw->dma_pool)
+ goto err_exit;
+
+ switch (hif_id) {
+ case CLDMA0:
+ hw->pci_ext_irq_id = mtk_hw_get_irq_id(hw->mdev, MTK_IRQ_SRC_CLDMA0);
+ hw->base_addr = CLDMA0_BASE_ADDR;
+ break;
+ case CLDMA1:
+ hw->pci_ext_irq_id = mtk_hw_get_irq_id(hw->mdev, MTK_IRQ_SRC_CLDMA1);
+ hw->base_addr = CLDMA1_BASE_ADDR;
+ break;
+ default:
+ break;
+ }
+
+ flag = WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_HIGHPRI;
+ hw->wq = alloc_workqueue("cldma%d_workq_%s", flag, 0, hif_id, hw->mdev->dev_str);
+
+ mtk_cldma_hw_init(hw->mdev, hw->base_addr);
+
+ /* mask/clear PCI CLDMA L1 interrupt */
+ mtk_hw_mask_irq(hw->mdev, hw->pci_ext_irq_id);
+ mtk_hw_clear_irq(hw->mdev, hw->pci_ext_irq_id);
+
+ /* register CLDMA interrupt handler */
+ mtk_hw_register_irq(hw->mdev, hw->pci_ext_irq_id, mtk_cldma_isr, hw);
+
+ /* unmask PCI CLDMA L1 interrupt */
+ mtk_hw_unmask_irq(hw->mdev, hw->pci_ext_irq_id);
+
+ cd->cldma_hw[hif_id] = hw;
+ return 0;
+
+err_exit:
+ devm_kfree(hw->mdev->dev, hw);
+
+ return -EIO;
+}
+
+int mtk_cldma_hw_exit_t800(struct cldma_dev *cd, int hif_id)
+{
+ struct mtk_md_dev *mdev;
+ struct cldma_hw *hw;
+ int i;
+
+ if (!cd->cldma_hw[hif_id])
+ return 0;
+
+ /* free cldma descriptor */
+ hw = cd->cldma_hw[hif_id];
+ mdev = cd->trans->mdev;
+ mtk_hw_mask_irq(mdev, hw->pci_ext_irq_id);
+ for (i = 0; i < HW_QUEUE_NUM; i++) {
+ if (hw->txq[i])
+ cd->hw_ops.txq_free(hw, hw->txq[i]->vqno);
+ if (hw->rxq[i])
+ cd->hw_ops.rxq_free(hw, hw->rxq[i]->vqno);
+ }
+
+ flush_workqueue(hw->wq);
+ destroy_workqueue(hw->wq);
+ mtk_dma_pool_destroy(hw->dma_pool);
+ mtk_hw_unregister_irq(mdev, hw->pci_ext_irq_id);
+
+ devm_kfree(mdev->dev, hw);
+ cd->cldma_hw[hif_id] = NULL;
+
+ return 0;
+}
+
+struct txq *mtk_cldma_txq_alloc_t800(struct cldma_hw *hw, struct sk_buff *skb)
+{
+ struct trb *trb = (struct trb *)skb->cb;
+ struct tx_req *next;
+ struct tx_req *req;
+ struct txq *txq;
+ int i;
+
+ txq = devm_kzalloc(hw->mdev->dev, sizeof(*txq), GFP_KERNEL);
+ if (!txq)
+ return NULL;
+
+ txq->hw = hw;
+ txq->vqno = trb->vqno;
+ txq->vq = hw->cd->trans->vq_tbl + trb->vqno;
+ txq->txqno = txq->vq->txqno;
+ txq->req_pool_size = txq->vq->tx_req_num;
+ txq->req_budget = txq->vq->tx_req_num;
+ txq->is_stopping = false;
+ mutex_init(&txq->lock);
+ if (unlikely(txq->txqno < 0 || txq->txqno >= HW_QUEUE_NUM))
+ goto err_exit;
+
+ txq->req_pool = devm_kcalloc(hw->mdev->dev, txq->req_pool_size, sizeof(*req), GFP_KERNEL);
+ if (!txq->req_pool)
+ goto err_exit;
+
+ for (i = 0; i < txq->req_pool_size; i++) {
+ req = txq->req_pool + i;
+ req->mtu = txq->vq->tx_mtu;
+ req->gpd = mtk_dma_pool_alloc(hw->dma_pool, GFP_KERNEL, &req->gpd_dma_addr);
+ if (!req->gpd)
+ goto exit_free_req;
+ }
+
+ for (i = 0; i < txq->req_pool_size; i++) {
+ req = txq->req_pool + i;
+ next = txq->req_pool + ((i + 1) % txq->req_pool_size);
+ req->gpd->tx_gpd.next_gpd_ptr_h = cpu_to_le32((u64)(next->gpd_dma_addr) >> 32);
+ req->gpd->tx_gpd.next_gpd_ptr_l = cpu_to_le32(next->gpd_dma_addr);
+ }
+
+ INIT_WORK(&txq->tx_done_work, mtk_cldma_tx_done_work);
+
+ mtk_cldma_stop_queue(hw->mdev, hw->base_addr, DIR_TX, txq->txqno);
+ txq->tx_started = false;
+ mtk_cldma_setup_start_addr(hw->mdev, hw->base_addr, DIR_TX, txq->txqno,
+ txq->req_pool[0].gpd_dma_addr);
+ mtk_cldma_unmask_intr(hw->mdev, hw->base_addr, DIR_TX, txq->txqno, QUEUE_ERROR);
+ mtk_cldma_unmask_intr(hw->mdev, hw->base_addr, DIR_TX, txq->txqno, QUEUE_XFER_DONE);
+
+ hw->txq[txq->txqno] = txq;
+ return txq;
+
+exit_free_req:
+ for (i--; i >= 0; i--) {
+ req = txq->req_pool + i;
+ mtk_dma_pool_free(hw->dma_pool, req->gpd, req->gpd_dma_addr);
+ }
+
+ devm_kfree(hw->mdev->dev, txq->req_pool);
+err_exit:
+ devm_kfree(hw->mdev->dev, txq);
+ return NULL;
+}
+
+int mtk_cldma_txq_free_t800(struct cldma_hw *hw, int vqno)
+{
+ struct virtq *vq = hw->cd->trans->vq_tbl + vqno;
+ unsigned int active;
+ struct tx_req *req;
+ struct txq *txq;
+ struct trb *trb;
+ int cnt = 0;
+ int irq_id;
+ int txqno;
+ int i;
+
+ txqno = vq->txqno;
+ if (unlikely(txqno < 0 || txqno >= HW_QUEUE_NUM))
+ return -EINVAL;
+ txq = hw->txq[txqno];
+ if (!txq)
+ return -EINVAL;
+
+ /* stop HW tx transaction */
+ mtk_cldma_stop_queue(hw->mdev, hw->base_addr, DIR_TX, txqno);
+ txq->tx_started = false;
+ do {
+ active = mtk_cldma_queue_status(hw->mdev, hw->base_addr, DIR_TX, txqno);
+ if (active == LINK_ERROR_VAL)
+ break;
+ msleep(CLDMA_STOP_HW_WAIT_TIME_MS); /* ensure HW tx transaction done */
+ cnt++;
+ } while (active && cnt < CLDMA_STOP_HW_POLLING_MAX_CNT);
+
+ irq_id = mtk_hw_get_virq_id(hw->mdev, hw->pci_ext_irq_id);
+ synchronize_irq(irq_id);
+
+ flush_work(&txq->tx_done_work);
+ mtk_cldma_mask_intr(hw->mdev, hw->base_addr, DIR_TX, txqno, QUEUE_XFER_DONE);
+ mtk_cldma_mask_intr(hw->mdev, hw->base_addr, DIR_TX, txqno, QUEUE_ERROR);
+
+ /* free tx req resource */
+ for (i = 0; i < txq->req_pool_size; i++) {
+ req = txq->req_pool + i;
+ if (req->data_dma_addr && req->data_len) {
+ mtk_dma_unmap_single(hw->mdev, req->data_dma_addr,
+ req->data_len, DMA_TO_DEVICE);
+ trb = (struct trb *)req->skb->cb;
+ trb->status = -EPIPE;
+ trb->trb_complete(req->skb);
+ }
+ mtk_dma_pool_free(hw->dma_pool, req->gpd, req->gpd_dma_addr);
+ }
+
+ devm_kfree(hw->mdev->dev, txq->req_pool);
+ devm_kfree(hw->mdev->dev, txq);
+ hw->txq[txqno] = NULL;
+
+ return 0;
+}
+
+struct rxq *mtk_cldma_rxq_alloc_t800(struct cldma_hw *hw, struct sk_buff *skb)
+{
+ struct trb_open_priv *trb_open_priv = (struct trb_open_priv *)skb->data;
+ struct trb *trb = (struct trb *)skb->cb;
+ struct mtk_bm_pool *bm_pool;
+ struct rx_req *next;
+ struct rx_req *req;
+ struct rxq *rxq;
+ int err;
+ int i;
+
+ rxq = devm_kzalloc(hw->mdev->dev, sizeof(*rxq), GFP_KERNEL);
+ if (!rxq)
+ return NULL;
+
+ rxq->hw = hw;
+ rxq->vqno = trb->vqno;
+ rxq->vq = hw->cd->trans->vq_tbl + trb->vqno;
+ rxq->rxqno = rxq->vq->rxqno;
+ rxq->req_pool_size = rxq->vq->rx_req_num;
+ rxq->arg = trb->priv;
+ rxq->rx_done = trb_open_priv->rx_done;
+ if (unlikely(rxq->rxqno < 0 || rxq->rxqno >= HW_QUEUE_NUM))
+ goto err_exit;
+
+ rxq->req_pool = devm_kcalloc(hw->mdev->dev, rxq->req_pool_size, sizeof(*req), GFP_KERNEL);
+ if (!rxq->req_pool)
+ goto err_exit;
+
+ if (rxq->vq->rx_mtu > VQ_MTU_3_5K)
+ bm_pool = hw->cd->trans->ctrl_blk->bm_pool_63K;
+ else
+ bm_pool = hw->cd->trans->ctrl_blk->bm_pool;
+
+ /* setup rx request */
+ for (i = 0; i < rxq->req_pool_size; i++) {
+ req = rxq->req_pool + i;
+ req->mtu = rxq->vq->rx_mtu;
+ req->gpd = mtk_dma_pool_alloc(hw->dma_pool, GFP_KERNEL, &req->gpd_dma_addr);
+ if (!req->gpd)
+ goto exit_free_req;
+
+ req->skb = mtk_bm_alloc(bm_pool);
+ if (!req->skb) {
+ mtk_dma_pool_free(hw->dma_pool, req->gpd, req->gpd_dma_addr);
+ goto exit_free_req;
+ }
+
+ err = mtk_dma_map_single(hw->mdev, &req->data_dma_addr, req->skb->data,
+ req->mtu, DMA_FROM_DEVICE);
+ if (err) {
+ i++;
+ goto exit_free_req;
+ }
+ }
+
+ for (i = 0; i < rxq->req_pool_size; i++) {
+ req = rxq->req_pool + i;
+ next = rxq->req_pool + ((i + 1) % rxq->req_pool_size);
+ req->gpd->rx_gpd.gpd_flags = CLDMA_GPD_FLAG_IOC | CLDMA_GPD_FLAG_HWO;
+ req->gpd->rx_gpd.data_allow_len = cpu_to_le16(req->mtu);
+ req->gpd->rx_gpd.next_gpd_ptr_h = cpu_to_le32((u64)(next->gpd_dma_addr) >> 32);
+ req->gpd->rx_gpd.next_gpd_ptr_l = cpu_to_le32(next->gpd_dma_addr);
+ req->gpd->rx_gpd.data_buff_ptr_h = cpu_to_le32((u64)(req->data_dma_addr) >> 32);
+ req->gpd->rx_gpd.data_buff_ptr_l = cpu_to_le32(req->data_dma_addr);
+ }
+
+ INIT_WORK(&rxq->rx_done_work, mtk_cldma_rx_done_work);
+
+ hw->rxq[rxq->rxqno] = rxq;
+ mtk_cldma_stop_queue(hw->mdev, hw->base_addr, DIR_RX, rxq->rxqno);
+ mtk_cldma_setup_start_addr(hw->mdev, hw->base_addr, DIR_RX,
+ rxq->rxqno, rxq->req_pool[0].gpd_dma_addr);
+ mtk_cldma_start_queue(hw->mdev, hw->base_addr, DIR_RX, rxq->rxqno);
+ mtk_cldma_unmask_intr(hw->mdev, hw->base_addr, DIR_RX, rxq->rxqno, QUEUE_ERROR);
+ mtk_cldma_unmask_intr(hw->mdev, hw->base_addr, DIR_RX, rxq->rxqno, QUEUE_XFER_DONE);
+
+ return rxq;
+
+exit_free_req:
+ for (i--; i >= 0; i--) {
+ req = rxq->req_pool + i;
+ mtk_dma_unmap_single(hw->mdev, req->data_dma_addr, req->mtu, DMA_FROM_DEVICE);
+ mtk_dma_pool_free(hw->dma_pool, req->gpd, req->gpd_dma_addr);
+ if (req->skb)
+ mtk_bm_free(bm_pool, req->skb);
+ }
+
+ devm_kfree(hw->mdev->dev, rxq->req_pool);
+err_exit:
+ devm_kfree(hw->mdev->dev, rxq);
+ return NULL;
+}
+
+int mtk_cldma_rxq_free_t800(struct cldma_hw *hw, int vqno)
+{
+ struct mtk_bm_pool *bm_pool;
+ struct mtk_md_dev *mdev;
+ unsigned int active;
+ struct rx_req *req;
+ struct virtq *vq;
+ struct rxq *rxq;
+ int cnt = 0;
+ int irq_id;
+ int rxqno;
+ int i;
+
+ mdev = hw->mdev;
+ vq = hw->cd->trans->vq_tbl + vqno;
+ rxqno = vq->rxqno;
+ if (unlikely(rxqno < 0 || rxqno >= HW_QUEUE_NUM))
+ return -EINVAL;
+ rxq = hw->rxq[rxqno];
+ if (!rxq)
+ return -EINVAL;
+
+ if (rxq->vq->rx_mtu > VQ_MTU_3_5K)
+ bm_pool = hw->cd->trans->ctrl_blk->bm_pool_63K;
+ else
+ bm_pool = hw->cd->trans->ctrl_blk->bm_pool;
+
+ mtk_cldma_stop_queue(mdev, hw->base_addr, DIR_RX, rxqno);
+ do {
+ /* check CLDMA HW state register */
+ active = mtk_cldma_queue_status(mdev, hw->base_addr, DIR_RX, rxqno);
+ if (active == LINK_ERROR_VAL)
+ break;
+ msleep(CLDMA_STOP_HW_WAIT_TIME_MS); /* ensure HW rx transaction done */
+ cnt++;
+ } while (active && cnt < CLDMA_STOP_HW_POLLING_MAX_CNT);
+
+ irq_id = mtk_hw_get_virq_id(hw->mdev, hw->pci_ext_irq_id);
+ synchronize_irq(irq_id);
+
+ flush_work(&rxq->rx_done_work);
+ mtk_cldma_mask_intr(mdev, hw->base_addr, DIR_RX, rxqno, QUEUE_XFER_DONE);
+ mtk_cldma_mask_intr(mdev, hw->base_addr, DIR_RX, rxqno, QUEUE_ERROR);
+
+ /* free rx req resource */
+ for (i = 0; i < rxq->req_pool_size; i++) {
+ req = rxq->req_pool + i;
+ if (!(req->gpd->rx_gpd.gpd_flags & CLDMA_GPD_FLAG_HWO) &&
+ le16_to_cpu(req->gpd->rx_gpd.data_recv_len)) {
+ mtk_dma_unmap_single(mdev, req->data_dma_addr,
+ req->mtu, DMA_FROM_DEVICE);
+ rxq->rx_done(req->skb, le16_to_cpu(req->gpd->rx_gpd.data_recv_len),
+ rxq->arg);
+ req->skb = NULL;
+ }
+
+ mtk_dma_pool_free(hw->dma_pool, req->gpd, req->gpd_dma_addr);
+ if (req->skb) {
+ mtk_bm_free(bm_pool, req->skb);
+ mtk_dma_unmap_single(mdev, req->data_dma_addr,
+ req->mtu, DMA_FROM_DEVICE);
+ }
+ }
+
+ devm_kfree(mdev->dev, rxq->req_pool);
+ devm_kfree(mdev->dev, rxq);
+ hw->rxq[rxqno] = NULL;
+
+ return 0;
+}
+
+int mtk_cldma_start_xfer_t800(struct cldma_hw *hw, int qno)
+{
+ struct txq *txq;
+ u32 addr, val;
+ int idx;
+
+ txq = hw->txq[qno];
+ addr = hw->base_addr + REG_CLDMA_UL_START_ADDRL_0 + qno * HW_QUEUE_NUM;
+ val = mtk_hw_read32(hw->mdev, addr);
+ if (unlikely(!val)) {
+ mtk_cldma_hw_init(hw->mdev, hw->base_addr);
+ txq = hw->txq[qno];
+ idx = (txq->wr_idx + txq->req_pool_size - 1) % txq->req_pool_size;
+ mtk_cldma_setup_start_addr(hw->mdev, hw->base_addr, DIR_TX, qno,
+ txq->req_pool[idx].gpd_dma_addr);
+ mtk_cldma_start_queue(hw->mdev, hw->base_addr, DIR_TX, qno);
+ txq->tx_started = true;
+ } else {
+ if (unlikely(!txq->tx_started)) {
+ mtk_cldma_start_queue(hw->mdev, hw->base_addr, DIR_TX, qno);
+ txq->tx_started = true;
+ } else {
+ mtk_cldma_resume_queue(hw->mdev, hw->base_addr, DIR_TX, qno);
+ }
+ }
+
+ return 0;
+}
new file mode 100644
@@ -0,0 +1,20 @@
+/* SPDX-License-Identifier: BSD-3-Clause-Clear
+ *
+ * Copyright (c) 2022, MediaTek Inc.
+ */
+
+#ifndef __MTK_CLDMA_DRV_T800_H__
+#define __MTK_CLDMA_DRV_T800_H__
+
+#include <linux/skbuff.h>
+
+#include "mtk_cldma.h"
+
+int mtk_cldma_hw_init_t800(struct cldma_dev *cd, int hif_id);
+int mtk_cldma_hw_exit_t800(struct cldma_dev *cd, int hif_id);
+struct txq *mtk_cldma_txq_alloc_t800(struct cldma_hw *hw, struct sk_buff *skb);
+int mtk_cldma_txq_free_t800(struct cldma_hw *hw, int vqno);
+struct rxq *mtk_cldma_rxq_alloc_t800(struct cldma_hw *hw, struct sk_buff *skb);
+int mtk_cldma_rxq_free_t800(struct cldma_hw *hw, int vqno);
+int mtk_cldma_start_xfer_t800(struct cldma_hw *hw, int qno);
+#endif