@@ -132,6 +132,11 @@ static inline struct nvme_rdma_ctrl *to_rdma_ctrl(struct nvme_ctrl *ctrl)
return container_of(ctrl, struct nvme_rdma_ctrl, ctrl);
}
+static inline int nvme_rdma_queue_id(struct nvme_rdma_queue *queue)
+{
+ return queue - queue->ctrl->queues;
+}
+
static LIST_HEAD(device_list);
static DEFINE_MUTEX(device_list_mutex);
@@ -566,13 +571,19 @@ static int nvme_rdma_create_queue_ib(struct nvme_rdma_queue *queue)
return ret;
}
-static int nvme_rdma_alloc_queue(struct nvme_rdma_ctrl *ctrl,
- int idx, size_t queue_size)
+static int nvme_rdma_alloc_queue(struct nvme_ctrl *nctrl, int idx)
{
+ struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
struct nvme_rdma_queue *queue;
struct sockaddr *src_addr = NULL;
+ size_t queue_size;
int ret;
+ if (idx == 0)
+ queue_size = NVME_AQ_DEPTH;
+ else
+ queue_size = ctrl->ctrl.sqsize + 1;
+
queue = &ctrl->queues[idx];
mutex_init(&queue->queue_lock);
queue->ctrl = ctrl;
@@ -636,16 +647,22 @@ static void __nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
ib_drain_qp(queue->qp);
}
-static void nvme_rdma_stop_queue(struct nvme_rdma_queue *queue)
+static void nvme_rdma_stop_queue(struct nvme_ctrl *nctrl, int qid)
{
+ struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
+ struct nvme_rdma_queue *queue = &ctrl->queues[qid];
+
mutex_lock(&queue->queue_lock);
if (test_and_clear_bit(NVME_RDMA_Q_LIVE, &queue->flags))
__nvme_rdma_stop_queue(queue);
mutex_unlock(&queue->queue_lock);
}
-static void nvme_rdma_free_queue(struct nvme_rdma_queue *queue)
+static void nvme_rdma_free_queue(struct nvme_ctrl *nctrl, int qid)
{
+ struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
+ struct nvme_rdma_queue *queue = &ctrl->queues[qid];
+
if (!test_and_clear_bit(NVME_RDMA_Q_ALLOCATED, &queue->flags))
return;
@@ -659,7 +676,7 @@ static void nvme_rdma_free_io_queues(struct nvme_rdma_ctrl *ctrl)
int i;
for (i = 1; i < ctrl->ctrl.queue_count; i++)
- nvme_rdma_free_queue(&ctrl->queues[i]);
+ nvme_rdma_free_queue(&ctrl->ctrl, i);
}
static void nvme_rdma_stop_io_queues(struct nvme_rdma_ctrl *ctrl)
@@ -667,18 +684,19 @@ static void nvme_rdma_stop_io_queues(struct nvme_rdma_ctrl *ctrl)
int i;
for (i = 1; i < ctrl->ctrl.queue_count; i++)
- nvme_rdma_stop_queue(&ctrl->queues[i]);
+ nvme_rdma_stop_queue(&ctrl->ctrl, i);
}
-static int nvme_rdma_start_queue(struct nvme_rdma_ctrl *ctrl, int idx)
+static int nvme_rdma_start_queue(struct nvme_ctrl *nctrl, int idx)
{
+ struct nvme_rdma_ctrl *ctrl = to_rdma_ctrl(nctrl);
struct nvme_rdma_queue *queue = &ctrl->queues[idx];
int ret;
if (idx)
- ret = nvmf_connect_io_queue(&ctrl->ctrl, idx);
+ ret = nvmf_connect_io_queue(nctrl, idx);
else
- ret = nvmf_connect_admin_queue(&ctrl->ctrl);
+ ret = nvmf_connect_admin_queue(nctrl);
if (!ret) {
set_bit(NVME_RDMA_Q_LIVE, &queue->flags);
@@ -697,7 +715,7 @@ static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl,
int i, ret = 0;
for (i = first; i < last; i++) {
- ret = nvme_rdma_start_queue(ctrl, i);
+ ret = nvme_rdma_start_queue(&ctrl->ctrl, i);
if (ret)
goto out_stop_queues;
}
@@ -706,7 +724,7 @@ static int nvme_rdma_start_io_queues(struct nvme_rdma_ctrl *ctrl,
out_stop_queues:
for (i--; i >= first; i--)
- nvme_rdma_stop_queue(&ctrl->queues[i]);
+ nvme_rdma_stop_queue(&ctrl->ctrl, i);
return ret;
}
@@ -768,8 +786,7 @@ static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl)
}
for (i = 1; i < ctrl->ctrl.queue_count; i++) {
- ret = nvme_rdma_alloc_queue(ctrl, i,
- ctrl->ctrl.sqsize + 1);
+ ret = nvme_rdma_alloc_queue(&ctrl->ctrl, i);
if (ret)
goto out_free_queues;
}
@@ -778,7 +795,7 @@ static int nvme_rdma_alloc_io_queues(struct nvme_rdma_ctrl *ctrl)
out_free_queues:
for (i--; i >= 1; i--)
- nvme_rdma_free_queue(&ctrl->queues[i]);
+ nvme_rdma_free_queue(&ctrl->ctrl, i);
return ret;
}
@@ -806,7 +823,7 @@ static void nvme_rdma_destroy_admin_queue(struct nvme_rdma_ctrl *ctrl)
sizeof(struct nvme_command), DMA_TO_DEVICE);
ctrl->async_event_sqe.data = NULL;
}
- nvme_rdma_free_queue(&ctrl->queues[0]);
+ nvme_rdma_free_queue(&ctrl->ctrl, 0);
}
static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
@@ -815,7 +832,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
bool pi_capable = false;
int error;
- error = nvme_rdma_alloc_queue(ctrl, 0, NVME_AQ_DEPTH);
+ error = nvme_rdma_alloc_queue(&ctrl->ctrl, 0);
if (error)
return error;
@@ -850,7 +867,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
}
- error = nvme_rdma_start_queue(ctrl, 0);
+ error = nvme_rdma_start_queue(&ctrl->ctrl, 0);
if (error)
goto out_remove_admin_tag_set;
@@ -877,7 +894,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
nvme_quiesce_admin_queue(&ctrl->ctrl);
blk_sync_queue(ctrl->ctrl.admin_q);
out_stop_queue:
- nvme_rdma_stop_queue(&ctrl->queues[0]);
+ nvme_rdma_stop_queue(&ctrl->ctrl, 0);
nvme_cancel_admin_tagset(&ctrl->ctrl);
out_remove_admin_tag_set:
if (new)
@@ -889,7 +906,7 @@ static int nvme_rdma_configure_admin_queue(struct nvme_rdma_ctrl *ctrl,
ctrl->async_event_sqe.data = NULL;
}
out_free_queue:
- nvme_rdma_free_queue(&ctrl->queues[0]);
+ nvme_rdma_free_queue(&ctrl->ctrl, 0);
return error;
}
@@ -962,7 +979,7 @@ static void nvme_rdma_teardown_admin_queue(struct nvme_rdma_ctrl *ctrl,
{
nvme_quiesce_admin_queue(&ctrl->ctrl);
blk_sync_queue(ctrl->ctrl.admin_q);
- nvme_rdma_stop_queue(&ctrl->queues[0]);
+ nvme_rdma_stop_queue(&ctrl->ctrl, 0);
nvme_cancel_admin_tagset(&ctrl->ctrl);
if (remove) {
nvme_unquiesce_admin_queue(&ctrl->ctrl);
@@ -1113,7 +1130,7 @@ static int nvme_rdma_setup_ctrl(struct nvme_rdma_ctrl *ctrl, bool new)
destroy_admin:
nvme_quiesce_admin_queue(&ctrl->ctrl);
blk_sync_queue(ctrl->ctrl.admin_q);
- nvme_rdma_stop_queue(&ctrl->queues[0]);
+ nvme_rdma_stop_queue(&ctrl->ctrl, 0);
nvme_cancel_admin_tagset(&ctrl->ctrl);
if (new)
nvme_remove_admin_tag_set(&ctrl->ctrl);
@@ -1960,9 +1977,10 @@ static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id,
static void nvme_rdma_complete_timed_out(struct request *rq)
{
struct nvme_rdma_request *req = blk_mq_rq_to_pdu(rq);
+ struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
struct nvme_rdma_queue *queue = req->queue;
- nvme_rdma_stop_queue(queue);
+ nvme_rdma_stop_queue(ctrl, nvme_rdma_queue_id(queue));
nvmf_complete_timed_out_request(rq);
}