@@ -4,6 +4,7 @@
#include <linux/sched.h>
#include <linux/xarray.h>
+#include <linux/bvec.h>
#include <uapi/linux/io_uring.h>
enum io_uring_cmd_flags {
@@ -20,6 +21,23 @@ enum io_uring_cmd_flags {
IO_URING_F_SQE128 = (1 << 8),
IO_URING_F_CQE32 = (1 << 9),
IO_URING_F_IOPOLL = (1 << 10),
+
+ /* for FUSED_CMD with IORING_FUSED_CMD_BUF only */
+ IO_URING_F_FUSED_BUF_DEST = (1 << 11), /* secondary write to buffer */
+ IO_URING_F_FUSED_BUF_SRC = (1 << 12), /* secondary read from buffer */
+ /* driver incapable of FUSED_CMD should fail cmd when seeing F_FUSED */
+ IO_URING_F_FUSED_BUF = IO_URING_F_FUSED_BUF_DEST |
+ IO_URING_F_FUSED_BUF_SRC,
+};
+
+struct io_uring_bvec_buf {
+ unsigned long len;
+ unsigned int nr_bvecs;
+
+ /* offset in the 1st bvec */
+ unsigned int offset;
+ const struct bio_vec *bvec;
+ struct bio_vec __bvec[];
};
union io_uring_fused_cmd_data {
@@ -50,6 +68,9 @@ struct io_uring_cmd {
};
#if defined(CONFIG_IO_URING)
+void io_fused_provide_buf_and_start(struct io_uring_cmd *cmd,
+ unsigned issue_flags, const struct io_uring_bvec_buf *buf,
+ void (*complete_tw_cb)(struct io_uring_cmd *, unsigned));
int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
struct iov_iter *iter, void *ioucmd);
void io_uring_cmd_done(struct io_uring_cmd *cmd, ssize_t ret, ssize_t res2,
@@ -80,6 +101,11 @@ static inline void io_uring_free(struct task_struct *tsk)
__io_uring_free(tsk);
}
#else
+static inline void io_fused_provide_buf_and_start(struct io_uring_cmd *cmd,
+ unsigned issue_flags, const struct io_uring_bvec_buf *buf,
+ void (*complete_tw_cb)(struct io_uring_cmd *, unsigned))
+{
+}
static inline int io_uring_cmd_import_fixed(u64 ubuf, unsigned long len, int rw,
struct iov_iter *iter, void *ioucmd)
{
@@ -562,6 +562,11 @@ struct io_kiocb {
*/
struct io_buffer_list *buf_list;
+ /*
+ * store kernel buffer provided by fused primary request
+ */
+ const struct io_uring_bvec_buf *fused_cmd_kbuf;
+
/* store fused command's primary request for the secondary */
struct io_kiocb *fused_primary_req;
};
@@ -245,8 +245,14 @@ enum io_uring_op {
* sqe->uring_cmd_flags
* IORING_URING_CMD_FIXED use registered buffer; pass this flag
* along with setting sqe->buf_index.
+ *
+ * IORING_FUSED_CMD_BUF fused primary command provides buffer
+ * for secondary requests which can retrieve
+ * any sub-buffer with offset(sqe->addr) and
+ * len(sqe->len)
*/
-#define IORING_URING_CMD_FIXED (1U << 0)
+#define IORING_URING_CMD_FIXED (1U << 0)
+#define IORING_FUSED_CMD_BUF (1U << 1)
/*
@@ -25,6 +25,23 @@ static bool io_fused_secondary_valid(u8 op)
return true;
}
+static int io_fused_prep_provide_buf(struct io_uring_cmd *ioucmd,
+ const struct io_uring_sqe *sqe)
+{
+ struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
+ unsigned int sqe_flags = READ_ONCE(sqe->flags);
+
+ /*
+ * Primary command is for providing buffer, non-sense to
+ * set buffer select any more
+ */
+ if (sqe_flags & REQ_F_BUFFER_SELECT)
+ return -EINVAL;
+
+ req->fused_cmd_kbuf = NULL;
+ return 0;
+}
+
int io_fused_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
__must_hold(&req->ctx->uring_lock)
{
@@ -50,8 +67,14 @@ int io_fused_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
return -EINVAL;
ioucmd->flags = READ_ONCE(sqe->uring_cmd_flags);
- if (unlikely(ioucmd->flags))
- return -EINVAL;
+
+ /* so far, only support plugin of providing buffer */
+ if (ioucmd->flags & IORING_FUSED_CMD_BUF)
+ ret = io_fused_prep_provide_buf(ioucmd, sqe);
+ else
+ ret = -EINVAL;
+ if (ret)
+ return ret;
if (unlikely(!io_get_secondary_sqe(ctx, &secondary_sqe)))
return -EINVAL;
@@ -88,8 +111,20 @@ int io_fused_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
int io_fused_cmd(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd);
+ const struct io_kiocb *secondary = ioucmd->fused.data.__secondary;
int ret = -EINVAL;
+ if (ioucmd->flags & IORING_FUSED_CMD_BUF) {
+ /*
+ * Pass buffer direction for driver for validating if the
+ * requested buffer direction is legal
+ */
+ if (io_issue_defs[secondary->opcode].buf_dir)
+ issue_flags |= IO_URING_F_FUSED_BUF_DEST;
+ else
+ issue_flags |= IO_URING_F_FUSED_BUF_SRC;
+ }
+
ret = io_uring_cmd(req, issue_flags);
if (ret != IOU_ISSUE_SKIP_COMPLETE)
io_free_req(ioucmd->fused.data.__secondary);
@@ -117,3 +152,88 @@ void io_fused_cmd_complete_secondary(struct io_kiocb *secondary)
io_uring_cmd_complete_in_task(ioucmd, ioucmd->task_work_cb);
}
+
+/* only for IORING_FUSED_CMD_BUF */
+int io_import_buf_from_fused(unsigned long buf_off, unsigned int len,
+ int dir, struct iov_iter *iter, struct io_kiocb *secondary)
+{
+ struct io_kiocb *req = secondary->fused_primary_req;
+ const struct io_uring_bvec_buf *kbuf;
+ struct io_uring_cmd *primary;
+ unsigned long offset;
+
+ if (unlikely(!(secondary->flags & REQ_F_FUSED_SECONDARY) || !req))
+ return -EINVAL;
+
+ if (unlikely(!req->fused_cmd_kbuf))
+ return -EINVAL;
+
+ primary = io_kiocb_to_cmd(req, struct io_uring_cmd);
+ if (unlikely(!(primary->flags & IORING_FUSED_CMD_BUF)))
+ return -EINVAL;
+
+ /* req->fused_cmd_kbuf is immutable */
+ kbuf = req->fused_cmd_kbuf;
+ offset = kbuf->offset;
+
+ if (!kbuf->bvec)
+ return -EINVAL;
+
+ if (unlikely(buf_off > kbuf->len))
+ return -EFAULT;
+
+ if (unlikely(len > kbuf->len - buf_off))
+ return -EFAULT;
+
+ /* don't use io_import_fixed which doesn't support multipage bvec */
+ offset += buf_off;
+ iov_iter_bvec(iter, dir, kbuf->bvec, kbuf->nr_bvecs, offset + len);
+
+ if (offset)
+ iov_iter_advance(iter, offset);
+
+ return 0;
+}
+
+/*
+ * Called for starting secondary request after primary command prepared io
+ * buffer, only for IORING_FUSED_CMD_BUF
+ *
+ * Secondary request borrows primary's io buffer for handling the secondary
+ * operation, and the buffer is returned back via io_fused_complete_secondary
+ * after the secondary request is completed. Meantime the primary command is
+ * completed. And driver gets completion notification by the passed callback
+ * of @complete_tw_cb.
+ */
+void io_fused_provide_buf_and_start(struct io_uring_cmd *ioucmd,
+ unsigned issue_flags,
+ const struct io_uring_bvec_buf *fused_cmd_kbuf,
+ void (*complete_tw_cb)(struct io_uring_cmd *, unsigned))
+{
+ struct io_kiocb *req = cmd_to_io_kiocb(ioucmd);
+ struct io_kiocb *secondary = ioucmd->fused.data.__secondary;
+ struct io_tw_state ts = {
+ .locked = !(issue_flags & IO_URING_F_UNLOCKED),
+ };
+
+ if (unlikely(!(ioucmd->flags & IORING_FUSED_CMD_BUF)))
+ return;
+
+ if (WARN_ON_ONCE(unlikely(!secondary || !(secondary->flags &
+ REQ_F_FUSED_SECONDARY))))
+ return;
+
+ /*
+ * Once the fused secondary request is completed and the buffer isn't be
+ * used, the driver will be notified by callback of complete_tw_cb
+ */
+ ioucmd->task_work_cb = complete_tw_cb;
+
+ /* now we get the buffer */
+ req->fused_cmd_kbuf = fused_cmd_kbuf;
+ secondary->fused_primary_req = req;
+
+ trace_io_uring_submit_sqe(secondary, true);
+ io_req_task_submit(secondary, &ts);
+}
+EXPORT_SYMBOL_GPL(io_fused_provide_buf_and_start);
@@ -5,5 +5,12 @@
int io_fused_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe);
int io_fused_cmd(struct io_kiocb *req, unsigned int issue_flags);
void io_fused_cmd_complete_secondary(struct io_kiocb *secondary);
+int io_import_buf_from_fused(unsigned long buf_off, unsigned int len,
+ int dir, struct iov_iter *iter, struct io_kiocb *secondary);
+
+static inline bool io_req_use_fused_buf(struct io_kiocb *req)
+{
+ return (req->flags & REQ_F_FUSED_SECONDARY) && req->fused_primary_req;
+}
#endif
@@ -31,6 +31,11 @@ struct io_issue_def {
unsigned manual_alloc : 1;
/* can be secondary op of fused command */
unsigned fused_secondary : 1;
+ /*
+ * buffer direction, 0 : read from buffer, 1: write to buffer, used
+ * for fused_secondary only
+ */
+ unsigned buf_dir : 1;
int (*issue)(struct io_kiocb *, unsigned int);
int (*prep)(struct io_kiocb *, const struct io_uring_sqe *);