@@ -86,10 +86,27 @@ struct virtio_fs_req_work {
struct work_struct done_work;
};
-struct virtio_fs_argbuf {
+struct virtio_fs_flat_argbuf {
DECLARE_FLEX_ARRAY(u8, buf);
};
+struct virtio_fs_scattered_argbuf {
+ unsigned int size;
+ unsigned int nr;
+ DECLARE_FLEX_ARRAY(struct bio_vec, bvec);
+};
+
+struct virtio_fs_argbuf {
+ bool is_flat;
+ /* There is flexible array in the end of these two struct
+ * definitions, so they must be the last field.
+ */
+ union {
+ struct virtio_fs_flat_argbuf f;
+ struct virtio_fs_scattered_argbuf s;
+ };
+};
+
static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
struct fuse_req *req, bool in_flight);
@@ -408,42 +425,143 @@ static void virtio_fs_request_dispatch_work(struct work_struct *work)
}
}
+static unsigned int virtio_fs_argbuf_len(unsigned int in_args_len,
+ unsigned int out_args_len,
+ bool is_flat)
+{
+ if (is_flat)
+ return in_args_len + out_args_len;
+
+ /*
+ * Align in_args_len with PAGE_SIZE to reduce the total number of
+ * sg entries when the value of out_args_len (e.g., the length of
+ * read buffer) is page-aligned.
+ */
+ return round_up(in_args_len, PAGE_SIZE) +
+ round_up(out_args_len, PAGE_SIZE);
+}
+
static void virtio_fs_argbuf_free(struct virtio_fs_argbuf *argbuf)
{
+ unsigned int i;
+
+ if (!argbuf)
+ return;
+
+ if (argbuf->is_flat)
+ goto free_argbuf;
+
+ for (i = 0; i < argbuf->s.nr; i++)
+ __free_page(argbuf->s.bvec[i].bv_page);
+
+free_argbuf:
kfree(argbuf);
}
static struct virtio_fs_argbuf *virtio_fs_argbuf_new(struct fuse_args *args,
- gfp_t gfp)
+ gfp_t gfp, bool is_flat)
{
struct virtio_fs_argbuf *argbuf;
unsigned int numargs;
- unsigned int len;
+ unsigned int in_len, out_len, len;
+ unsigned int i, nr;
numargs = args->in_numargs - args->in_pages;
- len = fuse_len_args(numargs, (struct fuse_arg *) args->in_args);
+ in_len = fuse_len_args(numargs, (struct fuse_arg *) args->in_args);
numargs = args->out_numargs - args->out_pages;
- len += fuse_len_args(numargs, args->out_args);
+ out_len = fuse_len_args(numargs, args->out_args);
+ len = virtio_fs_argbuf_len(in_len, out_len, is_flat);
+
+ if (is_flat) {
+ argbuf = kmalloc(struct_size(argbuf, f.buf, len), gfp);
+ if (argbuf)
+ argbuf->is_flat = true;
+
+ return argbuf;
+ }
+
+ nr = len >> PAGE_SHIFT;
+ argbuf = kmalloc(struct_size(argbuf, s.bvec, nr), gfp);
+ if (!argbuf)
+ return NULL;
+
+ argbuf->is_flat = false;
+ argbuf->s.size = len;
+ argbuf->s.nr = 0;
+ for (i = 0; i < nr; i++) {
+ struct page *page;
+
+ page = alloc_page(gfp);
+ if (!page) {
+ virtio_fs_argbuf_free(argbuf);
+ return NULL;
+ }
+ bvec_set_page(&argbuf->s.bvec[i], page, PAGE_SIZE, 0);
+ argbuf->s.nr++;
+ }
+
+ /* Zero the unused space for in_args */
+ if (in_len & ~PAGE_MASK) {
+ struct iov_iter iter;
+ unsigned int to_zero;
+
+ iov_iter_bvec(&iter, ITER_DEST, argbuf->s.bvec, argbuf->s.nr,
+ argbuf->s.size);
+ iov_iter_advance(&iter, in_len);
- argbuf = kmalloc(struct_size(argbuf, buf, len), gfp);
+ to_zero = PAGE_SIZE - (in_len & ~PAGE_MASK);
+ iov_iter_zero(to_zero, &iter);
+ }
return argbuf;
}
static unsigned int virtio_fs_argbuf_setup_sg(struct virtio_fs_argbuf *argbuf,
unsigned int offset,
- unsigned int len,
+ unsigned int *len,
struct scatterlist *sg)
{
- sg_init_one(sg, argbuf->buf + offset, len);
- return 1;
+ struct bvec_iter bi = {
+ .bi_size = offset + *len,
+ };
+ struct scatterlist *cur;
+ struct bio_vec bv;
+
+ if (argbuf->is_flat) {
+ sg_init_one(sg, argbuf->f.buf + offset, *len);
+ return 1;
+ }
+
+ cur = sg;
+ bvec_iter_advance(argbuf->s.bvec, &bi, offset);
+ for_each_bvec(bv, argbuf->s.bvec, bi, bi) {
+ sg_init_table(cur, 1);
+ sg_set_page(cur, bv.bv_page, bv.bv_len, bv.bv_offset);
+ cur++;
+ }
+ *len = round_up(*len, PAGE_SIZE);
+
+ return cur - sg;
}
static void virtio_fs_argbuf_copy_from_in_arg(struct virtio_fs_argbuf *argbuf,
unsigned int offset,
const void *src, unsigned int len)
{
- memcpy(argbuf->buf + offset, src, len);
+ struct iov_iter iter;
+ unsigned int copied;
+
+ if (argbuf->is_flat) {
+ memcpy(argbuf->f.buf + offset, src, len);
+ return;
+ }
+
+ iov_iter_bvec(&iter, ITER_DEST, argbuf->s.bvec,
+ argbuf->s.nr, argbuf->s.size);
+ iov_iter_advance(&iter, offset);
+
+ copied = _copy_to_iter(src, len, &iter);
+ WARN_ON_ONCE(copied != len);
}
static unsigned int
@@ -451,15 +569,32 @@ virtio_fs_argbuf_out_args_offset(struct virtio_fs_argbuf *argbuf,
const struct fuse_args *args)
{
unsigned int num_in = args->in_numargs - args->in_pages;
+ unsigned int offset = fuse_len_args(num_in,
+ (struct fuse_arg *)args->in_args);
- return fuse_len_args(num_in, (struct fuse_arg *)args->in_args);
+ if (argbuf->is_flat)
+ return offset;
+ return round_up(offset, PAGE_SIZE);
}
static void virtio_fs_argbuf_copy_to_out_arg(struct virtio_fs_argbuf *argbuf,
unsigned int offset, void *dst,
unsigned int len)
{
- memcpy(dst, argbuf->buf + offset, len);
+ struct iov_iter iter;
+ unsigned int copied;
+
+ if (argbuf->is_flat) {
+ memcpy(dst, argbuf->f.buf + offset, len);
+ return;
+ }
+
+ iov_iter_bvec(&iter, ITER_SOURCE, argbuf->s.bvec,
+ argbuf->s.nr, argbuf->s.size);
+ iov_iter_advance(&iter, offset);
+
+ copied = _copy_from_iter(dst, len, &iter);
+ WARN_ON_ONCE(copied != len);
}
/*
@@ -1154,7 +1289,7 @@ static unsigned int sg_init_fuse_args(struct scatterlist *sg,
len = fuse_len_args(numargs - argpages, args);
if (len)
total_sgs += virtio_fs_argbuf_setup_sg(req->argbuf, *len_used,
- len, &sg[total_sgs]);
+ &len, &sg[total_sgs]);
if (argpages)
total_sgs += sg_init_fuse_pages(&sg[total_sgs],
@@ -1199,7 +1334,7 @@ static int virtio_fs_enqueue_req(struct virtio_fs_vq *fsvq,
}
/* Use a bounce buffer since stack args cannot be mapped */
- req->argbuf = virtio_fs_argbuf_new(args, GFP_ATOMIC);
+ req->argbuf = virtio_fs_argbuf_new(args, GFP_ATOMIC, true);
if (!req->argbuf) {
ret = -ENOMEM;
goto out;