X-Git-Url: http://git.cascardo.info/?p=cascardo%2Flinux.git;a=blobdiff_plain;f=fs%2Fsplice.c;h=153d4f3bd441febd7004b1862cd218afc0ee6252;hp=58c322a87f44274b10c3d99f6a0efb0b8248b1fb;hb=c3a690240423fc4eb8a0c3c7df025d13eadf140b;hpb=82c156f853840645604acd7c2cebcb75ed1b6652 diff --git a/fs/splice.c b/fs/splice.c index 58c322a87f44..153d4f3bd441 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -238,8 +238,7 @@ ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) pipe->nrbufs++; return buf->len; } - buf->ops->release(pipe, buf); - buf->ops = NULL; + pipe_buf_release(pipe, buf); return ret; } EXPORT_SYMBOL(add_to_pipe); @@ -316,15 +315,9 @@ ssize_t generic_file_splice_read(struct file *in, loff_t *ppos, *ppos = kiocb.ki_pos; file_accessed(in); } else if (ret < 0) { - if (WARN_ON(to.idx != idx || to.iov_offset)) { - /* - * a bogus ->read_iter() has copied something and still - * returned an error instead of a short read. - */ - to.idx = idx; - to.iov_offset = 0; - iov_iter_advance(&to, 0); /* to free what was emitted */ - } + to.idx = idx; + to.iov_offset = 0; + iov_iter_advance(&to, 0); /* to free what was emitted */ /* * callers of ->splice_read() expect -EAGAIN on * "can't put anything in there", rather than -EFAULT. @@ -361,7 +354,7 @@ const struct pipe_buf_operations nosteal_pipe_buf_ops = { }; EXPORT_SYMBOL(nosteal_pipe_buf_ops); -static ssize_t kernel_readv(struct file *file, const struct iovec *vec, +static ssize_t kernel_readv(struct file *file, const struct kvec *vec, unsigned long vlen, loff_t offset) { mm_segment_t old_fs; @@ -397,96 +390,65 @@ static ssize_t default_file_splice_read(struct file *in, loff_t *ppos, struct pipe_inode_info *pipe, size_t len, unsigned int flags) { + struct kvec *vec, __vec[PIPE_DEF_BUFFERS]; + struct iov_iter to; + struct page **pages; unsigned int nr_pages; - unsigned int nr_freed; - size_t offset; - struct page *pages[PIPE_DEF_BUFFERS]; - struct partial_page partial[PIPE_DEF_BUFFERS]; - struct iovec *vec, __vec[PIPE_DEF_BUFFERS]; + size_t offset, dummy, copied = 0; ssize_t res; - size_t this_len; - int error; int i; - struct splice_pipe_desc spd = { - .pages = pages, - .partial = partial, - .nr_pages_max = PIPE_DEF_BUFFERS, - .flags = flags, - .ops = &default_pipe_buf_ops, - .spd_release = spd_release_page, - }; - if (splice_grow_spd(pipe, &spd)) + if (pipe->nrbufs == pipe->buffers) + return -EAGAIN; + + /* + * Try to keep page boundaries matching to source pagecache ones - + * it probably won't be much help, but... + */ + offset = *ppos & ~PAGE_MASK; + + iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len + offset); + + res = iov_iter_get_pages_alloc(&to, &pages, len + offset, &dummy); + if (res <= 0) return -ENOMEM; - res = -ENOMEM; + nr_pages = res / PAGE_SIZE; + vec = __vec; - if (spd.nr_pages_max > PIPE_DEF_BUFFERS) { - vec = kmalloc(spd.nr_pages_max * sizeof(struct iovec), GFP_KERNEL); - if (!vec) - goto shrink_ret; + if (nr_pages > PIPE_DEF_BUFFERS) { + vec = kmalloc(nr_pages * sizeof(struct kvec), GFP_KERNEL); + if (unlikely(!vec)) { + res = -ENOMEM; + goto out; + } } - offset = *ppos & ~PAGE_MASK; - nr_pages = (len + offset + PAGE_SIZE - 1) >> PAGE_SHIFT; - - for (i = 0; i < nr_pages && i < spd.nr_pages_max && len; i++) { - struct page *page; - - page = alloc_page(GFP_USER); - error = -ENOMEM; - if (!page) - goto err; + pipe->bufs[to.idx].offset = offset; + pipe->bufs[to.idx].len -= offset; - this_len = min_t(size_t, len, PAGE_SIZE - offset); - vec[i].iov_base = (void __user *) page_address(page); + for (i = 0; i < nr_pages; i++) { + size_t this_len = min_t(size_t, len, PAGE_SIZE - offset); + vec[i].iov_base = page_address(pages[i]) + offset; vec[i].iov_len = this_len; - spd.pages[i] = page; - spd.nr_pages++; len -= this_len; offset = 0; } - res = kernel_readv(in, vec, spd.nr_pages, *ppos); - if (res < 0) { - error = res; - goto err; - } - - error = 0; - if (!res) - goto err; - - nr_freed = 0; - for (i = 0; i < spd.nr_pages; i++) { - this_len = min_t(size_t, vec[i].iov_len, res); - spd.partial[i].offset = 0; - spd.partial[i].len = this_len; - if (!this_len) { - __free_page(spd.pages[i]); - spd.pages[i] = NULL; - nr_freed++; - } - res -= this_len; - } - spd.nr_pages -= nr_freed; - - res = splice_to_pipe(pipe, &spd); - if (res > 0) + res = kernel_readv(in, vec, nr_pages, *ppos); + if (res > 0) { + copied = res; *ppos += res; + } -shrink_ret: if (vec != __vec) kfree(vec); - splice_shrink_spd(&spd); +out: + for (i = 0; i < nr_pages; i++) + put_page(pages[i]); + kvfree(pages); + iov_iter_advance(&to, copied); /* truncates and discards */ return res; - -err: - for (i = 0; i < spd.nr_pages; i++) - __free_page(spd.pages[i]); - - res = error; - goto shrink_ret; } /* @@ -547,13 +509,12 @@ static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_des while (pipe->nrbufs) { struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; - const struct pipe_buf_operations *ops = buf->ops; sd->len = buf->len; if (sd->len > sd->total_len) sd->len = sd->total_len; - ret = buf->ops->confirm(pipe, buf); + ret = pipe_buf_confirm(pipe, buf); if (unlikely(ret)) { if (ret == -ENODATA) ret = 0; @@ -573,8 +534,7 @@ static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_des sd->total_len -= ret; if (!buf->len) { - buf->ops = NULL; - ops->release(pipe, buf); + pipe_buf_release(pipe, buf); pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); pipe->nrbufs--; if (pipe->files) @@ -793,7 +753,7 @@ iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, if (idx == pipe->buffers - 1) idx = -1; - ret = buf->ops->confirm(pipe, buf); + ret = pipe_buf_confirm(pipe, buf); if (unlikely(ret)) { if (ret == -ENODATA) ret = 0; @@ -820,11 +780,9 @@ iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, while (ret) { struct pipe_buffer *buf = pipe->bufs + pipe->curbuf; if (ret >= buf->len) { - const struct pipe_buf_operations *ops = buf->ops; ret -= buf->len; buf->len = 0; - buf->ops = NULL; - ops->release(pipe, buf); + pipe_buf_release(pipe, buf); pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1); pipe->nrbufs--; if (pipe->files) @@ -1063,10 +1021,8 @@ out_release: for (i = 0; i < pipe->buffers; i++) { struct pipe_buffer *buf = pipe->bufs + i; - if (buf->ops) { - buf->ops->release(pipe, buf); - buf->ops = NULL; - } + if (buf->ops) + pipe_buf_release(pipe, buf); } if (!bytes) @@ -1627,7 +1583,7 @@ retry: * Get a reference to this pipe buffer, * so we can copy the contents over. */ - ibuf->ops->get(ipipe, ibuf); + pipe_buf_get(ipipe, ibuf); *obuf = *ibuf; /* @@ -1699,7 +1655,7 @@ static int link_pipe(struct pipe_inode_info *ipipe, * Get a reference to this pipe buffer, * so we can copy the contents over. */ - ibuf->ops->get(ipipe, ibuf); + pipe_buf_get(ipipe, ibuf); obuf = opipe->bufs + nbuf; *obuf = *ibuf;