pipe->nrbufs++;
return buf->len;
}
- buf->ops->release(pipe, buf);
- buf->ops = NULL;
+ pipe_buf_release(pipe, buf);
return ret;
}
EXPORT_SYMBOL(add_to_pipe);
*ppos = kiocb.ki_pos;
file_accessed(in);
} else if (ret < 0) {
- if (WARN_ON(to.idx != idx || to.iov_offset)) {
- /*
- * a bogus ->read_iter() has copied something and still
- * returned an error instead of a short read.
- */
- to.idx = idx;
- to.iov_offset = 0;
- iov_iter_advance(&to, 0); /* to free what was emitted */
- }
+ to.idx = idx;
+ to.iov_offset = 0;
+ iov_iter_advance(&to, 0); /* to free what was emitted */
/*
* callers of ->splice_read() expect -EAGAIN on
* "can't put anything in there", rather than -EFAULT.
};
EXPORT_SYMBOL(nosteal_pipe_buf_ops);
-static ssize_t kernel_readv(struct file *file, const struct iovec *vec,
+static ssize_t kernel_readv(struct file *file, const struct kvec *vec,
unsigned long vlen, loff_t offset)
{
mm_segment_t old_fs;
struct pipe_inode_info *pipe, size_t len,
unsigned int flags)
{
+ struct kvec *vec, __vec[PIPE_DEF_BUFFERS];
+ struct iov_iter to;
+ struct page **pages;
unsigned int nr_pages;
- unsigned int nr_freed;
- size_t offset;
- struct page *pages[PIPE_DEF_BUFFERS];
- struct partial_page partial[PIPE_DEF_BUFFERS];
- struct iovec *vec, __vec[PIPE_DEF_BUFFERS];
+ size_t offset, dummy, copied = 0;
ssize_t res;
- size_t this_len;
- int error;
int i;
- struct splice_pipe_desc spd = {
- .pages = pages,
- .partial = partial,
- .nr_pages_max = PIPE_DEF_BUFFERS,
- .flags = flags,
- .ops = &default_pipe_buf_ops,
- .spd_release = spd_release_page,
- };
- if (splice_grow_spd(pipe, &spd))
+ if (pipe->nrbufs == pipe->buffers)
+ return -EAGAIN;
+
+ /*
+ * Try to keep page boundaries matching to source pagecache ones -
+ * it probably won't be much help, but...
+ */
+ offset = *ppos & ~PAGE_MASK;
+
+ iov_iter_pipe(&to, ITER_PIPE | READ, pipe, len + offset);
+
+ res = iov_iter_get_pages_alloc(&to, &pages, len + offset, &dummy);
+ if (res <= 0)
return -ENOMEM;
- res = -ENOMEM;
+ nr_pages = res / PAGE_SIZE;
+
vec = __vec;
- if (spd.nr_pages_max > PIPE_DEF_BUFFERS) {
- vec = kmalloc(spd.nr_pages_max * sizeof(struct iovec), GFP_KERNEL);
- if (!vec)
- goto shrink_ret;
+ if (nr_pages > PIPE_DEF_BUFFERS) {
+ vec = kmalloc(nr_pages * sizeof(struct kvec), GFP_KERNEL);
+ if (unlikely(!vec)) {
+ res = -ENOMEM;
+ goto out;
+ }
}
- offset = *ppos & ~PAGE_MASK;
- nr_pages = (len + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
-
- for (i = 0; i < nr_pages && i < spd.nr_pages_max && len; i++) {
- struct page *page;
-
- page = alloc_page(GFP_USER);
- error = -ENOMEM;
- if (!page)
- goto err;
+ pipe->bufs[to.idx].offset = offset;
+ pipe->bufs[to.idx].len -= offset;
- this_len = min_t(size_t, len, PAGE_SIZE - offset);
- vec[i].iov_base = (void __user *) page_address(page);
+ for (i = 0; i < nr_pages; i++) {
+ size_t this_len = min_t(size_t, len, PAGE_SIZE - offset);
+ vec[i].iov_base = page_address(pages[i]) + offset;
vec[i].iov_len = this_len;
- spd.pages[i] = page;
- spd.nr_pages++;
len -= this_len;
offset = 0;
}
- res = kernel_readv(in, vec, spd.nr_pages, *ppos);
- if (res < 0) {
- error = res;
- goto err;
- }
-
- error = 0;
- if (!res)
- goto err;
-
- nr_freed = 0;
- for (i = 0; i < spd.nr_pages; i++) {
- this_len = min_t(size_t, vec[i].iov_len, res);
- spd.partial[i].offset = 0;
- spd.partial[i].len = this_len;
- if (!this_len) {
- __free_page(spd.pages[i]);
- spd.pages[i] = NULL;
- nr_freed++;
- }
- res -= this_len;
- }
- spd.nr_pages -= nr_freed;
-
- res = splice_to_pipe(pipe, &spd);
- if (res > 0)
+ res = kernel_readv(in, vec, nr_pages, *ppos);
+ if (res > 0) {
+ copied = res;
*ppos += res;
+ }
-shrink_ret:
if (vec != __vec)
kfree(vec);
- splice_shrink_spd(&spd);
+out:
+ for (i = 0; i < nr_pages; i++)
+ put_page(pages[i]);
+ kvfree(pages);
+ iov_iter_advance(&to, copied); /* truncates and discards */
return res;
-
-err:
- for (i = 0; i < spd.nr_pages; i++)
- __free_page(spd.pages[i]);
-
- res = error;
- goto shrink_ret;
}
/*
while (pipe->nrbufs) {
struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
- const struct pipe_buf_operations *ops = buf->ops;
sd->len = buf->len;
if (sd->len > sd->total_len)
sd->len = sd->total_len;
- ret = buf->ops->confirm(pipe, buf);
+ ret = pipe_buf_confirm(pipe, buf);
if (unlikely(ret)) {
if (ret == -ENODATA)
ret = 0;
sd->total_len -= ret;
if (!buf->len) {
- buf->ops = NULL;
- ops->release(pipe, buf);
+ pipe_buf_release(pipe, buf);
pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
pipe->nrbufs--;
if (pipe->files)
if (idx == pipe->buffers - 1)
idx = -1;
- ret = buf->ops->confirm(pipe, buf);
+ ret = pipe_buf_confirm(pipe, buf);
if (unlikely(ret)) {
if (ret == -ENODATA)
ret = 0;
while (ret) {
struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
if (ret >= buf->len) {
- const struct pipe_buf_operations *ops = buf->ops;
ret -= buf->len;
buf->len = 0;
- buf->ops = NULL;
- ops->release(pipe, buf);
+ pipe_buf_release(pipe, buf);
pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
pipe->nrbufs--;
if (pipe->files)
for (i = 0; i < pipe->buffers; i++) {
struct pipe_buffer *buf = pipe->bufs + i;
- if (buf->ops) {
- buf->ops->release(pipe, buf);
- buf->ops = NULL;
- }
+ if (buf->ops)
+ pipe_buf_release(pipe, buf);
}
if (!bytes)
* Get a reference to this pipe buffer,
* so we can copy the contents over.
*/
- ibuf->ops->get(ipipe, ibuf);
+ pipe_buf_get(ipipe, ibuf);
*obuf = *ibuf;
/*
* Get a reference to this pipe buffer,
* so we can copy the contents over.
*/
- ibuf->ops->get(ipipe, ibuf);
+ pipe_buf_get(ipipe, ibuf);
obuf = opipe->bufs + nbuf;
*obuf = *ibuf;
projects / cascardo / linux.git / blobdiff