summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
authorAl Viro <viro@zeniv.linux.org.uk>2021-04-26 20:33:42 -0400
committerAl Viro <viro@zeniv.linux.org.uk>2021-06-10 11:45:18 -0400
commit1b4fb5ffd79bac27a7b9beda63c827c7d7457c45 (patch)
treebc2582992acea4f6e648d71eca5f0da2082524bb /lib
parent7491a2bf64e3a4f1699deba97728cd9f8856bdf3 (diff)
iov_iter: teach iterate_{bvec,xarray}() about possible short copies
... and now we finally can sort out the mess in _copy_mc_to_iter(). Provide a variant of iterate_and_advance() that does *NOT* ignore the return values of bvec, xarray and kvec callbacks, use that in _copy_mc_to_iter(). That gets rid of magic in those callbacks - we used to need it so we'd get at least the right return value in case of failure halfway through. As a bonus, now iterator is advanced by the amount actually copied for all flavours. That's what the callers expect and it used to do that correctly in iovec and xarray cases. However, in kvec and bvec cases the iterator had not been advanced on such failures, breaking the users. Fixed now... Signed-off-by: Al Viro <viro@zeniv.linux.org.uk>
Diffstat (limited to 'lib')
-rw-r--r--lib/iov_iter.c65
1 files changed, 24 insertions, 41 deletions
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 58d53deb62b8..ac7682734df2 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -41,22 +41,27 @@
size_t wanted = n; \
while (n) { \
unsigned offset = p->bv_offset + skip; \
+ unsigned left; \
__v.bv_offset = offset % PAGE_SIZE; \
__v.bv_page = p->bv_page + offset / PAGE_SIZE; \
__v.bv_len = min(min(n, p->bv_len - skip), \
(size_t)(PAGE_SIZE - offset % PAGE_SIZE)); \
- (void)(STEP); \
+ left = (STEP); \
+ __v.bv_len -= left; \
skip += __v.bv_len; \
if (skip == p->bv_len) { \
skip = 0; \
p++; \
} \
n -= __v.bv_len; \
+ if (left) \
+ break; \
} \
n = wanted - n; \
}
#define iterate_xarray(i, n, __v, skip, STEP) { \
+ __label__ __out; \
struct page *head = NULL; \
size_t wanted = n, seg, offset; \
loff_t start = i->xarray_start + skip; \
@@ -67,6 +72,7 @@
\
rcu_read_lock(); \
xas_for_each(&xas, head, ULONG_MAX) { \
+ unsigned left; \
if (xas_retry(&xas, head)) \
continue; \
if (WARN_ON(xa_is_value(head))) \
@@ -80,20 +86,20 @@
seg = PAGE_SIZE - offset; \
__v.bv_offset = offset; \
__v.bv_len = min(n, seg); \
- (void)(STEP); \
+ left = (STEP); \
+ __v.bv_len -= left; \
n -= __v.bv_len; \
skip += __v.bv_len; \
- if (n == 0) \
- break; \
+ if (left || n == 0) \
+ goto __out; \
} \
- if (n == 0) \
- break; \
} \
+__out: \
rcu_read_unlock(); \
n = wanted - n; \
}
-#define iterate_and_advance(i, n, v, I, B, K, X) { \
+#define __iterate_and_advance(i, n, v, I, B, K, X) { \
if (unlikely(i->count < n)) \
n = i->count; \
if (likely(n)) { \
@@ -113,8 +119,7 @@
} else if (iov_iter_is_kvec(i)) { \
const struct kvec *kvec = i->kvec; \
struct kvec v; \
- iterate_iovec(i, n, v, kvec, skip, \
- ((void)(K),0)) \
+ iterate_iovec(i, n, v, kvec, skip, (K)) \
i->nr_segs -= kvec - i->kvec; \
i->kvec = kvec; \
} else if (iov_iter_is_xarray(i)) { \
@@ -125,6 +130,9 @@
i->iov_offset = skip; \
} \
}
+#define iterate_and_advance(i, n, v, I, B, K, X) \
+ __iterate_and_advance(i, n, v, I, ((void)(B),0), \
+ ((void)(K),0), ((void)(X),0))
static int copyout(void __user *to, const void *from, size_t n)
{
@@ -698,45 +706,20 @@ static size_t copy_mc_pipe_to_iter(const void *addr, size_t bytes,
size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
{
const char *from = addr;
- unsigned long rem, curr_addr, s_addr = (unsigned long) addr;
if (unlikely(iov_iter_is_pipe(i)))
return copy_mc_pipe_to_iter(addr, bytes, i);
if (iter_is_iovec(i))
might_fault();
- iterate_and_advance(i, bytes, v,
+ __iterate_and_advance(i, bytes, v,
copyout_mc(v.iov_base, (from += v.iov_len) - v.iov_len,
v.iov_len),
- ({
- rem = copy_mc_to_page(v.bv_page, v.bv_offset,
- (from += v.bv_len) - v.bv_len, v.bv_len);
- if (rem) {
- curr_addr = (unsigned long) from;
- bytes = curr_addr - s_addr - rem;
- return bytes;
- }
- }),
- ({
- rem = copy_mc_to_kernel(v.iov_base, (from += v.iov_len)
- - v.iov_len, v.iov_len);
- if (rem) {
- curr_addr = (unsigned long) from;
- bytes = curr_addr - s_addr - rem;
- return bytes;
- }
- }),
- ({
- rem = copy_mc_to_page(v.bv_page, v.bv_offset,
- (from += v.bv_len) - v.bv_len, v.bv_len);
- if (rem) {
- curr_addr = (unsigned long) from;
- bytes = curr_addr - s_addr - rem;
- rcu_read_unlock();
- i->iov_offset += bytes;
- i->count -= bytes;
- return bytes;
- }
- })
+ copy_mc_to_page(v.bv_page, v.bv_offset,
+ (from += v.bv_len) - v.bv_len, v.bv_len),
+ copy_mc_to_kernel(v.iov_base, (from += v.iov_len)
+ - v.iov_len, v.iov_len),
+ copy_mc_to_page(v.bv_page, v.bv_offset,
+ (from += v.bv_len) - v.bv_len, v.bv_len)
)
return bytes;