diff options
Diffstat (limited to 'mm/iov_iter.c')
| -rw-r--r-- | mm/iov_iter.c | 240 |
1 files changed, 226 insertions, 14 deletions
diff --git a/mm/iov_iter.c b/mm/iov_iter.c index 9a09f2034fcc..eafcf60f6b83 100644 --- a/mm/iov_iter.c +++ b/mm/iov_iter.c @@ -4,6 +4,96 @@ #include <linux/slab.h> #include <linux/vmalloc.h> +static size_t copy_to_iter_iovec(void *from, size_t bytes, struct iov_iter *i) +{ + size_t skip, copy, left, wanted; + const struct iovec *iov; + char __user *buf; + + if (unlikely(bytes > i->count)) + bytes = i->count; + + if (unlikely(!bytes)) + return 0; + + wanted = bytes; + iov = i->iov; + skip = i->iov_offset; + buf = iov->iov_base + skip; + copy = min(bytes, iov->iov_len - skip); + + left = __copy_to_user(buf, from, copy); + copy -= left; + skip += copy; + from += copy; + bytes -= copy; + while (unlikely(!left && bytes)) { + iov++; + buf = iov->iov_base; + copy = min(bytes, iov->iov_len); + left = __copy_to_user(buf, from, copy); + copy -= left; + skip = copy; + from += copy; + bytes -= copy; + } + + if (skip == iov->iov_len) { + iov++; + skip = 0; + } + i->count -= wanted - bytes; + i->nr_segs -= iov - i->iov; + i->iov = iov; + i->iov_offset = skip; + return wanted - bytes; +} + +static size_t copy_from_iter_iovec(void *to, size_t bytes, struct iov_iter *i) +{ + size_t skip, copy, left, wanted; + const struct iovec *iov; + char __user *buf; + + if (unlikely(bytes > i->count)) + bytes = i->count; + + if (unlikely(!bytes)) + return 0; + + wanted = bytes; + iov = i->iov; + skip = i->iov_offset; + buf = iov->iov_base + skip; + copy = min(bytes, iov->iov_len - skip); + + left = __copy_from_user(to, buf, copy); + copy -= left; + skip += copy; + to += copy; + bytes -= copy; + while (unlikely(!left && bytes)) { + iov++; + buf = iov->iov_base; + copy = min(bytes, iov->iov_len); + left = __copy_from_user(to, buf, copy); + copy -= left; + skip = copy; + to += copy; + bytes -= copy; + } + + if (skip == iov->iov_len) { + iov++; + skip = 0; + } + i->count -= wanted - bytes; + i->nr_segs -= iov - i->iov; + i->iov = iov; + i->iov_offset = skip; + return wanted - bytes; +} + static size_t copy_page_to_iter_iovec(struct page *page, size_t offset, size_t bytes, struct iov_iter *i) { @@ -166,6 +256,50 @@ done: return wanted - bytes; } +static size_t zero_iovec(size_t bytes, struct iov_iter *i) +{ + size_t skip, copy, left, wanted; + const struct iovec *iov; + char __user *buf; + + if (unlikely(bytes > i->count)) + bytes = i->count; + + if (unlikely(!bytes)) + return 0; + + wanted = bytes; + iov = i->iov; + skip = i->iov_offset; + buf = iov->iov_base + skip; + copy = min(bytes, iov->iov_len - skip); + + left = __clear_user(buf, copy); + copy -= left; + skip += copy; + bytes -= copy; + + while (unlikely(!left && bytes)) { + iov++; + buf = iov->iov_base; + copy = min(bytes, iov->iov_len); + left = __clear_user(buf, copy); + copy -= left; + skip = copy; + bytes -= copy; + } + + if (skip == iov->iov_len) { + iov++; + skip = 0; + } + i->count -= wanted - bytes; + i->nr_segs -= iov - i->iov; + i->iov = iov; + i->iov_offset = skip; + return wanted - bytes; +} + static size_t __iovec_copy_from_user_inatomic(char *vaddr, const struct iovec *iov, size_t base, size_t bytes) { @@ -414,12 +548,17 @@ static void memcpy_to_page(struct page *page, size_t offset, char *from, size_t kunmap_atomic(to); } -static size_t copy_page_to_iter_bvec(struct page *page, size_t offset, size_t bytes, - struct iov_iter *i) +static void memzero_page(struct page *page, size_t offset, size_t len) +{ + char *addr = kmap_atomic(page); + memset(addr + offset, 0, len); + kunmap_atomic(addr); +} + +static size_t copy_to_iter_bvec(void *from, size_t bytes, struct iov_iter *i) { size_t skip, copy, wanted; const struct bio_vec *bvec; - void *kaddr, *from; if (unlikely(bytes > i->count)) bytes = i->count; @@ -432,8 +571,6 @@ static size_t copy_page_to_iter_bvec(struct page *page, size_t offset, size_t by skip = i->iov_offset; copy = min_t(size_t, bytes, bvec->bv_len - skip); - kaddr = kmap_atomic(page); - from = kaddr + offset; memcpy_to_page(bvec->bv_page, skip + bvec->bv_offset, from, copy); skip += copy; from += copy; @@ -446,7 +583,6 @@ static size_t copy_page_to_iter_bvec(struct page *page, size_t offset, size_t by from += copy; bytes -= copy; } - kunmap_atomic(kaddr); if (skip == bvec->bv_len) { bvec++; skip = 0; @@ -458,12 +594,10 @@ static size_t copy_page_to_iter_bvec(struct page *page, size_t offset, size_t by return wanted - bytes; } -static size_t copy_page_from_iter_bvec(struct page *page, size_t offset, size_t bytes, - struct iov_iter *i) +static size_t copy_from_iter_bvec(void *to, size_t bytes, struct iov_iter *i) { size_t skip, copy, wanted; const struct bio_vec *bvec; - void *kaddr, *to; if (unlikely(bytes > i->count)) bytes = i->count; @@ -475,10 +609,6 @@ static size_t copy_page_from_iter_bvec(struct page *page, size_t offset, size_t bvec = i->bvec; skip = i->iov_offset; - kaddr = kmap_atomic(page); - - to = kaddr + offset; - copy = min(bytes, bvec->bv_len - skip); memcpy_from_page(to, bvec->bv_page, bvec->bv_offset + skip, copy); @@ -495,7 +625,6 @@ static size_t copy_page_from_iter_bvec(struct page *page, size_t offset, size_t to += copy; bytes -= copy; } - kunmap_atomic(kaddr); if (skip == bvec->bv_len) { bvec++; skip = 0; @@ -507,6 +636,61 @@ static size_t copy_page_from_iter_bvec(struct page *page, size_t offset, size_t return wanted; } +static size_t copy_page_to_iter_bvec(struct page *page, size_t offset, + size_t bytes, struct iov_iter *i) +{ + void *kaddr = kmap_atomic(page); + size_t wanted = copy_to_iter_bvec(kaddr + offset, bytes, i); + kunmap_atomic(kaddr); + return wanted; +} + +static size_t copy_page_from_iter_bvec(struct page *page, size_t offset, + size_t bytes, struct iov_iter *i) +{ + void *kaddr = kmap_atomic(page); + size_t wanted = copy_from_iter_bvec(kaddr + offset, bytes, i); + kunmap_atomic(kaddr); + return wanted; +} + +static size_t zero_bvec(size_t bytes, struct iov_iter *i) +{ + size_t skip, copy, wanted; + const struct bio_vec *bvec; + + if (unlikely(bytes > i->count)) + bytes = i->count; + + if (unlikely(!bytes)) + return 0; + + wanted = bytes; + bvec = i->bvec; + skip = i->iov_offset; + copy = min_t(size_t, bytes, bvec->bv_len - skip); + + memzero_page(bvec->bv_page, skip + bvec->bv_offset, copy); + skip += copy; + bytes -= copy; + while (bytes) { + bvec++; + copy = min(bytes, (size_t)bvec->bv_len); + memzero_page(bvec->bv_page, bvec->bv_offset, copy); + skip = copy; + bytes -= copy; + } + if (skip == bvec->bv_len) { + bvec++; + skip = 0; + } + i->count -= wanted - bytes; + i->nr_segs -= bvec - i->bvec; + i->bvec = bvec; + i->iov_offset = skip; + return wanted - bytes; +} + static size_t copy_from_user_bvec(struct page *page, struct iov_iter *i, unsigned long offset, size_t bytes) { @@ -672,6 +856,34 @@ size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, } EXPORT_SYMBOL(copy_page_from_iter); +size_t copy_to_iter(void *addr, size_t bytes, struct iov_iter *i) +{ + if (i->type & ITER_BVEC) + return copy_to_iter_bvec(addr, bytes, i); + else + return copy_to_iter_iovec(addr, bytes, i); +} +EXPORT_SYMBOL(copy_to_iter); + +size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i) +{ + if (i->type & ITER_BVEC) + return copy_from_iter_bvec(addr, bytes, i); + else + return copy_from_iter_iovec(addr, bytes, i); +} +EXPORT_SYMBOL(copy_from_iter); + +size_t iov_iter_zero(size_t bytes, struct iov_iter *i) +{ + if (i->type & ITER_BVEC) { + return zero_bvec(bytes, i); + } else { + return zero_iovec(bytes, i); + } +} +EXPORT_SYMBOL(iov_iter_zero); + size_t iov_iter_copy_from_user_atomic(struct page *page, struct iov_iter *i, unsigned long offset, size_t bytes) { |