summaryrefslogtreecommitdiff
path: root/mm/mremap.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/mremap.c')
-rw-r--r--mm/mremap.c104
1 files changed, 65 insertions, 39 deletions
diff --git a/mm/mremap.c b/mm/mremap.c
index dee98ff2bbd6..60473413836b 100644
--- a/mm/mremap.c
+++ b/mm/mremap.c
@@ -140,6 +140,7 @@ static int move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
{
struct mm_struct *mm = vma->vm_mm;
pte_t *old_pte, *new_pte, pte;
+ pmd_t dummy_pmdval;
spinlock_t *old_ptl, *new_ptl;
bool force_flush = false;
unsigned long len = old_end - old_addr;
@@ -175,7 +176,15 @@ static int move_ptes(struct vm_area_struct *vma, pmd_t *old_pmd,
err = -EAGAIN;
goto out;
}
- new_pte = pte_offset_map_nolock(mm, new_pmd, new_addr, &new_ptl);
+ /*
+ * Now new_pte is none, so hpage_collapse_scan_file() path can not find
+ * this by traversing file->f_mapping, so there is no concurrency with
+ * retract_page_tables(). In addition, we already hold the exclusive
+ * mmap_lock, so this new_pte page is stable, so there is no need to get
+ * pmdval and do pmd_same() check.
+ */
+ new_pte = pte_offset_map_rw_nolock(mm, new_pmd, new_addr, &dummy_pmdval,
+ &new_ptl);
if (!new_pte) {
pte_unmap_unlock(old_pte, old_ptl);
err = -EAGAIN;
@@ -817,17 +826,24 @@ static unsigned long move_vma(struct vm_area_struct *vma,
return new_addr;
}
-static struct vm_area_struct *vma_to_resize(unsigned long addr,
+/*
+ * resize_is_valid() - Ensure the vma can be resized to the new length at the give
+ * address.
+ *
+ * @vma: The vma to resize
+ * @addr: The old address
+ * @old_len: The current size
+ * @new_len: The desired size
+ * @flags: The vma flags
+ *
+ * Return 0 on success, error otherwise.
+ */
+static int resize_is_valid(struct vm_area_struct *vma, unsigned long addr,
unsigned long old_len, unsigned long new_len, unsigned long flags)
{
struct mm_struct *mm = current->mm;
- struct vm_area_struct *vma;
unsigned long pgoff;
- vma = vma_lookup(mm, addr);
- if (!vma)
- return ERR_PTR(-EFAULT);
-
/*
* !old_len is a special case where an attempt is made to 'duplicate'
* a mapping. This makes no sense for private mappings as it will
@@ -838,39 +854,53 @@ static struct vm_area_struct *vma_to_resize(unsigned long addr,
*/
if (!old_len && !(vma->vm_flags & (VM_SHARED | VM_MAYSHARE))) {
pr_warn_once("%s (%d): attempted to duplicate a private mapping with mremap. This is not supported.\n", current->comm, current->pid);
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
}
if ((flags & MREMAP_DONTUNMAP) &&
(vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP)))
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
/* We can't remap across vm area boundaries */
if (old_len > vma->vm_end - addr)
- return ERR_PTR(-EFAULT);
+ return -EFAULT;
if (new_len == old_len)
- return vma;
+ return 0;
/* Need to be careful about a growing mapping */
pgoff = (addr - vma->vm_start) >> PAGE_SHIFT;
pgoff += vma->vm_pgoff;
if (pgoff + (new_len >> PAGE_SHIFT) < pgoff)
- return ERR_PTR(-EINVAL);
+ return -EINVAL;
if (vma->vm_flags & (VM_DONTEXPAND | VM_PFNMAP))
- return ERR_PTR(-EFAULT);
+ return -EFAULT;
if (!mlock_future_ok(mm, vma->vm_flags, new_len - old_len))
- return ERR_PTR(-EAGAIN);
+ return -EAGAIN;
if (!may_expand_vm(mm, vma->vm_flags,
(new_len - old_len) >> PAGE_SHIFT))
- return ERR_PTR(-ENOMEM);
+ return -ENOMEM;
- return vma;
+ return 0;
}
+/*
+ * mremap_to() - remap a vma to a new location
+ * @addr: The old address
+ * @old_len: The old size
+ * @new_addr: The target address
+ * @new_len: The new size
+ * @locked: If the returned vma is locked (VM_LOCKED)
+ * @flags: the mremap flags
+ * @uf: The mremap userfaultfd context
+ * @uf_unmap_early: The userfaultfd unmap early context
+ * @uf_unmap: The userfaultfd unmap context
+ *
+ * Returns: The new address of the vma or an error.
+ */
static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
unsigned long new_addr, unsigned long new_len, bool *locked,
unsigned long flags, struct vm_userfaultfd_ctx *uf,
@@ -879,18 +909,18 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
{
struct mm_struct *mm = current->mm;
struct vm_area_struct *vma;
- unsigned long ret = -EINVAL;
+ unsigned long ret;
unsigned long map_flags = 0;
if (offset_in_page(new_addr))
- goto out;
+ return -EINVAL;
if (new_len > TASK_SIZE || new_addr > TASK_SIZE - new_len)
- goto out;
+ return -EINVAL;
/* Ensure the old/new locations do not overlap */
if (addr + old_len > new_addr && new_addr + new_len > addr)
- goto out;
+ return -EINVAL;
/*
* move_vma() need us to stay 4 maps below the threshold, otherwise
@@ -917,27 +947,28 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
*/
ret = do_munmap(mm, new_addr, new_len, uf_unmap_early);
if (ret)
- goto out;
+ return ret;
}
if (old_len > new_len) {
ret = do_munmap(mm, addr+new_len, old_len - new_len, uf_unmap);
if (ret)
- goto out;
+ return ret;
old_len = new_len;
}
- vma = vma_to_resize(addr, old_len, new_len, flags);
- if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
- goto out;
- }
+ vma = vma_lookup(mm, addr);
+ if (!vma)
+ return -EFAULT;
+
+ ret = resize_is_valid(vma, addr, old_len, new_len, flags);
+ if (ret)
+ return ret;
/* MREMAP_DONTUNMAP expands by old_len since old_len == new_len */
if (flags & MREMAP_DONTUNMAP &&
!may_expand_vm(mm, vma->vm_flags, old_len >> PAGE_SHIFT)) {
- ret = -ENOMEM;
- goto out;
+ return -ENOMEM;
}
if (flags & MREMAP_FIXED)
@@ -950,17 +981,14 @@ static unsigned long mremap_to(unsigned long addr, unsigned long old_len,
((addr - vma->vm_start) >> PAGE_SHIFT),
map_flags);
if (IS_ERR_VALUE(ret))
- goto out;
+ return ret;
/* We got a new mapping */
if (!(flags & MREMAP_FIXED))
new_addr = ret;
- ret = move_vma(vma, addr, old_len, new_len, new_addr, locked, flags, uf,
- uf_unmap);
-
-out:
- return ret;
+ return move_vma(vma, addr, old_len, new_len, new_addr, locked, flags,
+ uf, uf_unmap);
}
static int vma_expandable(struct vm_area_struct *vma, unsigned long delta)
@@ -1105,11 +1133,9 @@ SYSCALL_DEFINE5(mremap, unsigned long, addr, unsigned long, old_len,
/*
* Ok, we need to grow..
*/
- vma = vma_to_resize(addr, old_len, new_len, flags);
- if (IS_ERR(vma)) {
- ret = PTR_ERR(vma);
+ ret = resize_is_valid(vma, addr, old_len, new_len, flags);
+ if (ret)
goto out;
- }
/* old_len exactly to the end of the area..
*/
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-02 05:04:10 +0000