@@ -707,7 +707,7 @@ static int shift_arg_pages(struct vm_area_struct *vma, unsigned long shift)
* process cleanup to remove whatever mess we made.
*/
if (length != move_page_tables(vma, old_start,
- vma, new_start, length, false))
+ vma, new_start, length, false, true))
return -ENOMEM;
lru_add_drain();
@@ -2265,7 +2265,7 @@ int get_cmdline(struct task_struct *task, char *buffer, int buflen);
extern unsigned long move_page_tables(struct vm_area_struct *vma,
unsigned long old_addr, struct vm_area_struct *new_vma,
unsigned long new_addr, unsigned long len,
- bool need_rmap_locks);
+ bool need_rmap_locks, bool for_stack);
/*
* Flags used by change_protection(). For now we make it a bitmap so
@@ -479,22 +479,23 @@ static bool move_pgt_entry(enum pgt_entry entry, struct vm_area_struct *vma,
}
/*
- * A helper to check if a previous mapping exists. Required for
- * move_page_tables() and realign_addr() to determine if a previous mapping
- * exists before we can do realignment optimizations.
+ * A helper to check if aligning down is OK. The newly aligned address should
+ * not fall on any existing mapping otherwise we don't align. For the stack
+ * moving down, that's a special move within the VMA that is created to span
+ * the source and destination of the move, so we make an exception for it.
*/
static bool can_align_down(struct vm_area_struct *vma, unsigned long addr_to_align,
- unsigned long mask)
+ unsigned long mask, bool for_stack)
{
unsigned long addr_masked = addr_to_align & mask;
struct vm_area_struct *prev = NULL, *cur = NULL;
/*
- * If @addr_to_align of either source or destination is not the beginning
- * of the corresponding VMA, we can't align down or we will destroy part
- * of the current mapping.
+ * Other than for stack moves, if @addr_to_align of either source or
+ * destination is not the beginning of the corresponding VMA, we can't
+ * align down or we will destroy part of the current mapping.
*/
- if (vma->vm_start != addr_to_align)
+ if (!for_stack && vma->vm_start != addr_to_align)
return false;
/*
@@ -511,13 +512,13 @@ static bool can_align_down(struct vm_area_struct *vma, unsigned long addr_to_ali
/* Opportunistically realign to specified boundary for faster copy. */
static void realign_addr(unsigned long *old_addr, struct vm_area_struct *old_vma,
unsigned long *new_addr, struct vm_area_struct *new_vma,
- unsigned long mask)
+ unsigned long mask, bool for_stack)
{
bool mutually_aligned = (*old_addr & ~mask) == (*new_addr & ~mask);
if ((*old_addr & ~mask) && mutually_aligned
- && can_align_down(old_vma, *old_addr, mask)
- && can_align_down(new_vma, *new_addr, mask)) {
+ && can_align_down(old_vma, *old_addr, mask, for_stack)
+ && can_align_down(new_vma, *new_addr, mask, for_stack)) {
*old_addr = *old_addr & mask;
*new_addr = *new_addr & mask;
}
@@ -526,7 +527,7 @@ static void realign_addr(unsigned long *old_addr, struct vm_area_struct *old_vma
unsigned long move_page_tables(struct vm_area_struct *vma,
unsigned long old_addr, struct vm_area_struct *new_vma,
unsigned long new_addr, unsigned long len,
- bool need_rmap_locks)
+ bool need_rmap_locks, bool for_stack)
{
unsigned long extent, old_end;
struct mmu_notifier_range range;
@@ -538,14 +539,9 @@ unsigned long move_page_tables(struct vm_area_struct *vma,
old_end = old_addr + len;
- /*
- * If possible, realign addresses to PMD boundary for faster copy.
- * Don't align for intra-VMA moves as we may destroy existing mappings.
- */
- if ((vma != new_vma)
- && (len >= PMD_SIZE - (old_addr & ~PMD_MASK))) {
- realign_addr(&old_addr, vma, &new_addr, new_vma, PMD_MASK);
- }
+ /* If possible, realign addresses to PMD boundary for faster copy. */
+ if (len >= PMD_SIZE - (old_addr & ~PMD_MASK))
+ realign_addr(&old_addr, vma, &new_addr, new_vma, PMD_MASK, for_stack);
if (is_vm_hugetlb_page(vma))
return move_hugetlb_page_tables(vma, new_vma, old_addr,
@@ -694,7 +690,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
}
moved_len = move_page_tables(vma, old_addr, new_vma, new_addr, old_len,
- need_rmap_locks);
+ need_rmap_locks, false);
if (moved_len < old_len) {
err = -ENOMEM;
} else if (vma->vm_ops && vma->vm_ops->mremap) {
@@ -708,7 +704,7 @@ static unsigned long move_vma(struct vm_area_struct *vma,
* and then proceed to unmap new area instead of old.
*/
move_page_tables(new_vma, new_addr, vma, old_addr, moved_len,
- true);
+ true, false);
vma = new_vma;
old_len = new_len;
old_addr = new_addr;