LoongArch: Share the same pmd between vDSO space and stack
Commit Message
Currently vDSO virtual address space is randomed within range 64M
below TASK_SIZE, and the next is stack space randomed within generic
range 8M. so vDSO space uses different pmd entry with stack.
With idea from x86, vDSO space can share the same pmd entry with
stack, only that randomed range is smaller than before. This will
save one PTE page for every thread.
Also this patch removes ____cacheline_aligned_in_smp property for
vdso_pcpu_data structure, cache line aligned property for small size
is generally used for frequently modified data like lock to avoid
cache-thrashing on smp system. The struct element node is almost constant
in struct vdso_pcpu_data, so cache line aligned property is not
necessary, and it will use more memory.
Signed-off-by: Bibo Mao <maobibo@loongson.cn>
---
arch/loongarch/include/asm/processor.h | 2 --
arch/loongarch/include/asm/vdso/vdso.h | 2 +-
arch/loongarch/kernel/process.c | 4 ---
arch/loongarch/kernel/vdso.c | 44 +++++++++++++++++++++-----
4 files changed, 37 insertions(+), 15 deletions(-)
@@ -43,8 +43,6 @@
#endif
-#define VDSO_RANDOMIZE_SIZE (TASK_IS_32BIT_ADDR ? SZ_1M : SZ_64M)
-
unsigned long stack_top(void);
#define STACK_TOP stack_top()
@@ -12,7 +12,7 @@
struct vdso_pcpu_data {
u32 node;
-} ____cacheline_aligned_in_smp;
+};
struct loongarch_vdso_data {
struct vdso_pcpu_data pdata[NR_CPUS];
@@ -296,10 +296,6 @@ unsigned long stack_top(void)
top -= PAGE_ALIGN(current->thread.vdso->size);
top -= VVAR_SIZE;
- /* Space to randomize the VDSO base */
- if (current->flags & PF_RANDOMIZE)
- top -= VDSO_RANDOMIZE_SIZE;
-
return top;
}
@@ -148,16 +148,44 @@ int vdso_join_timens(struct task_struct *task, struct time_namespace *ns)
}
#endif
-static unsigned long vdso_base(void)
+/*
+ * Put the vdso above the (randomized) stack with another randomized
+ * offset. This way there is no hole in the middle of address space.
+ * To save memory make sure it is still in the same PTE as the stack
+ * top. This doesn't give that many random bits.
+ *
+ * Note that this algorithm is imperfect: the distribution of the vdso
+ * start address within a PMD is biased toward the end.
+ */
+static unsigned long vdso_addr(unsigned long start, unsigned int len)
{
- unsigned long base = STACK_TOP;
+ unsigned long addr, end;
+ unsigned long offset;
- if (current->flags & PF_RANDOMIZE) {
- base += get_random_u32_below(VDSO_RANDOMIZE_SIZE);
- base = PAGE_ALIGN(base);
- }
+ /*
+ * Round up the start address. It can start out unaligned as a result
+ * of stack start randomization.
+ */
+ start = PAGE_ALIGN(start);
- return base;
+ /* Round the lowest possible end address up to a PMD boundary. */
+ end = (start + len + PMD_SIZE - 1) & PMD_MASK;
+ if (end >= TASK_SIZE)
+ end = TASK_SIZE;
+ end -= len;
+
+ if (end > start) {
+ offset = get_random_u32_below((end - start) >> PAGE_SHIFT);
+ addr = start + (offset << PAGE_SHIFT);
+ } else
+ addr = start;
+
+ return addr;
+}
+
+static inline unsigned long vdso_base(unsigned int len)
+{
+ return vdso_addr(current->mm->start_stack, len);
}
int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
@@ -177,7 +205,7 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
*/
size = VVAR_SIZE + info->size;
- data_addr = get_unmapped_area(NULL, vdso_base(), size, 0, 0);
+ data_addr = get_unmapped_area(NULL, vdso_base(size), size, 0, 0);
if (IS_ERR_VALUE(data_addr)) {
ret = data_addr;
goto out;