@@ -987,6 +987,7 @@ Example output. You may not have all of these fields.
AnonPages: 4654780 kB
Mapped: 266244 kB
Shmem: 9976 kB
+ PageMetadata: 513419 kB
KReclaimable: 517708 kB
Slab: 660044 kB
SReclaimable: 517708 kB
@@ -1089,6 +1090,8 @@ Mapped
files which have been mmapped, such as libraries
Shmem
Total memory used by shared memory (shmem) and tmpfs
+PageMetadata
+ Memory used for per-page metadata
KReclaimable
Kernel allocations that the kernel will attempt to reclaim
under memory pressure. Includes SReclaimable (below), and other
@@ -428,6 +428,7 @@ static ssize_t node_read_meminfo(struct device *dev,
"Node %d Mapped: %8lu kB\n"
"Node %d AnonPages: %8lu kB\n"
"Node %d Shmem: %8lu kB\n"
+ "Node %d PageMetadata: %8lu kB\n"
"Node %d KernelStack: %8lu kB\n"
#ifdef CONFIG_SHADOW_CALL_STACK
"Node %d ShadowCallStack:%8lu kB\n"
@@ -458,6 +459,7 @@ static ssize_t node_read_meminfo(struct device *dev,
nid, K(node_page_state(pgdat, NR_FILE_MAPPED)),
nid, K(node_page_state(pgdat, NR_ANON_MAPPED)),
nid, K(i.sharedram),
+ nid, K(node_page_state(pgdat, NR_PAGE_METADATA)),
nid, node_page_state(pgdat, NR_KERNEL_STACK_KB),
#ifdef CONFIG_SHADOW_CALL_STACK
nid, node_page_state(pgdat, NR_KERNEL_SCS_KB),
@@ -39,7 +39,9 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
long available;
unsigned long pages[NR_LRU_LISTS];
unsigned long sreclaimable, sunreclaim;
+ unsigned long nr_page_metadata;
int lru;
+ int nid;
si_meminfo(&i);
si_swapinfo(&i);
@@ -57,6 +59,10 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
sreclaimable = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B);
sunreclaim = global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B);
+ nr_page_metadata = 0;
+ for_each_online_node(nid)
+ nr_page_metadata += node_page_state(NODE_DATA(nid), NR_PAGE_METADATA);
+
show_val_kb(m, "MemTotal: ", i.totalram);
show_val_kb(m, "MemFree: ", i.freeram);
show_val_kb(m, "MemAvailable: ", available);
@@ -104,6 +110,7 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
show_val_kb(m, "Mapped: ",
global_node_page_state(NR_FILE_MAPPED));
show_val_kb(m, "Shmem: ", i.sharedram);
+ show_val_kb(m, "PageMetadata: ", nr_page_metadata);
show_val_kb(m, "KReclaimable: ", sreclaimable +
global_node_page_state(NR_KERNEL_MISC_RECLAIMABLE));
show_val_kb(m, "Slab: ", sreclaimable + sunreclaim);
@@ -207,6 +207,9 @@ enum node_stat_item {
PGPROMOTE_SUCCESS, /* promote successfully */
PGPROMOTE_CANDIDATE, /* candidate pages to promote */
#endif
+ NR_PAGE_METADATA, /* Page metadata size (struct page and page_ext)
+ * in pages
+ */
NR_VM_NODE_STAT_ITEMS
};
@@ -656,4 +656,8 @@ static inline void lruvec_stat_sub_folio(struct folio *folio,
{
lruvec_stat_mod_folio(folio, idx, -folio_nr_pages(folio));
}
+
+void __init mod_node_early_perpage_metadata(int nid, long delta);
+void __init writeout_early_perpage_metadata(void);
+
#endif /* _LINUX_VMSTAT_H */
@@ -1758,6 +1758,10 @@ static void __update_and_free_hugetlb_folio(struct hstate *h,
destroy_compound_gigantic_folio(folio, huge_page_order(h));
free_gigantic_folio(folio, huge_page_order(h));
} else {
+#ifndef CONFIG_SPARSEMEM_VMEMMAP
+ __mod_node_page_state(NODE_DATA(page_to_nid(&folio->page)),
+ NR_PAGE_METADATA, -huge_page_order(h));
+#endif
__free_pages(&folio->page, huge_page_order(h));
}
}
@@ -2143,7 +2147,9 @@ static struct folio *alloc_buddy_hugetlb_folio(struct hstate *h,
__count_vm_event(HTLB_BUDDY_PGALLOC_FAIL);
return NULL;
}
-
+#ifndef CONFIG_SPARSEMEM_VMEMMAP
+ __mod_node_page_state(NODE_DATA(nid), NR_PAGE_METADATA, huge_page_order(h));
+#endif
__count_vm_event(HTLB_BUDDY_PGALLOC);
return page_folio(page);
}
@@ -214,6 +214,8 @@ static inline void free_vmemmap_page(struct page *page)
free_bootmem_page(page);
else
__free_page(page);
+ __mod_node_page_state(NODE_DATA(page_to_nid(page)),
+ NR_PAGE_METADATA, -1);
}
/* Free a list of the vmemmap pages */
@@ -336,6 +338,7 @@ static int vmemmap_remap_free(unsigned long start, unsigned long end,
(void *)walk.reuse_addr);
list_add(&walk.reuse_page->lru, &vmemmap_pages);
}
+ __mod_node_page_state(NODE_DATA(nid), NR_PAGE_METADATA, 1);
/*
* In order to make remapping routine most efficient for the huge pages,
@@ -387,8 +390,12 @@ static int alloc_vmemmap_page_list(unsigned long start, unsigned long end,
while (nr_pages--) {
page = alloc_pages_node(nid, gfp_mask, 0);
- if (!page)
+ if (!page) {
goto out;
+ } else {
+ __mod_node_page_state(NODE_DATA(page_to_nid(page)),
+ NR_PAGE_METADATA, 1);
+ }
list_add_tail(&page->lru, list);
}
@@ -26,6 +26,7 @@
#include <linux/pgtable.h>
#include <linux/swap.h>
#include <linux/cma.h>
+#include <linux/vmstat.h>
#include "internal.h"
#include "slab.h"
#include "shuffle.h"
@@ -1656,6 +1657,8 @@ static void __init alloc_node_mem_map(struct pglist_data *pgdat)
panic("Failed to allocate %ld bytes for node %d memory map\n",
size, pgdat->node_id);
pgdat->node_mem_map = map + offset;
+ mod_node_early_perpage_metadata(pgdat->node_id,
+ PAGE_ALIGN(size) >> PAGE_SHIFT);
}
pr_debug("%s: node %d, pgdat %08lx, node_mem_map %08lx\n",
__func__, pgdat->node_id, (unsigned long)pgdat,
@@ -5443,6 +5443,7 @@ void __init setup_per_cpu_pageset(void)
for_each_online_pgdat(pgdat)
pgdat->per_cpu_nodestats =
alloc_percpu(struct per_cpu_nodestat);
+ writeout_early_perpage_metadata();
}
__meminit void zone_pcp_init(struct zone *zone)
@@ -201,6 +201,8 @@ static int __init alloc_node_page_ext(int nid)
return -ENOMEM;
NODE_DATA(nid)->node_page_ext = base;
total_usage += table_size;
+ __mod_node_page_state(NODE_DATA(nid), NR_PAGE_METADATA,
+ PAGE_ALIGN(table_size) >> PAGE_SHIFT);
return 0;
}
@@ -255,12 +257,15 @@ static void *__meminit alloc_page_ext(size_t size, int nid)
void *addr = NULL;
addr = alloc_pages_exact_nid(nid, size, flags);
- if (addr) {
+ if (addr)
kmemleak_alloc(addr, size, 1, flags);
- return addr;
- }
+ else
+ addr = vzalloc_node(size, nid);
- addr = vzalloc_node(size, nid);
+ if (addr) {
+ __mod_node_page_state(NODE_DATA(nid), NR_PAGE_METADATA,
+ PAGE_ALIGN(size) >> PAGE_SHIFT);
+ }
return addr;
}
@@ -314,6 +319,10 @@ static void free_page_ext(void *addr)
BUG_ON(PageReserved(page));
kmemleak_free(addr);
free_pages_exact(addr, table_size);
+
+ __mod_node_page_state(NODE_DATA(page_to_nid(page)), NR_PAGE_METADATA,
+ (long)-1 * (PAGE_ALIGN(table_size) >> PAGE_SHIFT));
+
}
}
@@ -469,5 +469,8 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn,
if (r < 0)
return NULL;
+ __mod_node_page_state(NODE_DATA(nid), NR_PAGE_METADATA,
+ PAGE_ALIGN(end - start) >> PAGE_SHIFT);
+
return pfn_to_page(pfn);
}
@@ -14,7 +14,7 @@
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/bootmem_info.h>
-
+#include <linux/vmstat.h>
#include "internal.h"
#include <asm/dma.h>
@@ -465,6 +465,9 @@ static void __init sparse_buffer_init(unsigned long size, int nid)
*/
sparsemap_buf = memmap_alloc(size, section_map_size(), addr, nid, true);
sparsemap_buf_end = sparsemap_buf + size;
+#ifndef CONFIG_SPARSEMEM_VMEMMAP
+ mod_node_early_perpage_metadata(nid, PAGE_ALIGN(size) >> PAGE_SHIFT);
+#endif
}
static void __init sparse_buffer_fini(void)
@@ -641,6 +644,8 @@ static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
unsigned long start = (unsigned long) pfn_to_page(pfn);
unsigned long end = start + nr_pages * sizeof(struct page);
+ __mod_node_page_state(NODE_DATA(page_to_nid(pfn_to_page(pfn))), NR_PAGE_METADATA,
+ (long)-1 * (PAGE_ALIGN(end - start) >> PAGE_SHIFT));
vmemmap_free(start, end, altmap);
}
static void free_map_bootmem(struct page *memmap)
@@ -1245,6 +1245,7 @@ const char * const vmstat_text[] = {
"pgpromote_success",
"pgpromote_candidate",
#endif
+ "nr_page_metadata",
/* enum writeback_stat_item counters */
"nr_dirty_threshold",
@@ -2274,4 +2275,24 @@ static int __init extfrag_debug_init(void)
}
module_init(extfrag_debug_init);
+
+// Page metadata size (struct page and page_ext) in pages
+unsigned long early_perpage_metadata[MAX_NUMNODES] __initdata;
+
+void __init mod_node_early_perpage_metadata(int nid, long delta)
+{
+ early_perpage_metadata[nid] += delta;
+}
+
+void __init writeout_early_perpage_metadata(void)
+{
+ int nid;
+ struct pglist_data *pgdat;
+
+ for_each_online_pgdat(pgdat) {
+ nid = pgdat->node_id;
+ __mod_node_page_state(NODE_DATA(nid), NR_PAGE_METADATA,
+ early_perpage_metadata[nid]);
+ }
+}
#endif