[5/5] arm64: mte: add compression support to mteswap.c
Commit Message
Define the internal mteswap.h interface:
- _mte_alloc_and_save_tags()
- _mte_free_saved_tags()
- _mte_restore_tags()
, that encapsulates saving tags for a struct page (together with memory
allocation), restoring tags, and deleting the storage allocated for them.
These functions accept opaque pointers, which may point to 128-byte
tag buffers, as well as smaller buffers containing compressed tags, or
have compressed tags stored directly in them.
The existing code from mteswap.c operating with uncompressed tags is split
away into mteswap_nocomp.c, and the newly introduced mteswap_comp.c
provides compression with the EA0 algorithm. The latter implementation
is picked if CONFIG_ARM64_MTE_COMP=y.
Soon after booting Android, tag compression saves ~2.5x memory previously
spent by mteswap.c on tag allocations. With the growing uptime, the
savings reach 20x and even more.
Signed-off-by: Alexander Potapenko <glider@google.com>
---
arch/arm64/mm/Makefile | 5 ++++
arch/arm64/mm/mteswap.c | 20 ++++++--------
arch/arm64/mm/mteswap.h | 12 ++++++++
arch/arm64/mm/mteswap_comp.c | 50 ++++++++++++++++++++++++++++++++++
arch/arm64/mm/mteswap_nocomp.c | 37 +++++++++++++++++++++++++
5 files changed, 113 insertions(+), 11 deletions(-)
create mode 100644 arch/arm64/mm/mteswap.h
create mode 100644 arch/arm64/mm/mteswap_comp.c
create mode 100644 arch/arm64/mm/mteswap_nocomp.c
@@ -11,6 +11,11 @@ obj-$(CONFIG_TRANS_TABLE) += trans_pgd-asm.o
obj-$(CONFIG_DEBUG_VIRTUAL) += physaddr.o
obj-$(CONFIG_ARM64_MTE) += mteswap.o
obj-$(CONFIG_ARM64_MTE_COMP) += mtecomp.o
+ifdef CONFIG_ARM64_MTE_COMP
+obj-$(CONFIG_ARM64_MTE) += mteswap_comp.o
+else
+obj-$(CONFIG_ARM64_MTE) += mteswap_nocomp.o
+endif
obj-$(CONFIG_ARM64_MTE_COMP_KUNIT_TEST) += test_mtecomp.o
KASAN_SANITIZE_physaddr.o += n
@@ -7,6 +7,8 @@
#include <linux/swapops.h>
#include <asm/mte.h>
+#include "mteswap.h"
+
static DEFINE_XARRAY(mte_pages);
void *mte_allocate_tag_storage(void)
@@ -27,20 +29,19 @@ int mte_save_tags(struct page *page)
if (!page_mte_tagged(page))
return 0;
- tag_storage = mte_allocate_tag_storage();
+ tag_storage = _mte_alloc_and_save_tags(page);
+ pr_err("HERE: %s:%d\n", __FILE__, __LINE__); // TODO
if (!tag_storage)
return -ENOMEM;
- mte_save_page_tags(page_address(page), tag_storage);
-
/* page_private contains the swap entry.val set in do_swap_page */
ret = xa_store(&mte_pages, page_private(page), tag_storage, GFP_KERNEL);
if (WARN(xa_is_err(ret), "Failed to store MTE tags")) {
- mte_free_tag_storage(tag_storage);
+ _mte_free_saved_tags(tag_storage);
return xa_err(ret);
} else if (ret) {
/* Entry is being replaced, free the old entry */
- mte_free_tag_storage(ret);
+ _mte_free_saved_tags(ret);
}
return 0;
@@ -53,10 +54,7 @@ void mte_restore_tags(swp_entry_t entry, struct page *page)
if (!tags)
return;
- if (try_page_mte_tagging(page)) {
- mte_restore_page_tags(page_address(page), tags);
- set_page_mte_tagged(page);
- }
+ _mte_restore_tags(tags, page);
}
void mte_invalidate_tags(int type, pgoff_t offset)
@@ -64,7 +62,7 @@ void mte_invalidate_tags(int type, pgoff_t offset)
swp_entry_t entry = swp_entry(type, offset);
void *tags = xa_erase(&mte_pages, entry.val);
- mte_free_tag_storage(tags);
+ _mte_free_saved_tags(tags);
}
void mte_invalidate_tags_area(int type)
@@ -78,7 +76,7 @@ void mte_invalidate_tags_area(int type)
xa_lock(&mte_pages);
xas_for_each(&xa_state, tags, last_entry.val - 1) {
__xa_erase(&mte_pages, xa_state.xa_index);
- mte_free_tag_storage(tags);
+ _mte_free_saved_tags(tags);
}
xa_unlock(&mte_pages);
}
new file mode 100644
@@ -0,0 +1,12 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef ARCH_ARM64_MM_MTESWAP_H_
+#define ARCH_ARM64_MM_MTESWAP_H_
+
+#include <linux/mm_types.h>
+
+void *_mte_alloc_and_save_tags(struct page *page);
+void _mte_free_saved_tags(void *tags);
+void _mte_restore_tags(void *tags, struct page *page);
+
+#endif // ARCH_ARM64_MM_MTESWAP_H_
new file mode 100644
@@ -0,0 +1,50 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* MTE tag storage management with EA0 compression. */
+
+#include <asm/mte.h>
+#include <asm/mtecomp.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/swap.h>
+#include <linux/swapops.h>
+#include <linux/xarray.h>
+
+#include "mteswap.h"
+
+void *_mte_alloc_and_save_tags(struct page *page)
+{
+ u8 tags[128];
+ u64 handle;
+
+ mte_save_page_tags(page_address(page), tags);
+ handle = ea0_compress(tags);
+ return xa_mk_value(handle);
+}
+
+void _mte_free_saved_tags(void *storage)
+{
+ unsigned long handle = xa_to_value(storage);
+ int size;
+
+ if (!handle)
+ return;
+ size = ea0_storage_size(handle);
+ ea0_release_handle(handle);
+}
+
+void _mte_restore_tags(void *tags, struct page *page)
+{
+ u64 handle = xa_to_value(tags);
+ u8 tags_decomp[128];
+
+ if (!handle)
+ return;
+
+ if (try_page_mte_tagging(page)) {
+ if (!ea0_decompress(handle, tags_decomp))
+ return;
+ mte_restore_page_tags(page_address(page), tags_decomp);
+ set_page_mte_tagged(page);
+ }
+}
new file mode 100644
@@ -0,0 +1,37 @@
+// SPDX-License-Identifier: GPL-2.0
+
+/* MTE tag storage management without compression support. */
+
+#include <asm/mte.h>
+#include <linux/pagemap.h>
+#include <linux/slab.h>
+#include <linux/swap.h>
+#include <linux/swapops.h>
+#include <linux/xarray.h>
+
+#include "mteswap.h"
+
+void *_mte_alloc_and_save_tags(struct page *page)
+{
+ void *storage;
+
+ storage = mte_allocate_tag_storage();
+ if (!storage)
+ return NULL;
+
+ mte_save_page_tags(page_address(page), storage);
+ return storage;
+}
+
+void _mte_free_saved_tags(void *storage)
+{
+ mte_free_tag_storage(storage);
+}
+
+void _mte_restore_tags(void *tags, struct page *page)
+{
+ if (try_page_mte_tagging(page)) {
+ mte_restore_page_tags(page_address(page), tags);
+ set_page_mte_tagged(page);
+ }
+}