diff mbox series

[RFC,13/14] mm/slub: sanity-check freepointers

Message ID 20230915105933.495735-14-matteorizzo@google.com
State New
Headers
Series Prevent cross-cache attacks in the SLUB allocator |

Commit Message

Matteo Rizzo Sept. 15, 2023, 10:59 a.m. UTC 
  From: Jann Horn <jannh@google.com>

Sanity-check that:
 - non-NULL freepointers point into the slab
 - freepointers look plausibly aligned

Signed-off-by: Jann Horn <jannh@google.com>
Co-developed-by: Matteo Rizzo <matteorizzo@google.com>
Signed-off-by: Matteo Rizzo <matteorizzo@google.com>
---
 lib/slub_kunit.c |  4 ++++
 mm/slab.h        |  8 +++++++
 mm/slub.c        | 57 ++++++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 69 insertions(+)

Comments

Kees Cook Sept. 15, 2023, 9:26 p.m. UTC | #1 
On Fri, Sep 15, 2023 at 10:59:32AM +0000, Matteo Rizzo wrote:
> From: Jann Horn <jannh@google.com>
> 
> Sanity-check that:
>  - non-NULL freepointers point into the slab
>  - freepointers look plausibly aligned
> 
> Signed-off-by: Jann Horn <jannh@google.com>
> Co-developed-by: Matteo Rizzo <matteorizzo@google.com>
> Signed-off-by: Matteo Rizzo <matteorizzo@google.com>
> ---
>  lib/slub_kunit.c |  4 ++++
>  mm/slab.h        |  8 +++++++
>  mm/slub.c        | 57 ++++++++++++++++++++++++++++++++++++++++++++++++
>  3 files changed, 69 insertions(+)
> 
> diff --git a/lib/slub_kunit.c b/lib/slub_kunit.c
> index d4a3730b08fa..acf8600bd1fd 100644
> --- a/lib/slub_kunit.c
> +++ b/lib/slub_kunit.c
> @@ -45,6 +45,10 @@ static void test_clobber_zone(struct kunit *test)
>  #ifndef CONFIG_KASAN
>  static void test_next_pointer(struct kunit *test)
>  {
> +	if (IS_ENABLED(CONFIG_SLAB_VIRTUAL))
> +		kunit_skip(test,
> +			"incompatible with freepointer corruption detection in CONFIG_SLAB_VIRTUAL");
> +
>  	struct kmem_cache *s = test_kmem_cache_create("TestSlub_next_ptr_free",
>  							64, SLAB_POISON);
>  	u8 *p = kmem_cache_alloc(s, GFP_KERNEL);
> diff --git a/mm/slab.h b/mm/slab.h
> index 460c802924bd..8d10a011bdf0 100644
> --- a/mm/slab.h
> +++ b/mm/slab.h
> @@ -79,6 +79,14 @@ struct slab {
>  
>  	struct list_head flush_list_elem;
>  
> +	/*
> +	 * Not in kmem_cache because it depends on whether the allocation is
> +	 * normal order or fallback order.
> +	 * an alternative might be to over-allocate virtual memory for
> +	 * fallback-order pages.
> +	 */
> +	unsigned long align_mask;
> +
>  	/* Replaces the page lock */
>  	spinlock_t slab_lock;
>  
> diff --git a/mm/slub.c b/mm/slub.c
> index 0f7f5bf0b174..57474c8a6569 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -392,6 +392,44 @@ static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s,
>  	return (freeptr_t){.v = encoded};
>  }
>  
> +/*
> + * Does some validation of freelist pointers. Without SLAB_VIRTUAL this is
> + * currently a no-op.
> + */
> +static inline bool freelist_pointer_corrupted(struct slab *slab, freeptr_t ptr,
> +	void *decoded)
> +{
> +#ifdef CONFIG_SLAB_VIRTUAL
> +	/*
> +	 * If the freepointer decodes to 0, use 0 as the slab_base so that
> +	 * the check below always passes (0 & slab->align_mask == 0).
> +	 */
> +	unsigned long slab_base = decoded ? (unsigned long)slab_to_virt(slab)
> +		: 0;
> +
> +	/*
> +	 * This verifies that the SLUB freepointer does not point outside the
> +	 * slab. Since at that point we can basically do it for free, it also
> +	 * checks that the pointer alignment looks vaguely sane.
> +	 * However, we probably don't want the cost of a proper division here,
> +	 * so instead we just do a cheap check whether the bottom bits that are
> +	 * clear in the size are also clear in the pointer.
> +	 * So for kmalloc-32, it does a perfect alignment check, but for
> +	 * kmalloc-192, it just checks that the pointer is a multiple of 32.
> +	 * This should probably be reconsidered - is this a good tradeoff, or
> +	 * should that part be thrown out, or do we want a proper accurate
> +	 * alignment check (and can we make it work with acceptable performance
> +	 * cost compared to the security improvement - probably not)?

Is it really that much more expensive to check the alignment exactly?

> +	 */
> +	return CHECK_DATA_CORRUPTION(
> +		((unsigned long)decoded & slab->align_mask) != slab_base,
> +		"bad freeptr (encoded %lx, ptr %p, base %lx, mask %lx",
> +		ptr.v, decoded, slab_base, slab->align_mask);
> +#else
> +	return false;
> +#endif
> +}
> +
>  static inline void *freelist_ptr_decode(const struct kmem_cache *s,
>  					freeptr_t ptr, unsigned long ptr_addr,
>  					struct slab *slab)
> @@ -403,6 +441,10 @@ static inline void *freelist_ptr_decode(const struct kmem_cache *s,
>  #else
>  	decoded = (void *)ptr.v;
>  #endif
> +
> +	if (unlikely(freelist_pointer_corrupted(slab, ptr, decoded)))
> +		return NULL;
> +
>  	return decoded;
>  }
>  
> @@ -2122,6 +2164,21 @@ static struct slab *get_free_slab(struct kmem_cache *s,
>  	if (slab == NULL)
>  		return NULL;
>  
> +	/*
> +	 * Bits that must be equal to start-of-slab address for all
> +	 * objects inside the slab.
> +	 * For compatibility with pointer tagging (like in HWASAN), this would
> +	 * need to clear the pointer tag bits from the mask.
> +	 */
> +	slab->align_mask = ~((PAGE_SIZE << oo_order(oo)) - 1);
> +
> +	/*
> +	 * Object alignment bits (must be zero, which is equal to the bits in
> +	 * the start-of-slab address)
> +	 */
> +	if (s->red_left_pad == 0)
> +		slab->align_mask |= (1 << (ffs(s->size) - 1)) - 1;
> +
>  	return slab;
>  }
>  
> -- 
> 2.42.0.459.ge4e396fd5e-goog
> 

We can improve the sanity checking in the future, so as-is, sure:

Reviewed-by: Kees Cook <keescook@chromium.org>
diff mbox series

Patch

diff --git a/lib/slub_kunit.c b/lib/slub_kunit.c
index d4a3730b08fa..acf8600bd1fd 100644
--- a/lib/slub_kunit.c
+++ b/lib/slub_kunit.c
@@ -45,6 +45,10 @@  static void test_clobber_zone(struct kunit *test)
 #ifndef CONFIG_KASAN
 static void test_next_pointer(struct kunit *test)
 {
+	if (IS_ENABLED(CONFIG_SLAB_VIRTUAL))
+		kunit_skip(test,
+			"incompatible with freepointer corruption detection in CONFIG_SLAB_VIRTUAL");
+
 	struct kmem_cache *s = test_kmem_cache_create("TestSlub_next_ptr_free",
 							64, SLAB_POISON);
 	u8 *p = kmem_cache_alloc(s, GFP_KERNEL);
diff --git a/mm/slab.h b/mm/slab.h
index 460c802924bd..8d10a011bdf0 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -79,6 +79,14 @@  struct slab {
 
 	struct list_head flush_list_elem;
 
+	/*
+	 * Not in kmem_cache because it depends on whether the allocation is
+	 * normal order or fallback order.
+	 * an alternative might be to over-allocate virtual memory for
+	 * fallback-order pages.
+	 */
+	unsigned long align_mask;
+
 	/* Replaces the page lock */
 	spinlock_t slab_lock;
 
diff --git a/mm/slub.c b/mm/slub.c
index 0f7f5bf0b174..57474c8a6569 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -392,6 +392,44 @@  static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s,
 	return (freeptr_t){.v = encoded};
 }
 
+/*
+ * Does some validation of freelist pointers. Without SLAB_VIRTUAL this is
+ * currently a no-op.
+ */
+static inline bool freelist_pointer_corrupted(struct slab *slab, freeptr_t ptr,
+	void *decoded)
+{
+#ifdef CONFIG_SLAB_VIRTUAL
+	/*
+	 * If the freepointer decodes to 0, use 0 as the slab_base so that
+	 * the check below always passes (0 & slab->align_mask == 0).
+	 */
+	unsigned long slab_base = decoded ? (unsigned long)slab_to_virt(slab)
+		: 0;
+
+	/*
+	 * This verifies that the SLUB freepointer does not point outside the
+	 * slab. Since at that point we can basically do it for free, it also
+	 * checks that the pointer alignment looks vaguely sane.
+	 * However, we probably don't want the cost of a proper division here,
+	 * so instead we just do a cheap check whether the bottom bits that are
+	 * clear in the size are also clear in the pointer.
+	 * So for kmalloc-32, it does a perfect alignment check, but for
+	 * kmalloc-192, it just checks that the pointer is a multiple of 32.
+	 * This should probably be reconsidered - is this a good tradeoff, or
+	 * should that part be thrown out, or do we want a proper accurate
+	 * alignment check (and can we make it work with acceptable performance
+	 * cost compared to the security improvement - probably not)?
+	 */
+	return CHECK_DATA_CORRUPTION(
+		((unsigned long)decoded & slab->align_mask) != slab_base,
+		"bad freeptr (encoded %lx, ptr %p, base %lx, mask %lx",
+		ptr.v, decoded, slab_base, slab->align_mask);
+#else
+	return false;
+#endif
+}
+
 static inline void *freelist_ptr_decode(const struct kmem_cache *s,
 					freeptr_t ptr, unsigned long ptr_addr,
 					struct slab *slab)
@@ -403,6 +441,10 @@  static inline void *freelist_ptr_decode(const struct kmem_cache *s,
 #else
 	decoded = (void *)ptr.v;
 #endif
+
+	if (unlikely(freelist_pointer_corrupted(slab, ptr, decoded)))
+		return NULL;
+
 	return decoded;
 }
 
@@ -2122,6 +2164,21 @@  static struct slab *get_free_slab(struct kmem_cache *s,
 	if (slab == NULL)
 		return NULL;
 
+	/*
+	 * Bits that must be equal to start-of-slab address for all
+	 * objects inside the slab.
+	 * For compatibility with pointer tagging (like in HWASAN), this would
+	 * need to clear the pointer tag bits from the mask.
+	 */
+	slab->align_mask = ~((PAGE_SIZE << oo_order(oo)) - 1);
+
+	/*
+	 * Object alignment bits (must be zero, which is equal to the bits in
+	 * the start-of-slab address)
+	 */
+	if (s->red_left_pad == 0)
+		slab->align_mask |= (1 << (ffs(s->size) - 1)) - 1;
+
 	return slab;
 }