diff mbox series

[v3,07/35] mm/slab: introduce SLAB_NO_OBJ_EXT to avoid obj_ext creation

Message ID 20240212213922.783301-8-surenb@google.com
State New
Headers
Series Memory allocation profiling |

Commit Message

Suren Baghdasaryan Feb. 12, 2024, 9:38 p.m. UTC 
  Slab extension objects can't be allocated before slab infrastructure is
initialized. Some caches, like kmem_cache and kmem_cache_node, are created
before slab infrastructure is initialized. Objects from these caches can't
have extension objects. Introduce SLAB_NO_OBJ_EXT slab flag to mark these
caches and avoid creating extensions for objects allocated from these
slabs.

Signed-off-by: Suren Baghdasaryan <surenb@google.com>
---
 include/linux/slab.h | 7 +++++++
 mm/slub.c            | 5 +++--
 2 files changed, 10 insertions(+), 2 deletions(-)

Comments

Kees Cook Feb. 12, 2024, 10:14 p.m. UTC | #1 
On Mon, Feb 12, 2024 at 01:38:53PM -0800, Suren Baghdasaryan wrote:
> Slab extension objects can't be allocated before slab infrastructure is
> initialized. Some caches, like kmem_cache and kmem_cache_node, are created
> before slab infrastructure is initialized. Objects from these caches can't
> have extension objects. Introduce SLAB_NO_OBJ_EXT slab flag to mark these
> caches and avoid creating extensions for objects allocated from these
> slabs.
> 
> Signed-off-by: Suren Baghdasaryan <surenb@google.com>

Reviewed-by: Kees Cook <keescook@chromium.org>
Vlastimil Babka Feb. 15, 2024, 9:31 p.m. UTC | #2 
On 2/12/24 22:38, Suren Baghdasaryan wrote:
> Slab extension objects can't be allocated before slab infrastructure is
> initialized. Some caches, like kmem_cache and kmem_cache_node, are created
> before slab infrastructure is initialized. Objects from these caches can't
> have extension objects. Introduce SLAB_NO_OBJ_EXT slab flag to mark these
> caches and avoid creating extensions for objects allocated from these
> slabs.
> 
> Signed-off-by: Suren Baghdasaryan <surenb@google.com>
> ---
>  include/linux/slab.h | 7 +++++++
>  mm/slub.c            | 5 +++--
>  2 files changed, 10 insertions(+), 2 deletions(-)
> 
> diff --git a/include/linux/slab.h b/include/linux/slab.h
> index b5f5ee8308d0..3ac2fc830f0f 100644
> --- a/include/linux/slab.h
> +++ b/include/linux/slab.h
> @@ -164,6 +164,13 @@
>  #endif
>  #define SLAB_TEMPORARY		SLAB_RECLAIM_ACCOUNT	/* Objects are short-lived */
>  
> +#ifdef CONFIG_SLAB_OBJ_EXT
> +/* Slab created using create_boot_cache */
> +#define SLAB_NO_OBJ_EXT         ((slab_flags_t __force)0x20000000U)

There's
   #define SLAB_SKIP_KFENCE        ((slab_flags_t __force)0x20000000U)
already, so need some other one?

> +#else
> +#define SLAB_NO_OBJ_EXT         0
> +#endif
> +
>  /*
>   * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
>   *
> diff --git a/mm/slub.c b/mm/slub.c
> index 1eb1050814aa..9fd96238ed39 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -5650,7 +5650,8 @@ void __init kmem_cache_init(void)
>  		node_set(node, slab_nodes);
>  
>  	create_boot_cache(kmem_cache_node, "kmem_cache_node",
> -		sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, 0, 0);
> +			sizeof(struct kmem_cache_node),
> +			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
>  
>  	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
>  
> @@ -5660,7 +5661,7 @@ void __init kmem_cache_init(void)
>  	create_boot_cache(kmem_cache, "kmem_cache",
>  			offsetof(struct kmem_cache, node) +
>  				nr_node_ids * sizeof(struct kmem_cache_node *),
> -		       SLAB_HWCACHE_ALIGN, 0, 0);
> +			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
>  
>  	kmem_cache = bootstrap(&boot_kmem_cache);
>  	kmem_cache_node = bootstrap(&boot_kmem_cache_node);
Kent Overstreet Feb. 15, 2024, 9:37 p.m. UTC | #3 
On Thu, Feb 15, 2024 at 10:31:06PM +0100, Vlastimil Babka wrote:
> On 2/12/24 22:38, Suren Baghdasaryan wrote:
> > Slab extension objects can't be allocated before slab infrastructure is
> > initialized. Some caches, like kmem_cache and kmem_cache_node, are created
> > before slab infrastructure is initialized. Objects from these caches can't
> > have extension objects. Introduce SLAB_NO_OBJ_EXT slab flag to mark these
> > caches and avoid creating extensions for objects allocated from these
> > slabs.
> > 
> > Signed-off-by: Suren Baghdasaryan <surenb@google.com>
> > ---
> >  include/linux/slab.h | 7 +++++++
> >  mm/slub.c            | 5 +++--
> >  2 files changed, 10 insertions(+), 2 deletions(-)
> > 
> > diff --git a/include/linux/slab.h b/include/linux/slab.h
> > index b5f5ee8308d0..3ac2fc830f0f 100644
> > --- a/include/linux/slab.h
> > +++ b/include/linux/slab.h
> > @@ -164,6 +164,13 @@
> >  #endif
> >  #define SLAB_TEMPORARY		SLAB_RECLAIM_ACCOUNT	/* Objects are short-lived */
> >  
> > +#ifdef CONFIG_SLAB_OBJ_EXT
> > +/* Slab created using create_boot_cache */
> > +#define SLAB_NO_OBJ_EXT         ((slab_flags_t __force)0x20000000U)
> 
> There's
>    #define SLAB_SKIP_KFENCE        ((slab_flags_t __force)0x20000000U)
> already, so need some other one?

What's up with the order of flags in that file? They don't seem to
follow any particular ordering.

Seems like some cleanup is in order, but any history/context we should
know first?
Vlastimil Babka Feb. 15, 2024, 9:50 p.m. UTC | #4 
On 2/15/24 22:37, Kent Overstreet wrote:
> On Thu, Feb 15, 2024 at 10:31:06PM +0100, Vlastimil Babka wrote:
>> On 2/12/24 22:38, Suren Baghdasaryan wrote:
>> > Slab extension objects can't be allocated before slab infrastructure is
>> > initialized. Some caches, like kmem_cache and kmem_cache_node, are created
>> > before slab infrastructure is initialized. Objects from these caches can't
>> > have extension objects. Introduce SLAB_NO_OBJ_EXT slab flag to mark these
>> > caches and avoid creating extensions for objects allocated from these
>> > slabs.
>> > 
>> > Signed-off-by: Suren Baghdasaryan <surenb@google.com>
>> > ---
>> >  include/linux/slab.h | 7 +++++++
>> >  mm/slub.c            | 5 +++--
>> >  2 files changed, 10 insertions(+), 2 deletions(-)
>> > 
>> > diff --git a/include/linux/slab.h b/include/linux/slab.h
>> > index b5f5ee8308d0..3ac2fc830f0f 100644
>> > --- a/include/linux/slab.h
>> > +++ b/include/linux/slab.h
>> > @@ -164,6 +164,13 @@
>> >  #endif
>> >  #define SLAB_TEMPORARY		SLAB_RECLAIM_ACCOUNT	/* Objects are short-lived */
>> >  
>> > +#ifdef CONFIG_SLAB_OBJ_EXT
>> > +/* Slab created using create_boot_cache */
>> > +#define SLAB_NO_OBJ_EXT         ((slab_flags_t __force)0x20000000U)
>> 
>> There's
>>    #define SLAB_SKIP_KFENCE        ((slab_flags_t __force)0x20000000U)
>> already, so need some other one?
> 
> What's up with the order of flags in that file? They don't seem to
> follow any particular ordering.

Seems mostly in increasing order, except commit 4fd0b46e89879 broke it for
SLAB_RECLAIM_ACCOUNT?

> Seems like some cleanup is in order, but any history/context we should
> know first?

Yeah noted, but no need to sidetrack you.
Suren Baghdasaryan Feb. 15, 2024, 10:10 p.m. UTC | #5 
On Thu, Feb 15, 2024 at 1:50 PM Vlastimil Babka <vbabka@suse.cz> wrote:
>
> On 2/15/24 22:37, Kent Overstreet wrote:
> > On Thu, Feb 15, 2024 at 10:31:06PM +0100, Vlastimil Babka wrote:
> >> On 2/12/24 22:38, Suren Baghdasaryan wrote:
> >> > Slab extension objects can't be allocated before slab infrastructure is
> >> > initialized. Some caches, like kmem_cache and kmem_cache_node, are created
> >> > before slab infrastructure is initialized. Objects from these caches can't
> >> > have extension objects. Introduce SLAB_NO_OBJ_EXT slab flag to mark these
> >> > caches and avoid creating extensions for objects allocated from these
> >> > slabs.
> >> >
> >> > Signed-off-by: Suren Baghdasaryan <surenb@google.com>
> >> > ---
> >> >  include/linux/slab.h | 7 +++++++
> >> >  mm/slub.c            | 5 +++--
> >> >  2 files changed, 10 insertions(+), 2 deletions(-)
> >> >
> >> > diff --git a/include/linux/slab.h b/include/linux/slab.h
> >> > index b5f5ee8308d0..3ac2fc830f0f 100644
> >> > --- a/include/linux/slab.h
> >> > +++ b/include/linux/slab.h
> >> > @@ -164,6 +164,13 @@
> >> >  #endif
> >> >  #define SLAB_TEMPORARY            SLAB_RECLAIM_ACCOUNT    /* Objects are short-lived */
> >> >
> >> > +#ifdef CONFIG_SLAB_OBJ_EXT
> >> > +/* Slab created using create_boot_cache */
> >> > +#define SLAB_NO_OBJ_EXT         ((slab_flags_t __force)0x20000000U)
> >>
> >> There's
> >>    #define SLAB_SKIP_KFENCE        ((slab_flags_t __force)0x20000000U)
> >> already, so need some other one?

Indeed. I somehow missed it. Thanks for noticing, will fix this in the
next version.

> >
> > What's up with the order of flags in that file? They don't seem to
> > follow any particular ordering.
>
> Seems mostly in increasing order, except commit 4fd0b46e89879 broke it for
> SLAB_RECLAIM_ACCOUNT?
>
> > Seems like some cleanup is in order, but any history/context we should
> > know first?
>
> Yeah noted, but no need to sidetrack you.
Suren Baghdasaryan Feb. 16, 2024, 6:41 p.m. UTC | #6 
On Thu, Feb 15, 2024 at 10:10 PM Suren Baghdasaryan <surenb@google.com> wrote:
>
> On Thu, Feb 15, 2024 at 1:50 PM Vlastimil Babka <vbabka@suse.cz> wrote:
> >
> > On 2/15/24 22:37, Kent Overstreet wrote:
> > > On Thu, Feb 15, 2024 at 10:31:06PM +0100, Vlastimil Babka wrote:
> > >> On 2/12/24 22:38, Suren Baghdasaryan wrote:
> > >> > Slab extension objects can't be allocated before slab infrastructure is
> > >> > initialized. Some caches, like kmem_cache and kmem_cache_node, are created
> > >> > before slab infrastructure is initialized. Objects from these caches can't
> > >> > have extension objects. Introduce SLAB_NO_OBJ_EXT slab flag to mark these
> > >> > caches and avoid creating extensions for objects allocated from these
> > >> > slabs.
> > >> >
> > >> > Signed-off-by: Suren Baghdasaryan <surenb@google.com>
> > >> > ---
> > >> >  include/linux/slab.h | 7 +++++++
> > >> >  mm/slub.c            | 5 +++--
> > >> >  2 files changed, 10 insertions(+), 2 deletions(-)
> > >> >
> > >> > diff --git a/include/linux/slab.h b/include/linux/slab.h
> > >> > index b5f5ee8308d0..3ac2fc830f0f 100644
> > >> > --- a/include/linux/slab.h
> > >> > +++ b/include/linux/slab.h
> > >> > @@ -164,6 +164,13 @@
> > >> >  #endif
> > >> >  #define SLAB_TEMPORARY            SLAB_RECLAIM_ACCOUNT    /* Objects are short-lived */
> > >> >
> > >> > +#ifdef CONFIG_SLAB_OBJ_EXT
> > >> > +/* Slab created using create_boot_cache */
> > >> > +#define SLAB_NO_OBJ_EXT         ((slab_flags_t __force)0x20000000U)
> > >>
> > >> There's
> > >>    #define SLAB_SKIP_KFENCE        ((slab_flags_t __force)0x20000000U)
> > >> already, so need some other one?
>
> Indeed. I somehow missed it. Thanks for noticing, will fix this in the
> next version.

Apparently the only unused slab flag is 0x00000200U, all others seem
to be taken. I'll use it if there are no objections.

>
> > >
> > > What's up with the order of flags in that file? They don't seem to
> > > follow any particular ordering.
> >
> > Seems mostly in increasing order, except commit 4fd0b46e89879 broke it for
> > SLAB_RECLAIM_ACCOUNT?
> >
> > > Seems like some cleanup is in order, but any history/context we should
> > > know first?
> >
> > Yeah noted, but no need to sidetrack you.
Vlastimil Babka Feb. 16, 2024, 6:49 p.m. UTC | #7 
On 2/16/24 19:41, Suren Baghdasaryan wrote:
> On Thu, Feb 15, 2024 at 10:10 PM Suren Baghdasaryan <surenb@google.com> wrote:
>>
>> On Thu, Feb 15, 2024 at 1:50 PM Vlastimil Babka <vbabka@suse.cz> wrote:
>> >
>> > On 2/15/24 22:37, Kent Overstreet wrote:
>> > > On Thu, Feb 15, 2024 at 10:31:06PM +0100, Vlastimil Babka wrote:
>> > >> On 2/12/24 22:38, Suren Baghdasaryan wrote:
>> > >> > Slab extension objects can't be allocated before slab infrastructure is
>> > >> > initialized. Some caches, like kmem_cache and kmem_cache_node, are created
>> > >> > before slab infrastructure is initialized. Objects from these caches can't
>> > >> > have extension objects. Introduce SLAB_NO_OBJ_EXT slab flag to mark these
>> > >> > caches and avoid creating extensions for objects allocated from these
>> > >> > slabs.
>> > >> >
>> > >> > Signed-off-by: Suren Baghdasaryan <surenb@google.com>
>> > >> > ---
>> > >> >  include/linux/slab.h | 7 +++++++
>> > >> >  mm/slub.c            | 5 +++--
>> > >> >  2 files changed, 10 insertions(+), 2 deletions(-)
>> > >> >
>> > >> > diff --git a/include/linux/slab.h b/include/linux/slab.h
>> > >> > index b5f5ee8308d0..3ac2fc830f0f 100644
>> > >> > --- a/include/linux/slab.h
>> > >> > +++ b/include/linux/slab.h
>> > >> > @@ -164,6 +164,13 @@
>> > >> >  #endif
>> > >> >  #define SLAB_TEMPORARY            SLAB_RECLAIM_ACCOUNT    /* Objects are short-lived */
>> > >> >
>> > >> > +#ifdef CONFIG_SLAB_OBJ_EXT
>> > >> > +/* Slab created using create_boot_cache */
>> > >> > +#define SLAB_NO_OBJ_EXT         ((slab_flags_t __force)0x20000000U)
>> > >>
>> > >> There's
>> > >>    #define SLAB_SKIP_KFENCE        ((slab_flags_t __force)0x20000000U)
>> > >> already, so need some other one?
>>
>> Indeed. I somehow missed it. Thanks for noticing, will fix this in the
>> next version.
> 
> Apparently the only unused slab flag is 0x00000200U, all others seem
> to be taken. I'll use it if there are no objections.

OK. Will look into the cleanup and consolidation - we already know
SLAB_MEM_SPREAD became dead with SLAB removed. If it comes to worst, we can
switch to 64 bits again.

>>
>> > >
>> > > What's up with the order of flags in that file? They don't seem to
>> > > follow any particular ordering.
>> >
>> > Seems mostly in increasing order, except commit 4fd0b46e89879 broke it for
>> > SLAB_RECLAIM_ACCOUNT?
>> >
>> > > Seems like some cleanup is in order, but any history/context we should
>> > > know first?
>> >
>> > Yeah noted, but no need to sidetrack you.
diff mbox series

Patch

diff --git a/include/linux/slab.h b/include/linux/slab.h
index b5f5ee8308d0..3ac2fc830f0f 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -164,6 +164,13 @@ 
 #endif
 #define SLAB_TEMPORARY		SLAB_RECLAIM_ACCOUNT	/* Objects are short-lived */
 
+#ifdef CONFIG_SLAB_OBJ_EXT
+/* Slab created using create_boot_cache */
+#define SLAB_NO_OBJ_EXT         ((slab_flags_t __force)0x20000000U)
+#else
+#define SLAB_NO_OBJ_EXT         0
+#endif
+
 /*
  * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
  *
diff --git a/mm/slub.c b/mm/slub.c
index 1eb1050814aa..9fd96238ed39 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -5650,7 +5650,8 @@  void __init kmem_cache_init(void)
 		node_set(node, slab_nodes);
 
 	create_boot_cache(kmem_cache_node, "kmem_cache_node",
-		sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, 0, 0);
+			sizeof(struct kmem_cache_node),
+			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
 
 	hotplug_memory_notifier(slab_memory_callback, SLAB_CALLBACK_PRI);
 
@@ -5660,7 +5661,7 @@  void __init kmem_cache_init(void)
 	create_boot_cache(kmem_cache, "kmem_cache",
 			offsetof(struct kmem_cache, node) +
 				nr_node_ids * sizeof(struct kmem_cache_node *),
-		       SLAB_HWCACHE_ALIGN, 0, 0);
+			SLAB_HWCACHE_ALIGN | SLAB_NO_OBJ_EXT, 0, 0);
 
 	kmem_cache = bootstrap(&boot_kmem_cache);
 	kmem_cache_node = bootstrap(&boot_kmem_cache_node);