diff mbox series

[v4,3/5] zsmalloc: Add a LRU to zs_pool to keep track of zspages in LRU order

Message ID 20221117163839.230900-4-nphamcs@gmail.com
State New
Headers
Series Implement writeback for zsmalloc |

Commit Message

Nhat Pham Nov. 17, 2022, 4:38 p.m. UTC 
  This helps determines the coldest zspages as candidates for writeback.

Signed-off-by: Nhat Pham <nphamcs@gmail.com>
---
 mm/zsmalloc.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)

--
2.30.2

Comments

Minchan Kim Nov. 17, 2022, 10:15 p.m. UTC | #1 
On Thu, Nov 17, 2022 at 08:38:37AM -0800, Nhat Pham wrote:
> This helps determines the coldest zspages as candidates for writeback.
> 
> Signed-off-by: Nhat Pham <nphamcs@gmail.com>
> ---
>  mm/zsmalloc.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 46 insertions(+)
> 
> diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
> index 326faa751f0a..2557b55ec767 100644
> --- a/mm/zsmalloc.c
> +++ b/mm/zsmalloc.c
> @@ -239,6 +239,11 @@ struct zs_pool {
>  	/* Compact classes */
>  	struct shrinker shrinker;
> 
> +#ifdef CONFIG_ZPOOL
> +	/* List tracking the zspages in LRU order by most recently added object */
> +	struct list_head lru;
> +#endif
> +
>  #ifdef CONFIG_ZSMALLOC_STAT
>  	struct dentry *stat_dentry;
>  #endif
> @@ -260,6 +265,12 @@ struct zspage {
>  	unsigned int freeobj;
>  	struct page *first_page;
>  	struct list_head list; /* fullness list */
> +
> +#ifdef CONFIG_ZPOOL
> +	/* links the zspage to the lru list in the pool */
> +	struct list_head lru;
> +#endif
> +
>  	struct zs_pool *pool;
>  #ifdef CONFIG_COMPACTION
>  	rwlock_t lock;
> @@ -352,6 +363,18 @@ static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
>  	kmem_cache_free(pool->zspage_cachep, zspage);
>  }
> 
> +#ifdef CONFIG_ZPOOL
> +/* Moves the zspage to the front of the zspool's LRU */
> +static void move_to_front(struct zs_pool *pool, struct zspage *zspage)
> +{
> +	assert_spin_locked(&pool->lock);
> +
> +	if (!list_empty(&zspage->lru))
> +		list_del(&zspage->lru);
> +	list_add(&zspage->lru, &pool->lru);
> +}
> +#endif
> +
>  /* pool->lock(which owns the handle) synchronizes races */
>  static void record_obj(unsigned long handle, unsigned long obj)
>  {
> @@ -953,6 +976,9 @@ static void free_zspage(struct zs_pool *pool, struct size_class *class,
>  	}
> 
>  	remove_zspage(class, zspage, ZS_EMPTY);
> +#ifdef CONFIG_ZPOOL
> +	list_del(&zspage->lru);
> +#endif
>  	__free_zspage(pool, class, zspage);
>  }
> 
> @@ -998,6 +1024,10 @@ static void init_zspage(struct size_class *class, struct zspage *zspage)
>  		off %= PAGE_SIZE;
>  	}
> 
> +#ifdef CONFIG_ZPOOL
> +	INIT_LIST_HEAD(&zspage->lru);
> +#endif
> +
>  	set_freeobj(zspage, 0);
>  }
> 
> @@ -1418,6 +1448,11 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
>  		fix_fullness_group(class, zspage);
>  		record_obj(handle, obj);
>  		class_stat_inc(class, OBJ_USED, 1);
> +
> +#ifdef CONFIG_ZPOOL
> +		/* Move the zspage to front of pool's LRU */
> +		move_to_front(pool, zspage);
> +#endif
>  		spin_unlock(&pool->lock);
> 
>  		return handle;
> @@ -1444,6 +1479,10 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
> 
>  	/* We completely set up zspage so mark them as movable */
>  	SetZsPageMovable(pool, zspage);
> +#ifdef CONFIG_ZPOOL
> +	/* Move the zspage to front of pool's LRU */
> +	move_to_front(pool, zspage);
> +#endif
>  	spin_unlock(&pool->lock);

Why do we move the zspage in the alloc instead of accessor?

Isn't zs_map_object better place since it's clear semantic
that user start to access the object?

If you are concerning unnecessary churning, can we do that
only for WO access?

Yeah, it is still weird layering since allocator couldn't know
what user will do over the this access(will keep or discard)
so the lru in the allocator is not a good design but I just
want to make little more sense.
Johannes Weiner Nov. 18, 2022, 4:30 a.m. UTC | #2 
On Thu, Nov 17, 2022 at 02:15:11PM -0800, Minchan Kim wrote:
> On Thu, Nov 17, 2022 at 08:38:37AM -0800, Nhat Pham wrote:
> > This helps determines the coldest zspages as candidates for writeback.
> > 
> > Signed-off-by: Nhat Pham <nphamcs@gmail.com>
> > ---
> >  mm/zsmalloc.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
> >  1 file changed, 46 insertions(+)
> > 
> > diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
> > index 326faa751f0a..2557b55ec767 100644
> > --- a/mm/zsmalloc.c
> > +++ b/mm/zsmalloc.c
> > @@ -239,6 +239,11 @@ struct zs_pool {
> >  	/* Compact classes */
> >  	struct shrinker shrinker;
> > 
> > +#ifdef CONFIG_ZPOOL
> > +	/* List tracking the zspages in LRU order by most recently added object */
> > +	struct list_head lru;
> > +#endif
> > +
> >  #ifdef CONFIG_ZSMALLOC_STAT
> >  	struct dentry *stat_dentry;
> >  #endif
> > @@ -260,6 +265,12 @@ struct zspage {
> >  	unsigned int freeobj;
> >  	struct page *first_page;
> >  	struct list_head list; /* fullness list */
> > +
> > +#ifdef CONFIG_ZPOOL
> > +	/* links the zspage to the lru list in the pool */
> > +	struct list_head lru;
> > +#endif
> > +
> >  	struct zs_pool *pool;
> >  #ifdef CONFIG_COMPACTION
> >  	rwlock_t lock;
> > @@ -352,6 +363,18 @@ static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
> >  	kmem_cache_free(pool->zspage_cachep, zspage);
> >  }
> > 
> > +#ifdef CONFIG_ZPOOL
> > +/* Moves the zspage to the front of the zspool's LRU */
> > +static void move_to_front(struct zs_pool *pool, struct zspage *zspage)
> > +{
> > +	assert_spin_locked(&pool->lock);
> > +
> > +	if (!list_empty(&zspage->lru))
> > +		list_del(&zspage->lru);
> > +	list_add(&zspage->lru, &pool->lru);
> > +}
> > +#endif
> > +
> >  /* pool->lock(which owns the handle) synchronizes races */
> >  static void record_obj(unsigned long handle, unsigned long obj)
> >  {
> > @@ -953,6 +976,9 @@ static void free_zspage(struct zs_pool *pool, struct size_class *class,
> >  	}
> > 
> >  	remove_zspage(class, zspage, ZS_EMPTY);
> > +#ifdef CONFIG_ZPOOL
> > +	list_del(&zspage->lru);
> > +#endif
> >  	__free_zspage(pool, class, zspage);
> >  }
> > 
> > @@ -998,6 +1024,10 @@ static void init_zspage(struct size_class *class, struct zspage *zspage)
> >  		off %= PAGE_SIZE;
> >  	}
> > 
> > +#ifdef CONFIG_ZPOOL
> > +	INIT_LIST_HEAD(&zspage->lru);
> > +#endif
> > +
> >  	set_freeobj(zspage, 0);
> >  }
> > 
> > @@ -1418,6 +1448,11 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
> >  		fix_fullness_group(class, zspage);
> >  		record_obj(handle, obj);
> >  		class_stat_inc(class, OBJ_USED, 1);
> > +
> > +#ifdef CONFIG_ZPOOL
> > +		/* Move the zspage to front of pool's LRU */
> > +		move_to_front(pool, zspage);
> > +#endif
> >  		spin_unlock(&pool->lock);
> > 
> >  		return handle;
> > @@ -1444,6 +1479,10 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
> > 
> >  	/* We completely set up zspage so mark them as movable */
> >  	SetZsPageMovable(pool, zspage);
> > +#ifdef CONFIG_ZPOOL
> > +	/* Move the zspage to front of pool's LRU */
> > +	move_to_front(pool, zspage);
> > +#endif
> >  	spin_unlock(&pool->lock);
> 
> Why do we move the zspage in the alloc instead of accessor?
> 
> Isn't zs_map_object better place since it's clear semantic
> that user start to access the object?

Remember that this is used for swap, and these entries aren't accessed
on an ongoing basis while in the pool. An access means swapin. So
functionally this is fine.

On cleaner choices, I would actually agree with you that map would be
more appropriate. But all I can do is repeat replies from previous
questions: We're not reinventing the wheel here. zbud and z3fold do it
this way, and this follows their precedent. We've talked about wanting
to generalize the LRU, and that's a heck of a lot easier if we have
copies of *one* implementation that we can deduplicate - instead of
having to merge multiple implementations with arbitrary differences.

So as you review these patches, please compare what they do to zbud
and z3fold. Those existing zpool implementations have been a common
answer in response to review questions.
Minchan Kim Nov. 18, 2022, 7:24 p.m. UTC | #3 
On Thu, Nov 17, 2022 at 11:30:11PM -0500, Johannes Weiner wrote:
> On Thu, Nov 17, 2022 at 02:15:11PM -0800, Minchan Kim wrote:
> > On Thu, Nov 17, 2022 at 08:38:37AM -0800, Nhat Pham wrote:
> > > This helps determines the coldest zspages as candidates for writeback.
> > > 
> > > Signed-off-by: Nhat Pham <nphamcs@gmail.com>
> > > ---
> > >  mm/zsmalloc.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
> > >  1 file changed, 46 insertions(+)
> > > 
> > > diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
> > > index 326faa751f0a..2557b55ec767 100644
> > > --- a/mm/zsmalloc.c
> > > +++ b/mm/zsmalloc.c
> > > @@ -239,6 +239,11 @@ struct zs_pool {
> > >  	/* Compact classes */
> > >  	struct shrinker shrinker;
> > > 
> > > +#ifdef CONFIG_ZPOOL
> > > +	/* List tracking the zspages in LRU order by most recently added object */
> > > +	struct list_head lru;
> > > +#endif
> > > +
> > >  #ifdef CONFIG_ZSMALLOC_STAT
> > >  	struct dentry *stat_dentry;
> > >  #endif
> > > @@ -260,6 +265,12 @@ struct zspage {
> > >  	unsigned int freeobj;
> > >  	struct page *first_page;
> > >  	struct list_head list; /* fullness list */
> > > +
> > > +#ifdef CONFIG_ZPOOL
> > > +	/* links the zspage to the lru list in the pool */
> > > +	struct list_head lru;
> > > +#endif
> > > +
> > >  	struct zs_pool *pool;
> > >  #ifdef CONFIG_COMPACTION
> > >  	rwlock_t lock;
> > > @@ -352,6 +363,18 @@ static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
> > >  	kmem_cache_free(pool->zspage_cachep, zspage);
> > >  }
> > > 
> > > +#ifdef CONFIG_ZPOOL
> > > +/* Moves the zspage to the front of the zspool's LRU */
> > > +static void move_to_front(struct zs_pool *pool, struct zspage *zspage)
> > > +{
> > > +	assert_spin_locked(&pool->lock);
> > > +
> > > +	if (!list_empty(&zspage->lru))
> > > +		list_del(&zspage->lru);
> > > +	list_add(&zspage->lru, &pool->lru);
> > > +}
> > > +#endif
> > > +
> > >  /* pool->lock(which owns the handle) synchronizes races */
> > >  static void record_obj(unsigned long handle, unsigned long obj)
> > >  {
> > > @@ -953,6 +976,9 @@ static void free_zspage(struct zs_pool *pool, struct size_class *class,
> > >  	}
> > > 
> > >  	remove_zspage(class, zspage, ZS_EMPTY);
> > > +#ifdef CONFIG_ZPOOL
> > > +	list_del(&zspage->lru);
> > > +#endif
> > >  	__free_zspage(pool, class, zspage);
> > >  }
> > > 
> > > @@ -998,6 +1024,10 @@ static void init_zspage(struct size_class *class, struct zspage *zspage)
> > >  		off %= PAGE_SIZE;
> > >  	}
> > > 
> > > +#ifdef CONFIG_ZPOOL
> > > +	INIT_LIST_HEAD(&zspage->lru);
> > > +#endif
> > > +
> > >  	set_freeobj(zspage, 0);
> > >  }
> > > 
> > > @@ -1418,6 +1448,11 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
> > >  		fix_fullness_group(class, zspage);
> > >  		record_obj(handle, obj);
> > >  		class_stat_inc(class, OBJ_USED, 1);
> > > +
> > > +#ifdef CONFIG_ZPOOL
> > > +		/* Move the zspage to front of pool's LRU */
> > > +		move_to_front(pool, zspage);
> > > +#endif
> > >  		spin_unlock(&pool->lock);
> > > 
> > >  		return handle;
> > > @@ -1444,6 +1479,10 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
> > > 
> > >  	/* We completely set up zspage so mark them as movable */
> > >  	SetZsPageMovable(pool, zspage);
> > > +#ifdef CONFIG_ZPOOL
> > > +	/* Move the zspage to front of pool's LRU */
> > > +	move_to_front(pool, zspage);
> > > +#endif
> > >  	spin_unlock(&pool->lock);
> > 
> > Why do we move the zspage in the alloc instead of accessor?
> > 
> > Isn't zs_map_object better place since it's clear semantic
> > that user start to access the object?
> 
> Remember that this is used for swap, and these entries aren't accessed
> on an ongoing basis while in the pool. An access means swapin. So
> functionally this is fine.
> 
> On cleaner choices, I would actually agree with you that map would be
> more appropriate. But all I can do is repeat replies from previous
> questions: We're not reinventing the wheel here. zbud and z3fold do it
> this way, and this follows their precedent. We've talked about wanting
> to generalize the LRU, and that's a heck of a lot easier if we have
> copies of *one* implementation that we can deduplicate - instead of
> having to merge multiple implementations with arbitrary differences.

It's already very weird to add the LRU logic into allocator but since
you have talked that you are working general LRU concept what I
preferred and this temporal work would be beneficial to get all the
requirements for the work, I wanted to help this as interim solution
since I know the general LRU is bigger work than this. However, 
If you also agree the suggestion(adding LRU into mapping function
rather than allocation) and it doesn't change any behavior, why not?

If the divergence would make harder for your upcoming LRU work, I
totally agree with you but I don't see such one line change makes your
duduplication work harder. At the same time, I am trying to make
zsmalloc *reasonable* rather than "Yeah, I am fine to take something
smelly code since others are already doing it" stance.

> 
> So as you review these patches, please compare what they do to zbud
> and z3fold. Those existing zpool implementations have been a common
> answer in response to review questions.

I did and thought it's weird and the correction is trivial.
Johannes Weiner Nov. 18, 2022, 7:54 p.m. UTC | #4 
On Fri, Nov 18, 2022 at 11:24:13AM -0800, Minchan Kim wrote:
> On Thu, Nov 17, 2022 at 11:30:11PM -0500, Johannes Weiner wrote:
> > On Thu, Nov 17, 2022 at 02:15:11PM -0800, Minchan Kim wrote:
> > > On Thu, Nov 17, 2022 at 08:38:37AM -0800, Nhat Pham wrote:
> > > > This helps determines the coldest zspages as candidates for writeback.
> > > > 
> > > > Signed-off-by: Nhat Pham <nphamcs@gmail.com>
> > > > ---
> > > >  mm/zsmalloc.c | 46 ++++++++++++++++++++++++++++++++++++++++++++++
> > > >  1 file changed, 46 insertions(+)
> > > > 
> > > > diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
> > > > index 326faa751f0a..2557b55ec767 100644
> > > > --- a/mm/zsmalloc.c
> > > > +++ b/mm/zsmalloc.c
> > > > @@ -239,6 +239,11 @@ struct zs_pool {
> > > >  	/* Compact classes */
> > > >  	struct shrinker shrinker;
> > > > 
> > > > +#ifdef CONFIG_ZPOOL
> > > > +	/* List tracking the zspages in LRU order by most recently added object */
> > > > +	struct list_head lru;
> > > > +#endif
> > > > +
> > > >  #ifdef CONFIG_ZSMALLOC_STAT
> > > >  	struct dentry *stat_dentry;
> > > >  #endif
> > > > @@ -260,6 +265,12 @@ struct zspage {
> > > >  	unsigned int freeobj;
> > > >  	struct page *first_page;
> > > >  	struct list_head list; /* fullness list */
> > > > +
> > > > +#ifdef CONFIG_ZPOOL
> > > > +	/* links the zspage to the lru list in the pool */
> > > > +	struct list_head lru;
> > > > +#endif
> > > > +
> > > >  	struct zs_pool *pool;
> > > >  #ifdef CONFIG_COMPACTION
> > > >  	rwlock_t lock;
> > > > @@ -352,6 +363,18 @@ static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
> > > >  	kmem_cache_free(pool->zspage_cachep, zspage);
> > > >  }
> > > > 
> > > > +#ifdef CONFIG_ZPOOL
> > > > +/* Moves the zspage to the front of the zspool's LRU */
> > > > +static void move_to_front(struct zs_pool *pool, struct zspage *zspage)
> > > > +{
> > > > +	assert_spin_locked(&pool->lock);
> > > > +
> > > > +	if (!list_empty(&zspage->lru))
> > > > +		list_del(&zspage->lru);
> > > > +	list_add(&zspage->lru, &pool->lru);
> > > > +}
> > > > +#endif
> > > > +
> > > >  /* pool->lock(which owns the handle) synchronizes races */
> > > >  static void record_obj(unsigned long handle, unsigned long obj)
> > > >  {
> > > > @@ -953,6 +976,9 @@ static void free_zspage(struct zs_pool *pool, struct size_class *class,
> > > >  	}
> > > > 
> > > >  	remove_zspage(class, zspage, ZS_EMPTY);
> > > > +#ifdef CONFIG_ZPOOL
> > > > +	list_del(&zspage->lru);
> > > > +#endif
> > > >  	__free_zspage(pool, class, zspage);
> > > >  }
> > > > 
> > > > @@ -998,6 +1024,10 @@ static void init_zspage(struct size_class *class, struct zspage *zspage)
> > > >  		off %= PAGE_SIZE;
> > > >  	}
> > > > 
> > > > +#ifdef CONFIG_ZPOOL
> > > > +	INIT_LIST_HEAD(&zspage->lru);
> > > > +#endif
> > > > +
> > > >  	set_freeobj(zspage, 0);
> > > >  }
> > > > 
> > > > @@ -1418,6 +1448,11 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
> > > >  		fix_fullness_group(class, zspage);
> > > >  		record_obj(handle, obj);
> > > >  		class_stat_inc(class, OBJ_USED, 1);
> > > > +
> > > > +#ifdef CONFIG_ZPOOL
> > > > +		/* Move the zspage to front of pool's LRU */
> > > > +		move_to_front(pool, zspage);
> > > > +#endif
> > > >  		spin_unlock(&pool->lock);
> > > > 
> > > >  		return handle;
> > > > @@ -1444,6 +1479,10 @@ unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
> > > > 
> > > >  	/* We completely set up zspage so mark them as movable */
> > > >  	SetZsPageMovable(pool, zspage);
> > > > +#ifdef CONFIG_ZPOOL
> > > > +	/* Move the zspage to front of pool's LRU */
> > > > +	move_to_front(pool, zspage);
> > > > +#endif
> > > >  	spin_unlock(&pool->lock);
> > > 
> > > Why do we move the zspage in the alloc instead of accessor?
> > > 
> > > Isn't zs_map_object better place since it's clear semantic
> > > that user start to access the object?
> > 
> > Remember that this is used for swap, and these entries aren't accessed
> > on an ongoing basis while in the pool. An access means swapin. So
> > functionally this is fine.
> > 
> > On cleaner choices, I would actually agree with you that map would be
> > more appropriate. But all I can do is repeat replies from previous
> > questions: We're not reinventing the wheel here. zbud and z3fold do it
> > this way, and this follows their precedent. We've talked about wanting
> > to generalize the LRU, and that's a heck of a lot easier if we have
> > copies of *one* implementation that we can deduplicate - instead of
> > having to merge multiple implementations with arbitrary differences.
> 
> It's already very weird to add the LRU logic into allocator but since
> you have talked that you are working general LRU concept what I
> preferred and this temporal work would be beneficial to get all the
> requirements for the work, I wanted to help this as interim solution
> since I know the general LRU is bigger work than this. However, 
> If you also agree the suggestion(adding LRU into mapping function
> rather than allocation) and it doesn't change any behavior, why not?

map is called during writeback and load, where we wouldn't want to
update the LRU. Sure, you can filter for RO/WO, but that's also not
generic from an interface POV because it's entirely specific to zswap
operations which type of accesses *should* rotate the LRU. We'd also
call LRU code from previously untested contexts and thereby lose our
existing prod testing of these patches.

I think your asking to take risks and introduce variance into a code
base we're already trying to unify, for something that isn't actually
cleaner or more generic.

So respectfully, I disagree with this change request rather strongly.
diff mbox series

Patch

diff --git a/mm/zsmalloc.c b/mm/zsmalloc.c
index 326faa751f0a..2557b55ec767 100644
--- a/mm/zsmalloc.c
+++ b/mm/zsmalloc.c
@@ -239,6 +239,11 @@  struct zs_pool {
 	/* Compact classes */
 	struct shrinker shrinker;

+#ifdef CONFIG_ZPOOL
+	/* List tracking the zspages in LRU order by most recently added object */
+	struct list_head lru;
+#endif
+
 #ifdef CONFIG_ZSMALLOC_STAT
 	struct dentry *stat_dentry;
 #endif
@@ -260,6 +265,12 @@  struct zspage {
 	unsigned int freeobj;
 	struct page *first_page;
 	struct list_head list; /* fullness list */
+
+#ifdef CONFIG_ZPOOL
+	/* links the zspage to the lru list in the pool */
+	struct list_head lru;
+#endif
+
 	struct zs_pool *pool;
 #ifdef CONFIG_COMPACTION
 	rwlock_t lock;
@@ -352,6 +363,18 @@  static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
 	kmem_cache_free(pool->zspage_cachep, zspage);
 }

+#ifdef CONFIG_ZPOOL
+/* Moves the zspage to the front of the zspool's LRU */
+static void move_to_front(struct zs_pool *pool, struct zspage *zspage)
+{
+	assert_spin_locked(&pool->lock);
+
+	if (!list_empty(&zspage->lru))
+		list_del(&zspage->lru);
+	list_add(&zspage->lru, &pool->lru);
+}
+#endif
+
 /* pool->lock(which owns the handle) synchronizes races */
 static void record_obj(unsigned long handle, unsigned long obj)
 {
@@ -953,6 +976,9 @@  static void free_zspage(struct zs_pool *pool, struct size_class *class,
 	}

 	remove_zspage(class, zspage, ZS_EMPTY);
+#ifdef CONFIG_ZPOOL
+	list_del(&zspage->lru);
+#endif
 	__free_zspage(pool, class, zspage);
 }

@@ -998,6 +1024,10 @@  static void init_zspage(struct size_class *class, struct zspage *zspage)
 		off %= PAGE_SIZE;
 	}

+#ifdef CONFIG_ZPOOL
+	INIT_LIST_HEAD(&zspage->lru);
+#endif
+
 	set_freeobj(zspage, 0);
 }

@@ -1418,6 +1448,11 @@  unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
 		fix_fullness_group(class, zspage);
 		record_obj(handle, obj);
 		class_stat_inc(class, OBJ_USED, 1);
+
+#ifdef CONFIG_ZPOOL
+		/* Move the zspage to front of pool's LRU */
+		move_to_front(pool, zspage);
+#endif
 		spin_unlock(&pool->lock);

 		return handle;
@@ -1444,6 +1479,10 @@  unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)

 	/* We completely set up zspage so mark them as movable */
 	SetZsPageMovable(pool, zspage);
+#ifdef CONFIG_ZPOOL
+	/* Move the zspage to front of pool's LRU */
+	move_to_front(pool, zspage);
+#endif
 	spin_unlock(&pool->lock);

 	return handle;
@@ -1967,6 +2006,9 @@  static void async_free_zspage(struct work_struct *work)
 		VM_BUG_ON(fullness != ZS_EMPTY);
 		class = pool->size_class[class_idx];
 		spin_lock(&pool->lock);
+#ifdef CONFIG_ZPOOL
+		list_del(&zspage->lru);
+#endif
 		__free_zspage(pool, class, zspage);
 		spin_unlock(&pool->lock);
 	}
@@ -2278,6 +2320,10 @@  struct zs_pool *zs_create_pool(const char *name)
 	 */
 	zs_register_shrinker(pool);

+#ifdef CONFIG_ZPOOL
+	INIT_LIST_HEAD(&pool->lru);
+#endif
+
 	return pool;

 err: