diff mbox series

[5/7] ftrace: Store direct called addresses in their ops

Message ID	20230316173811.1223508-6-revest@chromium.org
State	New
Headers	Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20; From: Florent Revest <revest@chromium.org> To: linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org Cc: rostedt@goodmis.org, mhiramat@kernel.org, mark.rutland@arm.com, ast@kernel.org, daniel@iogearbox.net, kpsingh@kernel.org, revest@chromium.org, jolsa@kernel.org Subject: [PATCH 5/7] ftrace: Store direct called addresses in their ops Date: Thu, 16 Mar 2023 18:38:09 +0100 Message-Id: <20230316173811.1223508-6-revest@chromium.org> In-Reply-To: <20230316173811.1223508-1-revest@chromium.org> References: <20230316173811.1223508-1-revest@chromium.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Precedence: bulk
Series	Refactor ftrace direct call APIs \| [0/7] Refactor ftrace direct call APIs [1/7] ftrace: Let unregister_ftrace_direct_multi() call ftrace_free_filter() [2/7] ftrace: Replace uses of _ftrace_direct APIs with _ftrace_direct_multi [3/7] ftrace: Remove the legacy _ftrace_direct API [4/7] ftrace: Rename _ftrace_direct_multi APIs to _ftrace_direct APIs [5/7] ftrace: Store direct called addresses in their ops [6/7] ftrace: Make DIRECT_CALLS work WITH_ARGS and !WITH_REGS [7/7] ftrace: selftest: remove broken trace_direct_tramp

Commit Message

Florent Revest March 16, 2023, 5:38 p.m. UTC

  All direct calls are now registered using the register_ftrace_direct API
so each ops can jump to only one direct-called trampoline.

By storing the direct called trampoline address directly in the ops we
can save one hashmap lookup in the direct call ops and implement arm64
direct calls on top of call ops.

Signed-off-by: Florent Revest <revest@chromium.org>
---
 include/linux/ftrace.h | 3 +++
 kernel/trace/ftrace.c  | 7 +++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

Comments

Jiri Olsa March 19, 2023, 3:29 p.m. UTC | #1

On Thu, Mar 16, 2023 at 06:38:09PM +0100, Florent Revest wrote:
> All direct calls are now registered using the register_ftrace_direct API
> so each ops can jump to only one direct-called trampoline.
> 
> By storing the direct called trampoline address directly in the ops we
> can save one hashmap lookup in the direct call ops and implement arm64
> direct calls on top of call ops.
> 
> Signed-off-by: Florent Revest <revest@chromium.org>
> ---
>  include/linux/ftrace.h | 3 +++
>  kernel/trace/ftrace.c  | 7 +++++--
>  2 files changed, 8 insertions(+), 2 deletions(-)
> 
> diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
> index abee60865fc7..6a532dd6789e 100644
> --- a/include/linux/ftrace.h
> +++ b/include/linux/ftrace.h
> @@ -321,6 +321,9 @@ struct ftrace_ops {
>  	unsigned long			trampoline_size;
>  	struct list_head		list;
>  	ftrace_ops_func_t		ops_func;
> +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
> +	unsigned long			direct_call;
> +#endif
>  #endif
>  };
>  
> diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
> index 33530198d1ca..66c91fa4b6ab 100644
> --- a/kernel/trace/ftrace.c
> +++ b/kernel/trace/ftrace.c
> @@ -2582,9 +2582,8 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
>  static void call_direct_funcs(unsigned long ip, unsigned long pip,
>  			      struct ftrace_ops *ops, struct ftrace_regs *fregs)
>  {
> -	unsigned long addr;
> +	unsigned long addr = ops->direct_call;

nice, should it be read with READ_ONCE ?

jirka

>  
> -	addr = ftrace_find_rec_direct(ip);
>  	if (!addr)
>  		return;
>  
> @@ -5380,6 +5379,7 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
>  	ops->func = call_direct_funcs;
>  	ops->flags = MULTI_FLAGS;
>  	ops->trampoline = FTRACE_REGS_ADDR;
> +	ops->direct_call = addr;
>  
>  	err = register_ftrace_function_nolock(ops);
>  
> @@ -5454,6 +5454,7 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
>  	/* Enable the tmp_ops to have the same functions as the direct ops */
>  	ftrace_ops_init(&tmp_ops);
>  	tmp_ops.func_hash = ops->func_hash;
> +	tmp_ops.direct_call = addr;
>  
>  	err = register_ftrace_function_nolock(&tmp_ops);
>  	if (err)
> @@ -5475,6 +5476,8 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
>  			entry->direct = addr;
>  		}
>  	}
> +	/* Prevent store tearing if a trampoline concurrently accesses the value */
> +	WRITE_ONCE(ops->direct_call, addr);
>  
>  	mutex_unlock(&ftrace_lock);
>  
> -- 
> 2.40.0.rc2.332.ga46443480c-goog
>

Steven Rostedt March 19, 2023, 5:54 p.m. UTC | #2

On Sun, 19 Mar 2023 16:29:22 +0100
Jiri Olsa <olsajiri@gmail.com> wrote:

> > +++ b/kernel/trace/ftrace.c
> > @@ -2582,9 +2582,8 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
> >  static void call_direct_funcs(unsigned long ip, unsigned long pip,
> >  			      struct ftrace_ops *ops, struct ftrace_regs *fregs)
> >  {
> > -	unsigned long addr;
> > +	unsigned long addr = ops->direct_call;  
> 
> nice, should it be read with READ_ONCE ?

Is there a "read tearing" too?

-- Steve

> 
> jirka
> 
> >  
> > -	addr = ftrace_find_rec_direct(ip);
> >  	if (!addr)
> >  		return;
> >  
> > @@ -5380,6 +5379,7 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
> >  	ops->func = call_direct_funcs;
> >  	ops->flags = MULTI_FLAGS;
> >  	ops->trampoline = FTRACE_REGS_ADDR;
> > +	ops->direct_call = addr;
> >  
> >  	err = register_ftrace_function_nolock(ops);
> >  
> > @@ -5454,6 +5454,7 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
> >  	/* Enable the tmp_ops to have the same functions as the direct ops */
> >  	ftrace_ops_init(&tmp_ops);
> >  	tmp_ops.func_hash = ops->func_hash;
> > +	tmp_ops.direct_call = addr;
> >  
> >  	err = register_ftrace_function_nolock(&tmp_ops);
> >  	if (err)
> > @@ -5475,6 +5476,8 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
> >  			entry->direct = addr;
> >  		}
> >  	}
> > +	/* Prevent store tearing if a trampoline concurrently accesses the value */
> > +	WRITE_ONCE(ops->direct_call, addr);
> >  
> >  	mutex_unlock(&ftrace_lock);

Jiri Olsa March 19, 2023, 6:54 p.m. UTC | #3

On Sun, Mar 19, 2023 at 01:54:43PM -0400, Steven Rostedt wrote:
> On Sun, 19 Mar 2023 16:29:22 +0100
> Jiri Olsa <olsajiri@gmail.com> wrote:
> 
> > > +++ b/kernel/trace/ftrace.c
> > > @@ -2582,9 +2582,8 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
> > >  static void call_direct_funcs(unsigned long ip, unsigned long pip,
> > >  			      struct ftrace_ops *ops, struct ftrace_regs *fregs)
> > >  {
> > > -	unsigned long addr;
> > > +	unsigned long addr = ops->direct_call;  
> > 
> > nice, should it be read with READ_ONCE ?
> 
> Is there a "read tearing" too?

don't know, saw the comment in __modify_ftrace_direct and got curious
why it's not in here.. feel free to ignore, I'll look it up

jirka

> 
> -- Steve
> 
> > 
> > jirka
> > 
> > >  
> > > -	addr = ftrace_find_rec_direct(ip);
> > >  	if (!addr)
> > >  		return;
> > >  
> > > @@ -5380,6 +5379,7 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
> > >  	ops->func = call_direct_funcs;
> > >  	ops->flags = MULTI_FLAGS;
> > >  	ops->trampoline = FTRACE_REGS_ADDR;
> > > +	ops->direct_call = addr;
> > >  
> > >  	err = register_ftrace_function_nolock(ops);
> > >  
> > > @@ -5454,6 +5454,7 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
> > >  	/* Enable the tmp_ops to have the same functions as the direct ops */
> > >  	ftrace_ops_init(&tmp_ops);
> > >  	tmp_ops.func_hash = ops->func_hash;
> > > +	tmp_ops.direct_call = addr;
> > >  
> > >  	err = register_ftrace_function_nolock(&tmp_ops);
> > >  	if (err)
> > > @@ -5475,6 +5476,8 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
> > >  			entry->direct = addr;
> > >  		}
> > >  	}
> > > +	/* Prevent store tearing if a trampoline concurrently accesses the value */
> > > +	WRITE_ONCE(ops->direct_call, addr);
> > >  
> > >  	mutex_unlock(&ftrace_lock);

Florent Revest March 20, 2023, 5:45 p.m. UTC | #4

On Sun, Mar 19, 2023 at 7:55 PM Jiri Olsa <olsajiri@gmail.com> wrote:
>
> On Sun, Mar 19, 2023 at 01:54:43PM -0400, Steven Rostedt wrote:
> > On Sun, 19 Mar 2023 16:29:22 +0100
> > Jiri Olsa <olsajiri@gmail.com> wrote:
> >
> > > > +++ b/kernel/trace/ftrace.c
> > > > @@ -2582,9 +2582,8 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
> > > >  static void call_direct_funcs(unsigned long ip, unsigned long pip,
> > > >                         struct ftrace_ops *ops, struct ftrace_regs *fregs)
> > > >  {
> > > > - unsigned long addr;
> > > > + unsigned long addr = ops->direct_call;
> > >
> > > nice, should it be read with READ_ONCE ?
> >
> > Is there a "read tearing" too?
>
> don't know, saw the comment in __modify_ftrace_direct and got curious
> why it's not in here.. feel free to ignore, I'll look it up
>
> jirka

Mhh, that's a good question. Based on my current understanding, it
seems that it should have a READ_ONCE, indeed. However, I'd like Mark
to confirm/deny this. :)

If this should be a READ_ONCE, I can send a v2 series with this fixed.

Steven Rostedt March 20, 2023, 9:31 p.m. UTC | #5

On Mon, 20 Mar 2023 18:45:08 +0100
Florent Revest <revest@chromium.org> wrote:

> On Sun, Mar 19, 2023 at 7:55 PM Jiri Olsa <olsajiri@gmail.com> wrote:
> >
> > On Sun, Mar 19, 2023 at 01:54:43PM -0400, Steven Rostedt wrote:  
> > > On Sun, 19 Mar 2023 16:29:22 +0100
> > > Jiri Olsa <olsajiri@gmail.com> wrote:
> > >  
> > > > > +++ b/kernel/trace/ftrace.c
> > > > > @@ -2582,9 +2582,8 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
> > > > >  static void call_direct_funcs(unsigned long ip, unsigned long pip,
> > > > >                         struct ftrace_ops *ops, struct ftrace_regs *fregs)
> > > > >  {
> > > > > - unsigned long addr;
> > > > > + unsigned long addr = ops->direct_call;  
> > > >
> > > > nice, should it be read with READ_ONCE ?  
> > >
> > > Is there a "read tearing" too?  
> >
> > don't know, saw the comment in __modify_ftrace_direct and got curious
> > why it's not in here.. feel free to ignore, I'll look it up
> >
> > jirka  
> 
> Mhh, that's a good question. Based on my current understanding, it
> seems that it should have a READ_ONCE, indeed. However, I'd like Mark
> to confirm/deny this. :)
> 
> If this should be a READ_ONCE, I can send a v2 series with this fixed.

After re-reading: https://lwn.net/Articles/793253/

I think we should add the READ_ONCE() (also with a comment).

-- Steve

Mark Rutland March 21, 2023, 10:20 a.m. UTC | #6

On Mon, Mar 20, 2023 at 05:31:55PM -0400, Steven Rostedt wrote:
> On Mon, 20 Mar 2023 18:45:08 +0100
> Florent Revest <revest@chromium.org> wrote:
> 
> > On Sun, Mar 19, 2023 at 7:55 PM Jiri Olsa <olsajiri@gmail.com> wrote:
> > >
> > > On Sun, Mar 19, 2023 at 01:54:43PM -0400, Steven Rostedt wrote:  
> > > > On Sun, 19 Mar 2023 16:29:22 +0100
> > > > Jiri Olsa <olsajiri@gmail.com> wrote:
> > > >  
> > > > > > +++ b/kernel/trace/ftrace.c
> > > > > > @@ -2582,9 +2582,8 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
> > > > > >  static void call_direct_funcs(unsigned long ip, unsigned long pip,
> > > > > >                         struct ftrace_ops *ops, struct ftrace_regs *fregs)
> > > > > >  {
> > > > > > - unsigned long addr;
> > > > > > + unsigned long addr = ops->direct_call;  
> > > > >
> > > > > nice, should it be read with READ_ONCE ?  
> > > >
> > > > Is there a "read tearing" too?  
> > >
> > > don't know, saw the comment in __modify_ftrace_direct and got curious
> > > why it's not in here.. feel free to ignore, I'll look it up
> > >
> > > jirka  
> > 
> > Mhh, that's a good question. Based on my current understanding, it
> > seems that it should have a READ_ONCE, indeed. However, I'd like Mark
> > to confirm/deny this. :)
> > 
> > If this should be a READ_ONCE, I can send a v2 series with this fixed.
> 
> After re-reading: https://lwn.net/Articles/793253/
> 
> I think we should add the READ_ONCE() (also with a comment).

I think so, too.

AFAICT there's nothing that prevents __modify_ftrace_direct() and
call_direct_funcs() from concurrently accessing ftrace_ops::direct_call, so we
need READ_ONCE() in call_direct_funcs() to prevent load tearing and other
issues mentioned in the article linked above.

The existing code has a similar pattern where __modify_ftrace_direct() and
ftrace_find_rec_direct() access ftrace_func_entry::direct concurrently. Do we
want a preparatory patch fixing that for stable?

Thanks,
Mark.

diff mbox series

Patch

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index abee60865fc7..6a532dd6789e 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -321,6 +321,9 @@  struct ftrace_ops {
 	unsigned long			trampoline_size;
 	struct list_head		list;
 	ftrace_ops_func_t		ops_func;
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+	unsigned long			direct_call;
+#endif
 #endif
 };
 
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index 33530198d1ca..66c91fa4b6ab 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -2582,9 +2582,8 @@  ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
 static void call_direct_funcs(unsigned long ip, unsigned long pip,
 			      struct ftrace_ops *ops, struct ftrace_regs *fregs)
 {
-	unsigned long addr;
+	unsigned long addr = ops->direct_call;
 
-	addr = ftrace_find_rec_direct(ip);
 	if (!addr)
 		return;
 
@@ -5380,6 +5379,7 @@  int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
 	ops->func = call_direct_funcs;
 	ops->flags = MULTI_FLAGS;
 	ops->trampoline = FTRACE_REGS_ADDR;
+	ops->direct_call = addr;
 
 	err = register_ftrace_function_nolock(ops);
 
@@ -5454,6 +5454,7 @@  __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
 	/* Enable the tmp_ops to have the same functions as the direct ops */
 	ftrace_ops_init(&tmp_ops);
 	tmp_ops.func_hash = ops->func_hash;
+	tmp_ops.direct_call = addr;
 
 	err = register_ftrace_function_nolock(&tmp_ops);
 	if (err)
@@ -5475,6 +5476,8 @@  __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
 			entry->direct = addr;
 		}
 	}
+	/* Prevent store tearing if a trampoline concurrently accesses the value */
+	WRITE_ONCE(ops->direct_call, addr);
 
 	mutex_unlock(&ftrace_lock);