diff mbox series

[4/8] ftrace: Store direct called addresses in their ops

Message ID	20230201163420.1579014-5-revest@chromium.org
State	New
Headers	Received-SPF: pass (google.com: domain of linux-kernel-owner@vger.kernel.org designates 2620:137:e000::1:20 as permitted sender) client-ip=2620:137:e000::1:20; From: Florent Revest <revest@chromium.org> To: linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, linux-trace-kernel@vger.kernel.org, bpf@vger.kernel.org Cc: catalin.marinas@arm.com, will@kernel.org, rostedt@goodmis.org, mhiramat@kernel.org, mark.rutland@arm.com, ast@kernel.org, daniel@iogearbox.net, andrii@kernel.org, kpsingh@kernel.org, jolsa@kernel.org, xukuohai@huaweicloud.com, Florent Revest <revest@chromium.org> Subject: [PATCH 4/8] ftrace: Store direct called addresses in their ops Date: Wed, 1 Feb 2023 17:34:16 +0100 Message-Id: <20230201163420.1579014-5-revest@chromium.org> In-Reply-To: <20230201163420.1579014-1-revest@chromium.org> References: <20230201163420.1579014-1-revest@chromium.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Precedence: bulk
Series	Add ftrace direct call for arm64 \| [0/8] Add ftrace direct call for arm64 [1/8] ftrace: Replace uses of _ftrace_direct APIs with _ftrace_direct_multi [2/8] ftrace: Remove the legacy _ftrace_direct API [3/8] ftrace: Rename _ftrace_direct_multi APIs to _ftrace_direct APIs [4/8] ftrace: Store direct called addresses in their ops [5/8] ftrace: Make DIRECT_CALLS work WITH_ARGS and !WITH_REGS [6/8] ftrace: Fix dead loop caused by direct call in ftrace selftest [7/8] arm64: ftrace: Add direct call support [8/8] arm64: ftrace: Add direct called trampoline samples support

Commit Message

Florent Revest Feb. 1, 2023, 4:34 p.m. UTC

  All direct calls are now registered using the register_ftrace_direct API
so each ops can jump to only one direct-called trampoline.

By storing the direct called trampoline address directly in the ops we
can save one hashmap lookup in the direct call ops and implement arm64
direct calls on top of call ops.

Signed-off-by: Florent Revest <revest@chromium.org>
---
 include/linux/ftrace.h | 3 +++
 kernel/trace/ftrace.c  | 6 ++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

Comments

Mark Rutland Feb. 2, 2023, 3:29 p.m. UTC | #1

On Wed, Feb 01, 2023 at 05:34:16PM +0100, Florent Revest wrote:
> All direct calls are now registered using the register_ftrace_direct API
> so each ops can jump to only one direct-called trampoline.
> 
> By storing the direct called trampoline address directly in the ops we
> can save one hashmap lookup in the direct call ops and implement arm64
> direct calls on top of call ops.
> 
> Signed-off-by: Florent Revest <revest@chromium.org>
> ---
>  include/linux/ftrace.h | 3 +++
>  kernel/trace/ftrace.c  | 6 ++++--
>  2 files changed, 7 insertions(+), 2 deletions(-)
> 
> diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
> index a7dbd307c3a4..84f717f8959e 100644
> --- a/include/linux/ftrace.h
> +++ b/include/linux/ftrace.h
> @@ -321,6 +321,9 @@ struct ftrace_ops {
>  	unsigned long			trampoline_size;
>  	struct list_head		list;
>  	ftrace_ops_func_t		ops_func;
> +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
> +	unsigned long			direct_call;
> +#endif
>  #endif
>  };
>  
> diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
> index cb77a0a208c7..b0426de11c45 100644
> --- a/kernel/trace/ftrace.c
> +++ b/kernel/trace/ftrace.c
> @@ -2577,9 +2577,8 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
>  static void call_direct_funcs(unsigned long ip, unsigned long pip,
>  			      struct ftrace_ops *ops, struct ftrace_regs *fregs)
>  {
> -	unsigned long addr;
> +	unsigned long addr = ops->direct_call;
>  
> -	addr = ftrace_find_rec_direct(ip);
>  	if (!addr)
>  		return;
>  
> @@ -5375,6 +5374,7 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
>  	ops->func = call_direct_funcs;
>  	ops->flags = MULTI_FLAGS;
>  	ops->trampoline = FTRACE_REGS_ADDR;
> +	ops->direct_call = addr;
>  
>  	err = register_ftrace_function_nolock(ops);
>  
> @@ -5445,6 +5445,7 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
>  	/* Enable the tmp_ops to have the same functions as the direct ops */
>  	ftrace_ops_init(&tmp_ops);
>  	tmp_ops.func_hash = ops->func_hash;
> +	tmp_ops.direct_call = addr;
>  
>  	err = register_ftrace_function_nolock(&tmp_ops);
>  	if (err)
> @@ -5466,6 +5467,7 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
>  			entry->direct = addr;
>  		}
>  	}
> +	ops->direct_call = addr;

AFAICT we don't synchronize threads when installing the tmp_ops, so IIUC on
arches with call_ops, there could be a a thread in the middle of ftrace_caller
which has loaded the ops pointer from the patch-site, but hasn't yet loaded the
ops::direct_func pointer, and could race with this assignment.

Given that, I think this needs to be:

	WRITE_ONCE(ops->direct_call, addr);

... in order to avoid the risk of the store being torn, and the ftrace_caller
trampoline loading a corrupted pointer.

Other than that, this looks good to me!

Thanks,
Mark.

>  
>  	mutex_unlock(&ftrace_lock);
>  
> -- 
> 2.39.1.519.gcb327c4b5f-goog
>

Florent Revest Feb. 2, 2023, 5:41 p.m. UTC | #2

On Thu, Feb 2, 2023 at 4:30 PM Mark Rutland <mark.rutland@arm.com> wrote:
>
> On Wed, Feb 01, 2023 at 05:34:16PM +0100, Florent Revest wrote:
> > All direct calls are now registered using the register_ftrace_direct API
> > so each ops can jump to only one direct-called trampoline.
> >
> > By storing the direct called trampoline address directly in the ops we
> > can save one hashmap lookup in the direct call ops and implement arm64
> > direct calls on top of call ops.
> >
> > Signed-off-by: Florent Revest <revest@chromium.org>
> > ---
> >  include/linux/ftrace.h | 3 +++
> >  kernel/trace/ftrace.c  | 6 ++++--
> >  2 files changed, 7 insertions(+), 2 deletions(-)
> >
> > diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
> > index a7dbd307c3a4..84f717f8959e 100644
> > --- a/include/linux/ftrace.h
> > +++ b/include/linux/ftrace.h
> > @@ -321,6 +321,9 @@ struct ftrace_ops {
> >       unsigned long                   trampoline_size;
> >       struct list_head                list;
> >       ftrace_ops_func_t               ops_func;
> > +#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
> > +     unsigned long                   direct_call;
> > +#endif
> >  #endif
> >  };
> >
> > diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
> > index cb77a0a208c7..b0426de11c45 100644
> > --- a/kernel/trace/ftrace.c
> > +++ b/kernel/trace/ftrace.c
> > @@ -2577,9 +2577,8 @@ ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
> >  static void call_direct_funcs(unsigned long ip, unsigned long pip,
> >                             struct ftrace_ops *ops, struct ftrace_regs *fregs)
> >  {
> > -     unsigned long addr;
> > +     unsigned long addr = ops->direct_call;
> >
> > -     addr = ftrace_find_rec_direct(ip);
> >       if (!addr)
> >               return;
> >
> > @@ -5375,6 +5374,7 @@ int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
> >       ops->func = call_direct_funcs;
> >       ops->flags = MULTI_FLAGS;
> >       ops->trampoline = FTRACE_REGS_ADDR;
> > +     ops->direct_call = addr;
> >
> >       err = register_ftrace_function_nolock(ops);
> >
> > @@ -5445,6 +5445,7 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
> >       /* Enable the tmp_ops to have the same functions as the direct ops */
> >       ftrace_ops_init(&tmp_ops);
> >       tmp_ops.func_hash = ops->func_hash;
> > +     tmp_ops.direct_call = addr;
> >
> >       err = register_ftrace_function_nolock(&tmp_ops);
> >       if (err)
> > @@ -5466,6 +5467,7 @@ __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
> >                       entry->direct = addr;
> >               }
> >       }
> > +     ops->direct_call = addr;
>
> AFAICT we don't synchronize threads when installing the tmp_ops, so IIUC on
> arches with call_ops, there could be a a thread in the middle of ftrace_caller
> which has loaded the ops pointer from the patch-site, but hasn't yet loaded the
> ops::direct_func pointer, and could race with this assignment.
>
> Given that, I think this needs to be:
>
>         WRITE_ONCE(ops->direct_call, addr);
>
> ... in order to avoid the risk of the store being torn, and the ftrace_caller
> trampoline loading a corrupted pointer.

Good point, I'll do that in v2.

diff mbox series

Patch

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
index a7dbd307c3a4..84f717f8959e 100644
--- a/include/linux/ftrace.h
+++ b/include/linux/ftrace.h
@@ -321,6 +321,9 @@  struct ftrace_ops {
 	unsigned long			trampoline_size;
 	struct list_head		list;
 	ftrace_ops_func_t		ops_func;
+#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
+	unsigned long			direct_call;
+#endif
 #endif
 };
 
diff --git a/kernel/trace/ftrace.c b/kernel/trace/ftrace.c
index cb77a0a208c7..b0426de11c45 100644
--- a/kernel/trace/ftrace.c
+++ b/kernel/trace/ftrace.c
@@ -2577,9 +2577,8 @@  ftrace_add_rec_direct(unsigned long ip, unsigned long addr,
 static void call_direct_funcs(unsigned long ip, unsigned long pip,
 			      struct ftrace_ops *ops, struct ftrace_regs *fregs)
 {
-	unsigned long addr;
+	unsigned long addr = ops->direct_call;
 
-	addr = ftrace_find_rec_direct(ip);
 	if (!addr)
 		return;
 
@@ -5375,6 +5374,7 @@  int register_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
 	ops->func = call_direct_funcs;
 	ops->flags = MULTI_FLAGS;
 	ops->trampoline = FTRACE_REGS_ADDR;
+	ops->direct_call = addr;
 
 	err = register_ftrace_function_nolock(ops);
 
@@ -5445,6 +5445,7 @@  __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
 	/* Enable the tmp_ops to have the same functions as the direct ops */
 	ftrace_ops_init(&tmp_ops);
 	tmp_ops.func_hash = ops->func_hash;
+	tmp_ops.direct_call = addr;
 
 	err = register_ftrace_function_nolock(&tmp_ops);
 	if (err)
@@ -5466,6 +5467,7 @@  __modify_ftrace_direct(struct ftrace_ops *ops, unsigned long addr)
 			entry->direct = addr;
 		}
 	}
+	ops->direct_call = addr;
 
 	mutex_unlock(&ftrace_lock);