@@ -3,7 +3,7 @@
#define _ASM_ARM_PARAVIRT_H
#ifdef CONFIG_PARAVIRT
-#include <linux/static_call_types.h>
+#include <linux/static_call.h>
struct static_key;
extern struct static_key paravirt_steal_enabled;
@@ -3,7 +3,7 @@
#define _ASM_ARM64_PARAVIRT_H
#ifdef CONFIG_PARAVIRT
-#include <linux/static_call_types.h>
+#include <linux/static_call.h>
struct static_key;
extern struct static_key paravirt_steal_enabled;
@@ -15,7 +15,7 @@
#include <linux/bug.h>
#include <linux/types.h>
#include <linux/cpumask.h>
-#include <linux/static_call_types.h>
+#include <linux/static_call.h>
#include <asm/frame.h>
u64 dummy_steal_clock(int cpu);
@@ -7,7 +7,7 @@
#include <asm/current.h>
#include <linux/thread_info.h>
-#include <linux/static_call_types.h>
+#include <linux/static_call.h>
/* We use the MSB mostly because its available */
#define PREEMPT_NEED_RESCHED 0x80000000
@@ -2,7 +2,8 @@
#ifndef _ASM_STATIC_CALL_H
#define _ASM_STATIC_CALL_H
-#include <asm/text-patching.h>
+#include <linux/objtool.h>
+#include <linux/static_call_types.h>
/*
* For CONFIG_HAVE_STATIC_CALL_INLINE, this is a temporary trampoline which
@@ -33,6 +33,7 @@
#include <asm/tlb.h>
#include <asm/io_bitmap.h>
#include <asm/gsseg.h>
+#include <asm/text-patching.h>
/*
* nop stub, which must not clobber anything *including the stack* to
@@ -2,7 +2,7 @@
#ifndef __LINUX_ENTRYCOMMON_H
#define __LINUX_ENTRYCOMMON_H
-#include <linux/static_call_types.h>
+#include <linux/static_call.h>
#include <linux/ptrace.h>
#include <linux/syscalls.h>
#include <linux/seccomp.h>
@@ -2,7 +2,7 @@
#ifndef __LINUX_ENTRYKVM_H
#define __LINUX_ENTRYKVM_H
-#include <linux/static_call_types.h>
+#include <linux/static_call.h>
#include <linux/resume_user_mode.h>
#include <linux/syscalls.h>
#include <linux/seccomp.h>
@@ -28,7 +28,7 @@
#include <linux/panic.h>
#include <linux/printk.h>
#include <linux/build_bug.h>
-#include <linux/static_call_types.h>
+#include <linux/static_call.h>
#include <linux/instruction_pointer.h>
#include <asm/byteorder.h>
@@ -26,7 +26,7 @@
#include <linux/error-injection.h>
#include <linux/tracepoint-defs.h>
#include <linux/srcu.h>
-#include <linux/static_call_types.h>
+#include <linux/static_call.h>
#include <linux/percpu.h>
#include <asm/module.h>
@@ -134,178 +134,141 @@
#include <linux/types.h>
#include <linux/static_call_types.h>
-#ifdef CONFIG_HAVE_STATIC_CALL
-#include <asm/static_call.h>
-
-/*
- * Either @site or @tramp can be NULL.
- */
-extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);
-
-#define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
-
-#else
-#define STATIC_CALL_TRAMP_ADDR(name) NULL
-#endif
-
-#define static_call_update(name, func) \
-({ \
- typeof(&STATIC_CALL_TRAMP(name)) __F = (func); \
- __static_call_update(&STATIC_CALL_KEY(name), \
- STATIC_CALL_TRAMP_ADDR(name), __F); \
-})
-
-#define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func))
-
+struct static_call_mods;
+struct static_call_key {
+ void *func;
#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
-
-extern int __init static_call_init(void);
-
-extern void static_call_force_reinit(void);
-
-struct static_call_mod {
- struct static_call_mod *next;
- struct module *mod; /* for vmlinux, mod == NULL */
- struct static_call_site *sites;
+ union {
+ /* bit 0: 0 = sites, 1 = mods */
+ unsigned long type;
+ struct static_call_site *_sites;
+ struct static_call_mod *_mods;
+ };
+#endif
};
-/* For finding the key associated with a trampoline */
-struct static_call_tramp_key {
- s32 tramp;
- s32 key;
-};
+#define DECLARE_STATIC_CALL(name, func) \
+ extern struct static_call_key STATIC_CALL_KEY(name); \
+ extern typeof(func) STATIC_CALL_TRAMP(name);
-extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
-extern int static_call_text_reserved(void *start, void *end);
-
-extern long __static_call_return0(void);
-
-#define DEFINE_STATIC_CALL(name, _func) \
- DECLARE_STATIC_CALL(name, _func); \
+#define __DEFINE_STATIC_CALL(name, type, _func) \
+ DECLARE_STATIC_CALL(name, type); \
struct static_call_key STATIC_CALL_KEY(name) = { \
.func = _func, \
- }; \
- ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
+ }
-#define DEFINE_STATIC_CALL_NULL(name, _func) \
- DECLARE_STATIC_CALL(name, _func); \
- struct static_call_key STATIC_CALL_KEY(name) = { \
- .func = NULL, \
- }; \
- ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
+#define DEFINE_STATIC_CALL(name, func) \
+ __DEFINE_STATIC_CALL(name, func, func); \
+ __DEFINE_STATIC_CALL_TRAMP(name, func)
-#define DEFINE_STATIC_CALL_RET0(name, _func) \
- DECLARE_STATIC_CALL(name, _func); \
- struct static_call_key STATIC_CALL_KEY(name) = { \
- .func = __static_call_return0, \
- }; \
- ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)
+#define DEFINE_STATIC_CALL_NULL(name, type) \
+ __DEFINE_STATIC_CALL(name, type, NULL); \
+ __DEFINE_STATIC_CALL_NULL_TRAMP(name)
-#define static_call_cond(name) (void)__static_call(name)
+#define DEFINE_STATIC_CALL_RET0(name, type) \
+ __DEFINE_STATIC_CALL(name, type, __static_call_return0); \
+ __DEFINE_STATIC_CALL_RET0_TRAMP(name)
#define EXPORT_STATIC_CALL(name) \
EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
- EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
+ __EXPORT_STATIC_CALL_TRAMP(name)
#define EXPORT_STATIC_CALL_GPL(name) \
EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
- EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
+ __EXPORT_STATIC_CALL_TRAMP_GPL(name)
/*
* Read-only exports: export the trampoline but not the key, so modules can't
* change call targets.
+ *
+ * These are called via static_call_ro().
*/
#define EXPORT_STATIC_CALL_RO(name) \
- EXPORT_SYMBOL(STATIC_CALL_TRAMP(name)); \
+ __EXPORT_STATIC_CALL_TRAMP(name); \
__STATIC_CALL_ADD_TRAMP_KEY(name)
-#define EXPORT_STATIC_CALL_RO_GPL(name) \
- EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name)); \
+#define EXPORT_STATIC_CALL_RO_GPL(name) \
+ __EXPORT_STATIC_CALL_TRAMP_GPL(name); \
__STATIC_CALL_ADD_TRAMP_KEY(name)
-/* Unexported key lookup table */
-#define __STATIC_CALL_ADD_TRAMP_KEY(name) \
- asm(".pushsection .static_call_tramp_key, \"a\" \n" \
- ".long " STATIC_CALL_TRAMP_STR(name) " - . \n" \
- ".long " STATIC_CALL_KEY_STR(name) " - . \n" \
- ".popsection \n")
+/*
+ * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from
+ * the symbol table so that objtool can reference it when it generates the
+ * .static_call_sites section.
+ */
+#define __STATIC_CALL_ADDRESSABLE(name) __ADDRESSABLE(STATIC_CALL_KEY(name))
-#elif defined(CONFIG_HAVE_STATIC_CALL)
+#define static_call(name) \
+({ \
+ __STATIC_CALL_ADDRESSABLE(name); \
+ __static_call(name); \
+})
-static inline int static_call_init(void) { return 0; }
+#define static_call_cond(name) (void)__static_call_cond(name)
-#define DEFINE_STATIC_CALL(name, _func) \
- DECLARE_STATIC_CALL(name, _func); \
- struct static_call_key STATIC_CALL_KEY(name) = { \
- .func = _func, \
- }; \
- ARCH_DEFINE_STATIC_CALL_TRAMP(name, _func)
+/* Use static_call_ro() to call a read-only-exported static call. */
+#define static_call_ro(name) __static_call_ro(name)
-#define DEFINE_STATIC_CALL_NULL(name, _func) \
- DECLARE_STATIC_CALL(name, _func); \
- struct static_call_key STATIC_CALL_KEY(name) = { \
- .func = NULL, \
- }; \
- ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
+#if defined(MODULE) || !defined(CONFIG_HAVE_STATIC_CALL_INLINE)
+#define __STATIC_CALL_RO_ADDRESSABLE(name)
+#define __static_call_ro(name) __static_call(name)
+#else
+#define __STATIC_CALL_RO_ADDRESSABLE(name) __STATIC_CALL_ADDRESSABLE(name)
+#define __static_call_ro(name) static_call(name)
+#endif
-#define DEFINE_STATIC_CALL_RET0(name, _func) \
- DECLARE_STATIC_CALL(name, _func); \
- struct static_call_key STATIC_CALL_KEY(name) = { \
- .func = __static_call_return0, \
- }; \
- ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)
+#define static_call_update(name, func) \
+({ \
+ typeof(&STATIC_CALL_TRAMP(name)) __F = (func); \
+ __static_call_update(&STATIC_CALL_KEY(name), \
+ STATIC_CALL_TRAMP_ADDR(name), __F); \
+})
-#define static_call_cond(name) (void)__static_call(name)
+#define static_call_query(name) (READ_ONCE(STATIC_CALL_KEY(name).func))
-extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
-static inline int static_call_text_reserved(void *start, void *end)
-{
- return 0;
-}
+#ifdef CONFIG_HAVE_STATIC_CALL
-extern long __static_call_return0(void);
+#include <asm/static_call.h>
-#define EXPORT_STATIC_CALL(name) \
- EXPORT_SYMBOL(STATIC_CALL_KEY(name)); \
- EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
-#define EXPORT_STATIC_CALL_GPL(name) \
- EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name)); \
- EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
+#define __DEFINE_STATIC_CALL_TRAMP(name, func) \
+ ARCH_DEFINE_STATIC_CALL_TRAMP(name, func)
-/*
- * Read-only exports: export the trampoline but not the key, so modules can't
- * change call targets.
- */
-#define EXPORT_STATIC_CALL_RO(name) \
+#define __DEFINE_STATIC_CALL_NULL_TRAMP(name) \
+ ARCH_DEFINE_STATIC_CALL_NULL_TRAMP(name)
+
+#define __DEFINE_STATIC_CALL_RET0_TRAMP(name) \
+ ARCH_DEFINE_STATIC_CALL_RET0_TRAMP(name)
+
+#define __EXPORT_STATIC_CALL_TRAMP(name) \
EXPORT_SYMBOL(STATIC_CALL_TRAMP(name))
-#define EXPORT_STATIC_CALL_RO_GPL(name) \
+
+#define __EXPORT_STATIC_CALL_TRAMP_GPL(name) \
EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(name))
-#else /* Generic implementation */
+#define __static_call(name) (&STATIC_CALL_TRAMP(name))
+#define __static_call_cond __static_call
-static inline int static_call_init(void) { return 0; }
+#define STATIC_CALL_TRAMP_ADDR(name) &STATIC_CALL_TRAMP(name)
-static inline long __static_call_return0(void)
-{
- return 0;
-}
+extern long __static_call_return0(void);
+extern void __static_call_update(struct static_call_key *key, void *tramp, void *func);
-#define __DEFINE_STATIC_CALL(name, _func, _func_init) \
- DECLARE_STATIC_CALL(name, _func); \
- struct static_call_key STATIC_CALL_KEY(name) = { \
- .func = _func_init, \
- }
+/*
+ * Either @site or @tramp can be NULL.
+ */
+extern void arch_static_call_transform(void *site, void *tramp, void *func, bool tail);
-#define DEFINE_STATIC_CALL(name, _func) \
- __DEFINE_STATIC_CALL(name, _func, _func)
+#else /* !CONFIG_HAVE_STATIC_CALL */
-#define DEFINE_STATIC_CALL_NULL(name, _func) \
- __DEFINE_STATIC_CALL(name, _func, NULL)
+#define __DEFINE_STATIC_CALL_TRAMP(name, func)
+#define __DEFINE_STATIC_CALL_NULL_TRAMP(name)
+#define __DEFINE_STATIC_CALL_RET0_TRAMP(name)
+#define __EXPORT_STATIC_CALL_TRAMP(name)
+#define __EXPORT_STATIC_CALL_TRAMP_GPL(name)
-#define DEFINE_STATIC_CALL_RET0(name, _func) \
- __DEFINE_STATIC_CALL(name, _func, __static_call_return0)
+#define __static_call(name) \
+ ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func))
static inline void __static_call_nop(void) { }
-
/*
* This horrific hack takes care of two things:
*
@@ -326,7 +289,9 @@ static inline void __static_call_nop(void) { }
(typeof(STATIC_CALL_TRAMP(name))*)func; \
})
-#define static_call_cond(name) (void)__static_call_cond(name)
+#define STATIC_CALL_TRAMP_ADDR(name) NULL
+
+static inline long __static_call_return0(void) { return 0; }
static inline
void __static_call_update(struct static_call_key *key, void *tramp, void *func)
@@ -334,14 +299,29 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func)
WRITE_ONCE(key->func, func);
}
-static inline int static_call_text_reserved(void *start, void *end)
-{
- return 0;
-}
+#endif /* CONFIG_HAVE_STATIC_CALL */
-#define EXPORT_STATIC_CALL(name) EXPORT_SYMBOL(STATIC_CALL_KEY(name))
-#define EXPORT_STATIC_CALL_GPL(name) EXPORT_SYMBOL_GPL(STATIC_CALL_KEY(name))
-#endif /* CONFIG_HAVE_STATIC_CALL */
+#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
+
+/* Unexported key lookup table */
+#define __STATIC_CALL_ADD_TRAMP_KEY(name) \
+ asm(".pushsection .static_call_tramp_key, \"a\" \n" \
+ ".long " STATIC_CALL_TRAMP_STR(name) " - . \n" \
+ ".long " STATIC_CALL_KEY_STR(name) " - . \n" \
+ ".popsection \n")
+
+extern int static_call_init(void);
+extern int static_call_text_reserved(void *start, void *end);
+extern void static_call_force_reinit(void);
+
+#else /* !CONFIG_HAVE_STATIC_CALL_INLINE*/
+
+#define __STATIC_CALL_ADD_TRAMP_KEY(name)
+static inline int static_call_init(void) { return 0; }
+static inline int static_call_text_reserved(void *start, void *end) { return 0; }
+static inline void static_call_force_reinit(void) {}
+
+#endif /* CONFIG_HAVE_STATIC_CALL_INLINE */
#endif /* _LINUX_STATIC_CALL_H */
@@ -2,6 +2,10 @@
#ifndef _STATIC_CALL_TYPES_H
#define _STATIC_CALL_TYPES_H
+/*
+ * Static call types for sharing with objtool
+ */
+
#include <linux/types.h>
#include <linux/stringify.h>
#include <linux/compiler.h>
@@ -34,70 +38,4 @@ struct static_call_site {
s32 key;
};
-#define DECLARE_STATIC_CALL(name, func) \
- extern struct static_call_key STATIC_CALL_KEY(name); \
- extern typeof(func) STATIC_CALL_TRAMP(name);
-
-#ifdef CONFIG_HAVE_STATIC_CALL
-
-#define __raw_static_call(name) (&STATIC_CALL_TRAMP(name))
-
-#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
-
-/*
- * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from
- * the symbol table so that objtool can reference it when it generates the
- * .static_call_sites section.
- */
-#define __STATIC_CALL_ADDRESSABLE(name) \
- __ADDRESSABLE(STATIC_CALL_KEY(name))
-
-#define __static_call(name) \
-({ \
- __STATIC_CALL_ADDRESSABLE(name); \
- __raw_static_call(name); \
-})
-
-struct static_call_key {
- void *func;
- union {
- /* bit 0: 0 = sites, 1 = mods */
- unsigned long type;
- struct static_call_site *_sites;
- struct static_call_mod *_mods;
- };
-};
-
-#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */
-
-#define __STATIC_CALL_ADDRESSABLE(name)
-#define __static_call(name) __raw_static_call(name)
-
-struct static_call_key {
- void *func;
-};
-
-#endif /* CONFIG_HAVE_STATIC_CALL_INLINE */
-
-#ifdef MODULE
-#define __STATIC_CALL_RO_ADDRESSABLE(name)
-#define static_call_ro(name) __raw_static_call(name)
-#else
-#define __STATIC_CALL_RO_ADDRESSABLE(name) __STATIC_CALL_ADDRESSABLE(name)
-#define static_call_ro(name) __static_call(name)
-#endif
-
-#define static_call(name) __static_call(name)
-
-#else
-
-struct static_call_key {
- void *func;
-};
-
-#define static_call(name) \
- ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func))
-
-#endif /* CONFIG_HAVE_STATIC_CALL */
-
#endif /* _STATIC_CALL_TYPES_H */
@@ -1,4 +1,5 @@
// SPDX-License-Identifier: GPL-2.0
+#include <linux/export.h>
#include <linux/static_call.h>
#include <linux/cpu.h>
@@ -9,6 +9,19 @@
#include <linux/cpu.h>
#include <linux/processor.h>
#include <asm/sections.h>
+#include <asm/text-patching.h>
+
+/* For finding the key associated with a trampoline */
+struct static_call_tramp_key {
+ s32 tramp;
+ s32 key;
+};
+
+struct static_call_mod {
+ struct static_call_mod *next;
+ struct module *mod; /* for vmlinux, mod == NULL */
+ struct static_call_site *sites;
+};
extern struct static_call_site __start_static_call_sites[],
__stop_static_call_sites[];
@@ -2,6 +2,10 @@
#ifndef _STATIC_CALL_TYPES_H
#define _STATIC_CALL_TYPES_H
+/*
+ * Static call types for sharing with objtool
+ */
+
#include <linux/types.h>
#include <linux/stringify.h>
#include <linux/compiler.h>
@@ -34,70 +38,4 @@ struct static_call_site {
s32 key;
};
-#define DECLARE_STATIC_CALL(name, func) \
- extern struct static_call_key STATIC_CALL_KEY(name); \
- extern typeof(func) STATIC_CALL_TRAMP(name);
-
-#ifdef CONFIG_HAVE_STATIC_CALL
-
-#define __raw_static_call(name) (&STATIC_CALL_TRAMP(name))
-
-#ifdef CONFIG_HAVE_STATIC_CALL_INLINE
-
-/*
- * __ADDRESSABLE() is used to ensure the key symbol doesn't get stripped from
- * the symbol table so that objtool can reference it when it generates the
- * .static_call_sites section.
- */
-#define __STATIC_CALL_ADDRESSABLE(name) \
- __ADDRESSABLE(STATIC_CALL_KEY(name))
-
-#define __static_call(name) \
-({ \
- __STATIC_CALL_ADDRESSABLE(name); \
- __raw_static_call(name); \
-})
-
-struct static_call_key {
- void *func;
- union {
- /* bit 0: 0 = sites, 1 = mods */
- unsigned long type;
- struct static_call_site *_sites;
- struct static_call_mod *_mods;
- };
-};
-
-#else /* !CONFIG_HAVE_STATIC_CALL_INLINE */
-
-#define __STATIC_CALL_ADDRESSABLE(name)
-#define __static_call(name) __raw_static_call(name)
-
-struct static_call_key {
- void *func;
-};
-
-#endif /* CONFIG_HAVE_STATIC_CALL_INLINE */
-
-#ifdef MODULE
-#define __STATIC_CALL_RO_ADDRESSABLE(name)
-#define static_call_ro(name) __raw_static_call(name)
-#else
-#define __STATIC_CALL_RO_ADDRESSABLE(name) __STATIC_CALL_ADDRESSABLE(name)
-#define static_call_ro(name) __static_call(name)
-#endif
-
-#define static_call(name) __static_call(name)
-
-#else
-
-struct static_call_key {
- void *func;
-};
-
-#define static_call(name) \
- ((typeof(STATIC_CALL_TRAMP(name))*)(STATIC_CALL_KEY(name).func))
-
-#endif /* CONFIG_HAVE_STATIC_CALL */
-
#endif /* _STATIC_CALL_TYPES_H */