@@ -7,6 +7,7 @@
#include <linux/ptrace.h>
#include <linux/randomize_kstack.h>
#include <linux/syscalls.h>
+#include <linux/dept.h>
#include <asm/daifflags.h>
#include <asm/debug-monitors.h>
@@ -105,6 +106,7 @@ static void el0_svc_common(struct pt_regs *regs, int scno, int sc_nr,
*/
local_daif_restore(DAIF_PROCCTX);
+ dept_kernel_enter();
if (flags & _TIF_MTE_ASYNC_FAULT) {
/*
@@ -19,6 +19,7 @@
#include <linux/nospec.h>
#include <linux/syscalls.h>
#include <linux/uaccess.h>
+#include <linux/dept.h>
#ifdef CONFIG_XEN_PV
#include <xen/xen-ops.h>
@@ -72,6 +73,7 @@ static __always_inline bool do_syscall_x32(struct pt_regs *regs, int nr)
__visible noinstr void do_syscall_64(struct pt_regs *regs, int nr)
{
+ dept_kernel_enter();
add_random_kstack_offset();
nr = syscall_enter_from_user_mode(regs, nr);
@@ -120,6 +122,7 @@ __visible noinstr void do_int80_syscall_32(struct pt_regs *regs)
{
int nr = syscall_32_enter(regs);
+ dept_kernel_enter();
add_random_kstack_offset();
/*
* Subtlety here: if ptrace pokes something larger than 2^31-1 into
@@ -140,6 +143,7 @@ static noinstr bool __do_fast_syscall_32(struct pt_regs *regs)
int nr = syscall_32_enter(regs);
int res;
+ dept_kernel_enter();
add_random_kstack_offset();
/*
* This cannot use syscall_enter_from_user_mode() as it has to
@@ -25,11 +25,16 @@ struct task_struct;
#define DEPT_MAX_SUBCLASSES_USR (DEPT_MAX_SUBCLASSES / DEPT_MAX_SUBCLASSES_EVT)
#define DEPT_MAX_SUBCLASSES_CACHE 2
-#define DEPT_SIRQ 0
-#define DEPT_HIRQ 1
-#define DEPT_IRQS_NR 2
-#define DEPT_SIRQF (1UL << DEPT_SIRQ)
-#define DEPT_HIRQF (1UL << DEPT_HIRQ)
+enum {
+ DEPT_CXT_SIRQ = 0,
+ DEPT_CXT_HIRQ,
+ DEPT_CXT_IRQS_NR,
+ DEPT_CXT_PROCESS = DEPT_CXT_IRQS_NR,
+ DEPT_CXTS_NR
+};
+
+#define DEPT_SIRQF (1UL << DEPT_CXT_SIRQ)
+#define DEPT_HIRQF (1UL << DEPT_CXT_HIRQ)
struct dept_ecxt;
struct dept_iecxt {
@@ -94,8 +99,8 @@ struct dept_class {
/*
* for tracking IRQ dependencies
*/
- struct dept_iecxt iecxt[DEPT_IRQS_NR];
- struct dept_iwait iwait[DEPT_IRQS_NR];
+ struct dept_iecxt iecxt[DEPT_CXT_IRQS_NR];
+ struct dept_iwait iwait[DEPT_CXT_IRQS_NR];
/*
* classified by a map embedded in task_struct,
@@ -207,8 +212,8 @@ struct dept_ecxt {
/*
* where the IRQ-enabled happened
*/
- unsigned long enirq_ip[DEPT_IRQS_NR];
- struct dept_stack *enirq_stack[DEPT_IRQS_NR];
+ unsigned long enirq_ip[DEPT_CXT_IRQS_NR];
+ struct dept_stack *enirq_stack[DEPT_CXT_IRQS_NR];
/*
* where the event context started
@@ -252,8 +257,8 @@ struct dept_wait {
/*
* where the IRQ wait happened
*/
- unsigned long irq_ip[DEPT_IRQS_NR];
- struct dept_stack *irq_stack[DEPT_IRQS_NR];
+ unsigned long irq_ip[DEPT_CXT_IRQS_NR];
+ struct dept_stack *irq_stack[DEPT_CXT_IRQS_NR];
/*
* where the wait happened
@@ -406,19 +411,19 @@ struct dept_task {
int wait_hist_pos;
/*
- * sequential id to identify each IRQ context
+ * sequential id to identify each context
*/
- unsigned int irq_id[DEPT_IRQS_NR];
+ unsigned int cxt_id[DEPT_CXTS_NR];
/*
* for tracking IRQ-enabled points with cross-event
*/
- unsigned int wgen_enirq[DEPT_IRQS_NR];
+ unsigned int wgen_enirq[DEPT_CXT_IRQS_NR];
/*
* for keeping up-to-date IRQ-enabled points
*/
- unsigned long enirq_ip[DEPT_IRQS_NR];
+ unsigned long enirq_ip[DEPT_CXT_IRQS_NR];
/*
* current effective IRQ-enabled flag
@@ -470,7 +475,7 @@ struct dept_task {
.wait_hist = { { .wait = NULL, } }, \
.ecxt_held_pos = 0, \
.wait_hist_pos = 0, \
- .irq_id = { 0U }, \
+ .cxt_id = { 0U }, \
.wgen_enirq = { 0U }, \
.enirq_ip = { 0UL }, \
.eff_enirqf = 0UL, \
@@ -509,6 +514,7 @@ extern void dept_event(struct dept_map *m, unsigned long e_f, unsigned long ip,
extern void dept_ecxt_exit(struct dept_map *m, unsigned long e_f, unsigned long ip);
extern void dept_sched_enter(void);
extern void dept_sched_exit(void);
+extern void dept_kernel_enter(void);
static inline void dept_ecxt_enter_nokeep(struct dept_map *m)
{
@@ -560,6 +566,7 @@ struct dept_task { };
#define dept_ecxt_exit(m, e_f, ip) do { } while (0)
#define dept_sched_enter() do { } while (0)
#define dept_sched_exit() do { } while (0)
+#define dept_kernel_enter() do { } while (0)
#define dept_ecxt_enter_nokeep(m) do { } while (0)
#define dept_key_init(k) do { (void)(k); } while (0)
#define dept_key_destroy(k) do { (void)(k); } while (0)
@@ -221,9 +221,9 @@ static inline struct dept_class *dep_tc(struct dept_dep *d)
static inline const char *irq_str(int irq)
{
- if (irq == DEPT_SIRQ)
+ if (irq == DEPT_CXT_SIRQ)
return "softirq";
- if (irq == DEPT_HIRQ)
+ if (irq == DEPT_CXT_HIRQ)
return "hardirq";
return "(unknown)";
}
@@ -407,7 +407,7 @@ static void initialize_class(struct dept_class *c)
{
int i;
- for (i = 0; i < DEPT_IRQS_NR; i++) {
+ for (i = 0; i < DEPT_CXT_IRQS_NR; i++) {
struct dept_iecxt *ie = &c->iecxt[i];
struct dept_iwait *iw = &c->iwait[i];
@@ -432,7 +432,7 @@ static void initialize_ecxt(struct dept_ecxt *e)
{
int i;
- for (i = 0; i < DEPT_IRQS_NR; i++) {
+ for (i = 0; i < DEPT_CXT_IRQS_NR; i++) {
e->enirq_stack[i] = NULL;
e->enirq_ip[i] = 0UL;
}
@@ -448,7 +448,7 @@ static void initialize_wait(struct dept_wait *w)
{
int i;
- for (i = 0; i < DEPT_IRQS_NR; i++) {
+ for (i = 0; i < DEPT_CXT_IRQS_NR; i++) {
w->irq_stack[i] = NULL;
w->irq_ip[i] = 0UL;
}
@@ -487,7 +487,7 @@ static void destroy_ecxt(struct dept_ecxt *e)
{
int i;
- for (i = 0; i < DEPT_IRQS_NR; i++)
+ for (i = 0; i < DEPT_CXT_IRQS_NR; i++)
if (e->enirq_stack[i])
put_stack(e->enirq_stack[i]);
if (e->class)
@@ -503,7 +503,7 @@ static void destroy_wait(struct dept_wait *w)
{
int i;
- for (i = 0; i < DEPT_IRQS_NR; i++)
+ for (i = 0; i < DEPT_CXT_IRQS_NR; i++)
if (w->irq_stack[i])
put_stack(w->irq_stack[i]);
if (w->class)
@@ -652,7 +652,7 @@ static void print_diagram(struct dept_dep *d)
const char *tc_n = tc->sched_map ? "<sched>" : (tc->name ?: "(unknown)");
irqf = e->enirqf & w->irqf;
- for_each_set_bit(irq, &irqf, DEPT_IRQS_NR) {
+ for_each_set_bit(irq, &irqf, DEPT_CXT_IRQS_NR) {
if (!firstline)
pr_warn("\nor\n\n");
firstline = false;
@@ -685,7 +685,7 @@ static void print_dep(struct dept_dep *d)
const char *tc_n = tc->sched_map ? "<sched>" : (tc->name ?: "(unknown)");
irqf = e->enirqf & w->irqf;
- for_each_set_bit(irq, &irqf, DEPT_IRQS_NR) {
+ for_each_set_bit(irq, &irqf, DEPT_CXT_IRQS_NR) {
pr_warn("%s has been enabled:\n", irq_str(irq));
print_ip_stack(e->enirq_ip[irq], e->enirq_stack[irq]);
pr_warn("\n");
@@ -911,7 +911,7 @@ static void bfs(struct dept_class *c, bfs_f *cb, void *in, void **out)
*/
static inline unsigned long cur_enirqf(void);
-static inline int cur_irq(void);
+static inline int cur_cxt(void);
static inline unsigned int cur_ctxt_id(void);
static inline struct dept_iecxt *iecxt(struct dept_class *c, int irq)
@@ -1459,7 +1459,7 @@ static void add_dep(struct dept_ecxt *e, struct dept_wait *w)
if (d) {
check_dl_bfs(d);
- for (i = 0; i < DEPT_IRQS_NR; i++) {
+ for (i = 0; i < DEPT_CXT_IRQS_NR; i++) {
struct dept_iwait *fiw = iwait(fc, i);
struct dept_iecxt *found_ie;
struct dept_iwait *found_iw;
@@ -1495,7 +1495,7 @@ static void add_wait(struct dept_class *c, unsigned long ip,
struct dept_task *dt = dept_task();
struct dept_wait *w;
unsigned int wg = 0U;
- int irq;
+ int cxt;
int i;
if (DEPT_WARN_ON(!valid_class(c)))
@@ -1511,9 +1511,9 @@ static void add_wait(struct dept_class *c, unsigned long ip,
w->wait_stack = get_current_stack();
w->sched_sleep = sched_sleep;
- irq = cur_irq();
- if (irq < DEPT_IRQS_NR)
- add_iwait(c, irq, w);
+ cxt = cur_cxt();
+ if (cxt == DEPT_CXT_HIRQ || cxt == DEPT_CXT_SIRQ)
+ add_iwait(c, cxt, w);
/*
* Avoid adding dependency between user aware nested ecxt and
@@ -1594,7 +1594,7 @@ static bool add_ecxt(struct dept_map *m, struct dept_class *c,
eh->sub_l = sub_l;
irqf = cur_enirqf();
- for_each_set_bit(irq, &irqf, DEPT_IRQS_NR)
+ for_each_set_bit(irq, &irqf, DEPT_CXT_IRQS_NR)
add_iecxt(c, irq, e, false);
del_ecxt(e);
@@ -1746,7 +1746,7 @@ static void do_event(struct dept_map *m, struct dept_class *c,
add_dep(eh->ecxt, wh->wait);
}
- for (i = 0; i < DEPT_IRQS_NR; i++) {
+ for (i = 0; i < DEPT_CXT_IRQS_NR; i++) {
struct dept_ecxt *e;
if (before(dt->wgen_enirq[i], wg))
@@ -1788,7 +1788,7 @@ static void disconnect_class(struct dept_class *c)
call_rcu(&d->rh, del_dep_rcu);
}
- for (i = 0; i < DEPT_IRQS_NR; i++) {
+ for (i = 0; i < DEPT_CXT_IRQS_NR; i++) {
stale_iecxt(iecxt(c, i));
stale_iwait(iwait(c, i));
}
@@ -1813,27 +1813,21 @@ static inline unsigned long cur_enirqf(void)
return 0UL;
}
-static inline int cur_irq(void)
+static inline int cur_cxt(void)
{
if (lockdep_softirq_context(current))
- return DEPT_SIRQ;
+ return DEPT_CXT_SIRQ;
if (lockdep_hardirq_context())
- return DEPT_HIRQ;
- return DEPT_IRQS_NR;
+ return DEPT_CXT_HIRQ;
+ return DEPT_CXT_PROCESS;
}
static inline unsigned int cur_ctxt_id(void)
{
struct dept_task *dt = dept_task();
- int irq = cur_irq();
+ int cxt = cur_cxt();
- /*
- * Normal process context
- */
- if (irq == DEPT_IRQS_NR)
- return 0U;
-
- return dt->irq_id[irq] | (1UL << irq);
+ return dt->cxt_id[cxt] | (1UL << cxt);
}
static void enirq_transition(int irq)
@@ -1884,7 +1878,7 @@ static void enirq_update(unsigned long ip)
/*
* Do enirq_transition() only on an OFF -> ON transition.
*/
- for_each_set_bit(irq, &irqf, DEPT_IRQS_NR) {
+ for_each_set_bit(irq, &irqf, DEPT_CXT_IRQS_NR) {
if (prev & (1UL << irq))
continue;
@@ -1983,6 +1977,13 @@ void dept_hardirqs_off_ip(unsigned long ip)
}
EXPORT_SYMBOL_GPL(dept_hardirqs_off_ip);
+void dept_kernel_enter(void)
+{
+ struct dept_task *dt = dept_task();
+
+ dt->cxt_id[DEPT_CXT_PROCESS] += 1UL << DEPT_CXTS_NR;
+}
+
/*
* Ensure it's the outmost softirq context.
*/
@@ -1990,7 +1991,7 @@ void dept_softirq_enter(void)
{
struct dept_task *dt = dept_task();
- dt->irq_id[DEPT_SIRQ] += 1UL << DEPT_IRQS_NR;
+ dt->cxt_id[DEPT_CXT_SIRQ] += 1UL << DEPT_CXTS_NR;
}
/*
@@ -2000,7 +2001,7 @@ void dept_hardirq_enter(void)
{
struct dept_task *dt = dept_task();
- dt->irq_id[DEPT_HIRQ] += 1UL << DEPT_IRQS_NR;
+ dt->cxt_id[DEPT_CXT_HIRQ] += 1UL << DEPT_CXTS_NR;
}
void dept_sched_enter(void)