Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
---
include/linux/sched.h | 2 --
kernel/sched/core.c | 17 +++++++----------
kernel/sched/fair.c | 29 ++++++++++++-----------------
kernel/sched/sched.h | 2 +-
4 files changed, 20 insertions(+), 30 deletions(-)
@@ -573,8 +573,6 @@ struct sched_entity {
/* cached value of my_q->h_nr_running */
unsigned long runnable_weight;
#endif
- /* preemption offset in ns */
- long latency_offset;
#ifdef CONFIG_SMP
/*
@@ -1285,9 +1285,10 @@ static void set_load_weight(struct task_
}
}
-static void set_latency_offset(struct task_struct *p)
+static inline void set_latency_prio(struct task_struct *p, int prio)
{
- p->se.latency_offset = calc_latency_offset(p->latency_prio - MAX_RT_PRIO);
+ p->latency_prio = prio;
+ set_latency_fair(&p->se, prio - MAX_RT_PRIO);
}
#ifdef CONFIG_UCLAMP_TASK
@@ -4442,7 +4443,7 @@ static void __sched_fork(unsigned long c
p->se.vlag = 0;
INIT_LIST_HEAD(&p->se.group_node);
- set_latency_offset(p);
+ set_latency_prio(p, p->latency_prio);
#ifdef CONFIG_FAIR_GROUP_SCHED
p->se.cfs_rq = NULL;
@@ -4694,9 +4695,7 @@ int sched_fork(unsigned long clone_flags
p->prio = p->normal_prio = p->static_prio;
set_load_weight(p, false);
-
- p->latency_prio = NICE_TO_PRIO(0);
- set_latency_offset(p);
+ set_latency_prio(p, NICE_TO_PRIO(0));
/*
* We don't need the reset flag anymore after the fork. It has
@@ -7469,10 +7468,8 @@ static void __setscheduler_params(struct
static void __setscheduler_latency(struct task_struct *p,
const struct sched_attr *attr)
{
- if (attr->sched_flags & SCHED_FLAG_LATENCY_NICE) {
- p->latency_prio = NICE_TO_PRIO(attr->sched_latency_nice);
- set_latency_offset(p);
- }
+ if (attr->sched_flags & SCHED_FLAG_LATENCY_NICE)
+ set_latency_prio(p, NICE_TO_PRIO(attr->sched_latency_nice));
}
/*
@@ -919,12 +919,19 @@ int sched_update_scaling(void)
}
#endif
-long calc_latency_offset(int prio)
+void set_latency_fair(struct sched_entity *se, int prio)
{
u32 weight = sched_prio_to_weight[prio];
u64 base = sysctl_sched_base_slice;
- return div_u64(base << SCHED_FIXEDPOINT_SHIFT, weight);
+ /*
+ * For EEVDF the virtual time slope is determined by w_i (iow.
+ * nice) while the request time r_i is determined by
+ * latency-nice.
+ *
+ * Smaller request gets better latency.
+ */
+ se->slice = div_u64(base << SCHED_FIXEDPOINT_SHIFT, weight);
}
/*
@@ -937,13 +944,6 @@ static void update_deadline(struct cfs_r
return;
/*
- * For EEVDF the virtual time slope is determined by w_i (iow.
- * nice) while the request time r_i is determined by
- * latency-nice.
- */
- se->slice = se->latency_offset;
-
- /*
* EEVDF: vd_i = ve_i + r_i / w_i
*/
se->deadline = se->vruntime + calc_delta_fair(se->slice, se);
@@ -12231,7 +12231,7 @@ void init_tg_cfs_entry(struct task_group
se->my_q = cfs_rq;
- se->latency_offset = calc_latency_offset(tg->latency_prio - MAX_RT_PRIO);
+ set_latency_fair(se, tg->latency_prio - MAX_RT_PRIO);
/* guarantee group entities always have weight */
update_load_set(&se->load, NICE_0_LOAD);
@@ -12365,7 +12365,6 @@ int sched_group_set_idle(struct task_gro
int sched_group_set_latency(struct task_group *tg, int prio)
{
- long latency_offset;
int i;
if (tg == &root_task_group)
@@ -12379,13 +12378,9 @@ int sched_group_set_latency(struct task_
}
tg->latency_prio = prio;
- latency_offset = calc_latency_offset(prio - MAX_RT_PRIO);
- for_each_possible_cpu(i) {
- struct sched_entity *se = tg->se[i];
-
- WRITE_ONCE(se->latency_offset, latency_offset);
- }
+ for_each_possible_cpu(i)
+ set_latency_fair(tg->se[i], prio - MAX_RT_PRIO);
mutex_unlock(&shares_mutex);
return 0;
@@ -2477,7 +2477,7 @@ extern unsigned int sysctl_numa_balancin
extern unsigned int sysctl_numa_balancing_hot_threshold;
#endif
-extern long calc_latency_offset(int prio);
+extern void set_latency_fair(struct sched_entity *se, int prio);
#ifdef CONFIG_SCHED_HRTICK