28 Feb
2022
28 Feb
'22
2:37 a.m.
From: Srinath Sridharan <srinathsr(a)google.com>
ohos inclusion
category: feature
issue: #I4TNS2
CVE: NA
Signed-off-by: Hu Zhaodong <huzhaodong(a)huawei.com>
-------------------------------------------
EAS scheduler optimization
Allow running cfs task migration in clock interrupt
Signed-off-by: Satya Durga Srinivasu Prabhala <satyap(a)codeaurora.org>
Signed-off-by: Vikram Mulukutla <markivx(a)codeaurora.org>
Signed-off-by: Srinath Sridharan <srinathsr(a)google.com>
---
init/Kconfig | 6 ++
kernel/sched/core.c | 35 +++++++++++
kernel/sched/fair.c | 141 ++++++++++++++++++++++++++++++++++++++++++-
kernel/sched/sched.h | 7 +++
4 files changed, 188 insertions(+), 1 deletion(-)
diff --git a/init/Kconfig b/init/Kconfig
index 1d248e9c5a89..db7449e779c6 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -860,6 +860,12 @@ config UCLAMP_BUCKETS_COUNT
source "kernel/sched/rtg/Kconfig"
+config SCHED_EAS
+ bool "EAS scheduler optimization"
+ default n
+ help
+ Check and migrate the CFS process to a more suitable CPU in the tick.
+
endmenu
#
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 8e506f6efc73..471b2129ea84 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4134,6 +4134,11 @@ void scheduler_tick(void)
#ifdef CONFIG_SMP
rq->idle_balance = idle_cpu(cpu);
trigger_load_balance(rq);
+
+#ifdef CONFIG_SCHED_EAS
+ if (curr->sched_class->check_for_migration)
+ curr->sched_class->check_for_migration(rq, curr);
+#endif
#endif
}
@@ -7025,6 +7030,32 @@ void migrate_tasks(struct rq *dead_rq, struct rq_flags *rf,
attach_tasks_core(&tasks, rq);
}
+#ifdef CONFIG_SCHED_EAS
+static void clear_eas_migration_request(int cpu)
+{
+ struct rq *rq = cpu_rq(cpu);
+ unsigned long flags;
+
+ clear_reserved(cpu);
+ if (rq->push_task) {
+ struct task_struct *push_task = NULL;
+
+ raw_spin_lock_irqsave(&rq->lock, flags);
+ if (rq->push_task) {
+ clear_reserved(rq->push_cpu);
+ push_task = rq->push_task;
+ rq->push_task = NULL;
+ }
+ rq->active_balance = 0;
+ raw_spin_unlock_irqrestore(&rq->lock, flags);
+ if (push_task)
+ put_task_struct(push_task);
+ }
+}
+#else
+static inline void clear_eas_migration_request(int cpu) {}
+#endif
+
#ifdef CONFIG_CPU_ISOLATION_OPT
int do_isolation_work_cpu_stop(void *data)
{
@@ -7058,6 +7089,7 @@ int do_isolation_work_cpu_stop(void *data)
set_rq_online(rq);
rq_unlock(rq, &rf);
+ clear_eas_migration_request(cpu);
local_irq_enable();
return 0;
}
@@ -7425,6 +7457,7 @@ int sched_cpu_starting(unsigned int cpu)
{
sched_rq_cpu_starting(cpu);
sched_tick_start(cpu);
+ clear_eas_migration_request(cpu);
return 0;
}
@@ -7447,6 +7480,8 @@ int sched_cpu_dying(unsigned int cpu)
BUG_ON(rq->nr_running != 1);
rq_unlock_irqrestore(rq, &rf);
+ clear_eas_migration_request(cpu);
+
calc_load_migrate(rq);
update_max_interval();
nohz_balance_exit_idle(rq);
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 3b8d6c1dfc30..9789a385fecd 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -10110,9 +10110,13 @@ static int active_load_balance_cpu_stop(void *data)
int busiest_cpu = cpu_of(busiest_rq);
int target_cpu = busiest_rq->push_cpu;
struct rq *target_rq = cpu_rq(target_cpu);
- struct sched_domain *sd;
+ struct sched_domain *sd = NULL;
struct task_struct *p = NULL;
struct rq_flags rf;
+#ifdef CONFIG_SCHED_EAS
+ struct task_struct *push_task;
+ int push_task_detached = 0;
+#endif
rq_lock_irq(busiest_rq, &rf);
/*
@@ -10139,6 +10143,32 @@ static int active_load_balance_cpu_stop(void *data)
*/
BUG_ON(busiest_rq == target_rq);
+#ifdef CONFIG_SCHED_EAS
+ push_task = busiest_rq->push_task;
+ target_cpu = busiest_rq->push_cpu;
+ if (push_task) {
+ struct lb_env env = {
+ .sd = sd,
+ .dst_cpu = target_cpu,
+ .dst_rq = target_rq,
+ .src_cpu = busiest_rq->cpu,
+ .src_rq = busiest_rq,
+ .idle = CPU_IDLE,
+ .flags = 0,
+ .loop = 0,
+ };
+ if (task_on_rq_queued(push_task) &&
+ push_task->state == TASK_RUNNING &&
+ task_cpu(push_task) == busiest_cpu &&
+ cpu_online(target_cpu)) {
+ update_rq_clock(busiest_rq);
+ detach_task(push_task, &env);
+ push_task_detached = 1;
+ }
+ goto out_unlock;
+ }
+#endif
+
/* Search for an sd spanning us and the target CPU. */
rcu_read_lock();
for_each_domain(target_cpu, sd) {
@@ -10178,8 +10208,23 @@ static int active_load_balance_cpu_stop(void *data)
rcu_read_unlock();
out_unlock:
busiest_rq->active_balance = 0;
+
+#ifdef CONFIG_SCHED_EAS
+ push_task = busiest_rq->push_task;
+ if (push_task)
+ busiest_rq->push_task = NULL;
+#endif
rq_unlock(busiest_rq, &rf);
+#ifdef CONFIG_SCHED_EAS
+ if (push_task) {
+ if (push_task_detached)
+ attach_one_task(target_rq, push_task);
+
+ put_task_struct(push_task);
+ }
+#endif
+
if (p)
attach_one_task(target_rq, p);
@@ -10979,6 +11024,97 @@ static void rq_offline_fair(struct rq *rq)
unthrottle_offline_cfs_rqs(rq);
}
+#ifdef CONFIG_SCHED_EAS
+static inline int
+kick_active_balance(struct rq *rq, struct task_struct *p, int new_cpu)
+{
+ unsigned long flags;
+ int rc = 0;
+
+ if (cpu_of(rq) == new_cpu)
+ return rc;
+
+ /* Invoke active balance to force migrate currently running task */
+ raw_spin_lock_irqsave(&rq->lock, flags);
+ if (!rq->active_balance) {
+ rq->active_balance = 1;
+ rq->push_cpu = new_cpu;
+ get_task_struct(p);
+ rq->push_task = p;
+ rc = 1;
+ }
+ raw_spin_unlock_irqrestore(&rq->lock, flags);
+ return rc;
+}
+
+DEFINE_RAW_SPINLOCK(migration_lock);
+void check_for_migration_fair(struct rq *rq, struct task_struct *p)
+{
+ int active_balance;
+ int new_cpu = -1;
+ int prev_cpu = task_cpu(p);
+ int ret;
+
+#ifdef CONFIG_SCHED_RTG
+ bool need_down_migrate = false;
+ struct cpumask *rtg_target = find_rtg_target(p);
+
+ if (rtg_target &&
+ (capacity_orig_of(prev_cpu) >
+ capacity_orig_of(cpumask_first(rtg_target))))
+ need_down_migrate = true;
+#endif
+
+ if (rq->misfit_task_load) {
+ if (rq->curr->state != TASK_RUNNING ||
+ rq->curr->nr_cpus_allowed == 1)
+ return;
+
+ raw_spin_lock(&migration_lock);
+#ifdef CONFIG_SCHED_RTG
+ if (rtg_target) {
+ new_cpu = find_rtg_cpu(p);
+
+ if (new_cpu != -1 && need_down_migrate &&
+ cpumask_test_cpu(new_cpu, rtg_target) &&
+ idle_cpu(new_cpu))
+ goto do_active_balance;
+
+ if (new_cpu != -1 &&
+ capacity_orig_of(new_cpu) > capacity_orig_of(prev_cpu))
+ goto do_active_balance;
+
+ goto out_unlock;
+ }
+#endif
+ rcu_read_lock();
+ new_cpu = find_energy_efficient_cpu(p, prev_cpu);
+ rcu_read_unlock();
+
+ if (new_cpu == -1 ||
+ capacity_orig_of(new_cpu) <= capacity_orig_of(prev_cpu))
+ goto out_unlock;
+#ifdef CONFIG_SCHED_RTG
+do_active_balance:
+#endif
+ active_balance = kick_active_balance(rq, p, new_cpu);
+ if (active_balance) {
+ mark_reserved(new_cpu);
+ raw_spin_unlock(&migration_lock);
+ ret = stop_one_cpu_nowait(prev_cpu,
+ active_load_balance_cpu_stop, rq,
+ &rq->active_balance_work);
+ if (!ret)
+ clear_reserved(new_cpu);
+ else
+ wake_up_if_idle(new_cpu);
+ return;
+ }
+out_unlock:
+ raw_spin_unlock(&migration_lock);
+ }
+}
+#endif /* CONFIG_SCHED_EAS */
#endif /* CONFIG_SMP */
/*
@@ -11530,6 +11666,9 @@ const struct sched_class fair_sched_class
#ifdef CONFIG_SCHED_WALT
.fixup_walt_sched_stats = walt_fixup_sched_stats_fair,
#endif
+#ifdef CONFIG_SCHED_EAS
+ .check_for_migration = check_for_migration_fair,
+#endif
};
#ifdef CONFIG_SCHED_DEBUG
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index d79744dcc048..1a4f1806eb78 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1047,6 +1047,9 @@ struct rq {
/* For active balancing */
int active_balance;
int push_cpu;
+#ifdef CONFIG_SCHED_EAS
+ struct task_struct *push_task;
+#endif
struct cpu_stop_work active_balance_work;
/* CPU of this runqueue: */
@@ -1925,6 +1928,9 @@ struct sched_class {
void (*fixup_walt_sched_stats)(struct rq *rq, struct task_struct *p,
u16 updated_demand_scaled);
#endif
+#ifdef CONFIG_SCHED_EAS
+ void (*check_for_migration)(struct rq *rq, struct task_struct *p);
+#endif
} __aligned(STRUCT_ALIGNMENT); /* STRUCT_ALIGN(), vmlinux.lds.h */
static inline void put_prev_task(struct rq *rq, struct task_struct *prev)
@@ -2768,6 +2774,7 @@ extern bool task_fits_max(struct task_struct *p, int cpu);
extern unsigned long capacity_spare_without(int cpu, struct task_struct *p);
extern int update_preferred_cluster(struct related_thread_group *grp,
struct task_struct *p, u32 old_load, bool from_tick);
+extern struct cpumask *find_rtg_target(struct task_struct *p);
#endif
#ifdef CONFIG_SCHED_WALT
--
2.25.1
28 Feb
28 Feb
2:37 a.m.
New subject: [PATCH OpenHarmony-5.10 2/3] sched: rt-active-lb optimization
From: Hu Zhaodong <huzhaodong(a)huawei.com>
ohos inclusion
category: feature
issue: #I4TNS2
CVE: NA
Signed-off-by: Hu Zhaodong <huzhaodong(a)huawei.com>
-------------------------------------------
allow migrating running rt task in clock interrupt
Signed-off-by: gaochao <gaochao49(a)huawei.com>
---
include/linux/sched/sysctl.h | 4 ++
init/Kconfig | 7 +++
kernel/sched/rt.c | 90 ++++++++++++++++++++++++++++++++++++
kernel/sched/sched.h | 7 +++
kernel/sysctl.c | 9 ++++
5 files changed, 117 insertions(+)
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h
index 210909cd4141..acec3b1fd469 100644
--- a/include/linux/sched/sysctl.h
+++ b/include/linux/sched/sysctl.h
@@ -41,6 +41,10 @@ sysctl_sched_walt_init_task_load_pct_sysctl_handler(struct ctl_table
*table,
int write, void __user *buffer, size_t *length, loff_t *ppos);
#endif
+#ifdef CONFIG_SCHED_RT_ACTIVE_LB
+extern unsigned int sysctl_sched_enable_rt_active_lb;
+#endif
+
enum sched_tunable_scaling {
SCHED_TUNABLESCALING_NONE,
SCHED_TUNABLESCALING_LOG,
diff --git a/init/Kconfig b/init/Kconfig
index db7449e779c6..57554d795040 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -866,6 +866,13 @@ config SCHED_EAS
help
Check and migrate the CFS process to a more suitable CPU in the tick.
+config SCHED_RT_ACTIVE_LB
+ bool "RT Capacity Aware Misfit Task"
+ depends on SCHED_EAS
+ default n
+ help
+ Check and migrate the RT process to a more suitable CPU in the tick.
+
endmenu
#
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 6c1475950441..9adcbf0e0bee 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -17,6 +17,10 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int
overrun);
struct rt_bandwidth def_rt_bandwidth;
+#ifdef CONFIG_SCHED_RT_ACTIVE_LB
+unsigned int sysctl_sched_enable_rt_active_lb = 1;
+#endif
+
static enum hrtimer_restart sched_rt_period_timer(struct hrtimer *timer)
{
struct rt_bandwidth *rt_b =
@@ -2443,6 +2447,89 @@ static void task_tick_rt(struct rq *rq, struct task_struct *p, int
queued)
}
}
+#ifdef CONFIG_SCHED_RT_ACTIVE_LB
+static int rt_active_load_balance_cpu_stop(void *data)
+{
+ struct rq *busiest_rq = data;
+ struct task_struct *next_task = busiest_rq->rt_push_task;
+ struct rq *lowest_rq = NULL;
+ unsigned long flags;
+
+ raw_spin_lock_irqsave(&busiest_rq->lock, flags);
+ busiest_rq->rt_active_balance = 0;
+
+ /* find_lock_lowest_rq locks the rq if found */
+ lowest_rq = find_lock_lowest_rq(next_task, busiest_rq);
+ if (!lowest_rq)
+ goto out;
+
+ if (capacity_orig_of(cpu_of(lowest_rq)) <= capacity_orig_of(task_cpu(next_task)))
+ goto unlock;
+
+ deactivate_task(busiest_rq, next_task, 0);
+ set_task_cpu(next_task, lowest_rq->cpu);
+ activate_task(lowest_rq, next_task, 0);
+
+ resched_curr(lowest_rq);
+unlock:
+ double_unlock_balance(busiest_rq, lowest_rq);
+out:
+ put_task_struct(next_task);
+ raw_spin_unlock_irqrestore(&busiest_rq->lock, flags);
+
+ return 0;
+}
+
+void check_for_migration_rt(struct rq *rq, struct task_struct *p)
+{
+ bool need_actvie_lb = false;
+ bool misfit_task = false;
+ int cpu = task_cpu(p);
+ unsigned long cpu_orig_cap;
+#ifdef CONFIG_SCHED_RTG
+ struct cpumask *rtg_target = NULL;
+#endif
+
+ if (!sysctl_sched_enable_rt_active_lb)
+ return;
+
+ if (p->nr_cpus_allowed == 1)
+ return;
+
+ cpu_orig_cap = capacity_orig_of(cpu);
+ /* cpu has max capacity, no need to do balance */
+ if (cpu_orig_cap == rq->rd->max_cpu_capacity)
+ return;
+
+#ifdef CONFIG_SCHED_RTG
+ rtg_target = find_rtg_target(p);
+ if (rtg_target)
+ misfit_task = capacity_orig_of(cpumask_first(rtg_target)) >
+ cpu_orig_cap;
+ else
+ misfit_task = !rt_task_fits_capacity(p, cpu);
+#else
+ misfit_task = !rt_task_fits_capacity(p, cpu);
+#endif
+
+ if (misfit_task) {
+ raw_spin_lock(&rq->lock);
+ if (!rq->active_balance && !rq->rt_active_balance) {
+ rq->rt_active_balance = 1;
+ rq->rt_push_task = p;
+ get_task_struct(p);
+ need_actvie_lb = true;
+ }
+ raw_spin_unlock(&rq->lock);
+
+ if (need_actvie_lb)
+ stop_one_cpu_nowait(task_cpu(p),
+ rt_active_load_balance_cpu_stop,
+ rq, &rq->rt_active_balance_work);
+ }
+}
+#endif
+
static unsigned int get_rr_interval_rt(struct rq *rq, struct task_struct *task)
{
/*
@@ -2491,6 +2578,9 @@ const struct sched_class rt_sched_class
#ifdef CONFIG_SCHED_WALT
.fixup_walt_sched_stats = fixup_walt_sched_stats_common,
#endif
+#ifdef CONFIG_SCHED_RT_ACTIVE_LB
+ .check_for_migration = check_for_migration_rt,
+#endif
};
#ifdef CONFIG_RT_GROUP_SCHED
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 1a4f1806eb78..09ad491bed45 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1052,6 +1052,13 @@ struct rq {
#endif
struct cpu_stop_work active_balance_work;
+ /* For rt active balancing */
+#ifdef CONFIG_SCHED_RT_ACTIVE_LB
+ int rt_active_balance;
+ struct task_struct *rt_push_task;
+ struct cpu_stop_work rt_active_balance_work;
+#endif
+
/* CPU of this runqueue: */
int cpu;
int online;
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index f13b9e456f50..d5fef7aba276 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -1659,6 +1659,15 @@ static struct ctl_table kern_table[] = {
.mode = 0644,
.proc_handler = proc_dointvec,
},
+#ifdef CONFIG_SCHED_RT_ACTIVE_LB
+ {
+ .procname = "sched_enable_rt_active_lb",
+ .data = &sysctl_sched_enable_rt_active_lb,
+ .maxlen = sizeof(unsigned int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ },
+#endif
#ifdef CONFIG_SCHED_WALT
{
.procname = "sched_use_walt_cpu_util",
--
2.25.1
2:37 a.m.
New subject: [PATCH OpenHarmony-5.10 3/3] sched: rt-cas optimization
From: Hu Zhaodong <huzhaodong(a)huawei.com>
ohos inclusion
category: feature
issue: #I4TNS2
CVE: NA
Signed-off-by: Hu Zhaodong <huzhaodong(a)huawei.com>
-------------------------------------------
RT task detects capacity during CPU selection
Signed-off-by: gaochao <gaochao49(a)huawei.com>
---
include/linux/sched/sysctl.h | 3 +
include/trace/events/eas_sched.h | 76 +++++++++++++
include/trace/events/sched.h | 4 +
init/Kconfig | 7 ++
kernel/sched/fair.c | 8 ++
kernel/sched/rt.c | 177 +++++++++++++++++++++++++++++++
kernel/sched/sched.h | 17 +++
kernel/sched/topology.c | 14 +++
kernel/sysctl.c | 9 ++
9 files changed, 315 insertions(+)
create mode 100644 include/trace/events/eas_sched.h
diff --git a/include/linux/sched/sysctl.h b/include/linux/sched/sysctl.h
index acec3b1fd469..a08551ebd23d 100644
--- a/include/linux/sched/sysctl.h
+++ b/include/linux/sched/sysctl.h
@@ -41,6 +41,9 @@ sysctl_sched_walt_init_task_load_pct_sysctl_handler(struct ctl_table
*table,
int write, void __user *buffer, size_t *length, loff_t *ppos);
#endif
+#ifdef CONFIG_SCHED_RT_CAS
+extern unsigned int sysctl_sched_enable_rt_cas;
+#endif
#ifdef CONFIG_SCHED_RT_ACTIVE_LB
extern unsigned int sysctl_sched_enable_rt_active_lb;
#endif
diff --git a/include/trace/events/eas_sched.h b/include/trace/events/eas_sched.h
new file mode 100644
index 000000000000..bd24c9ef5b6e
--- /dev/null
+++ b/include/trace/events/eas_sched.h
@@ -0,0 +1,76 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifdef CONFIG_SCHED_RT_CAS
+TRACE_EVENT(sched_find_cas_cpu_each,
+
+ TP_PROTO(struct task_struct *task, int cpu, int target_cpu,
+ int isolated, int idle, unsigned long task_util,
+ unsigned long cpu_util, int cpu_cap),
+
+ TP_ARGS(task, cpu, target_cpu, isolated, idle, task_util, cpu_util, cpu_cap),
+
+ TP_STRUCT__entry(
+ __array(char, comm, TASK_COMM_LEN)
+ __field(pid_t, pid)
+ __field(int, prio)
+ __field(int, cpu)
+ __field(int, target_cpu)
+ __field(int, isolated)
+ __field(unsigned long, idle)
+ __field(unsigned long, task_util)
+ __field(unsigned long, cpu_util)
+ __field(unsigned long, cpu_cap)
+ ),
+
+ TP_fast_assign(
+ memcpy(__entry->comm, task->comm, TASK_COMM_LEN);
+ __entry->pid = task->pid;
+ __entry->prio = task->prio;
+ __entry->cpu = cpu;
+ __entry->target_cpu = target_cpu;
+ __entry->isolated = isolated;
+ __entry->idle = idle;
+ __entry->task_util = task_util;
+ __entry->cpu_util = cpu_util;
+ __entry->cpu_cap = cpu_cap;
+ ),
+
+ TP_printk("comm=%s pid=%d prio=%d cpu=%d target_cpu=%d isolated=%d idle=%d
task_util=%lu cpu_util=%lu cpu_cap=%lu",
+ __entry->comm, __entry->pid, __entry->prio,
+ __entry->cpu, __entry->target_cpu, __entry->isolated,
+ __entry->idle, __entry->task_util,
+ __entry->cpu_util, __entry->cpu_cap)
+);
+
+TRACE_EVENT(sched_find_cas_cpu,
+
+ TP_PROTO(struct task_struct *task, struct cpumask *lowest_mask,
+ unsigned long tutil, int prev_cpu, int target_cpu),
+
+ TP_ARGS(task, lowest_mask, tutil, prev_cpu, target_cpu),
+
+ TP_STRUCT__entry(
+ __array(char, comm, TASK_COMM_LEN)
+ __field(pid_t, pid)
+ __field(unsigned int, prio)
+ __bitmask(lowest, num_possible_cpus())
+ __field(unsigned long, tutil)
+ __field(int, prev_cpu)
+ __field(int, target_cpu)
+ ),
+
+ TP_fast_assign(
+ memcpy(__entry->comm, task->comm, TASK_COMM_LEN);
+ __entry->pid = task->pid;
+ __entry->prio = task->prio;
+ __assign_bitmask(lowest, cpumask_bits(lowest_mask), num_possible_cpus());
+ __entry->tutil = tutil;
+ __entry->prev_cpu = prev_cpu;
+ __entry->target_cpu = target_cpu;
+ ),
+
+ TP_printk("comm=%s pid=%d prio=%d lowest_mask=%s tutil=%lu perfer_idle=%u prev=%d
target=%d ",
+ __entry->comm, __entry->pid, __entry->prio,
+ __get_bitmask(lowest), __entry->tutil,
+ __entry->prev_cpu, __entry->target_cpu)
+);
+#endif /* CONFIG_SCHED_RT_CAS */
diff --git a/include/trace/events/sched.h b/include/trace/events/sched.h
index 27b6ed3c9e58..dd5fff2bb1b2 100644
--- a/include/trace/events/sched.h
+++ b/include/trace/events/sched.h
@@ -10,6 +10,10 @@
#include <linux/tracepoint.h>
#include <linux/binfmts.h>
+#ifdef CONFIG_SCHED_RT_CAS
+#include "eas_sched.h"
+#endif
+
/*
* Tracepoint for calling kthread_stop, performed to end a kthread:
*/
diff --git a/init/Kconfig b/init/Kconfig
index 57554d795040..ded631516e22 100644
--- a/init/Kconfig
+++ b/init/Kconfig
@@ -866,6 +866,13 @@ config SCHED_EAS
help
Check and migrate the CFS process to a more suitable CPU in the tick.
+config SCHED_RT_CAS
+ bool "rt-cas optimization"
+ depends on SCHED_EAS
+ default n
+ help
+ RT task detects capacity during CPU selection
+
config SCHED_RT_ACTIVE_LB
bool "RT Capacity Aware Misfit Task"
depends on SCHED_EAS
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 9789a385fecd..945bd3fb2478 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -3953,14 +3953,22 @@ static inline unsigned long task_util_est(struct task_struct *p)
}
#ifdef CONFIG_UCLAMP_TASK
+#ifdef CONFIG_SCHED_RT_CAS
+unsigned long uclamp_task_util(struct task_struct *p)
+#else
static inline unsigned long uclamp_task_util(struct task_struct *p)
+#endif
{
return clamp(task_util_est(p),
uclamp_eff_value(p, UCLAMP_MIN),
uclamp_eff_value(p, UCLAMP_MAX));
}
#else
+#ifdef CONFIG_SCHED_RT_CAS
+unsigned long uclamp_task_util(struct task_struct *p)
+#else
static inline unsigned long uclamp_task_util(struct task_struct *p)
+#endif
{
return task_util_est(p);
}
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index 9adcbf0e0bee..5926d6f6efbb 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -17,6 +17,10 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int
overrun);
struct rt_bandwidth def_rt_bandwidth;
+#ifdef CONFIG_SCHED_RT_CAS
+unsigned int sysctl_sched_enable_rt_cas = 1;
+#endif
+
#ifdef CONFIG_SCHED_RT_ACTIVE_LB
unsigned int sysctl_sched_enable_rt_active_lb = 1;
#endif
@@ -1709,6 +1713,170 @@ static struct task_struct *pick_highest_pushable_task(struct rq
*rq, int cpu)
return NULL;
}
+#ifdef CONFIG_SCHED_RT_CAS
+int find_cas_cpu(struct sched_domain *sd,
+ struct task_struct *task, struct cpumask *lowest_mask)
+{
+ struct root_domain *rd = cpu_rq(smp_processor_id())->rd;
+ struct sched_group *sg = NULL;
+ struct sched_group *sg_target = NULL;
+ struct sched_group *sg_backup = NULL;
+ struct cpumask search_cpu, backup_search_cpu;
+ int cpu = -1;
+ int target_cpu = -1;
+ unsigned long cpu_capacity;
+ unsigned long boosted_tutil = uclamp_task_util(task);
+ unsigned long target_capacity = ULONG_MAX;
+ unsigned long util;
+ unsigned long target_cpu_util = ULONG_MAX;
+ int prev_cpu = task_cpu(task);
+#ifdef CONFIG_SCHED_RTG
+ struct cpumask *rtg_target = NULL;
+#endif
+ bool boosted = uclamp_boosted(task);
+
+ if (!sysctl_sched_enable_rt_cas)
+ return -1;
+
+ rcu_read_lock();
+
+#ifdef CONFIG_SCHED_RTG
+ rtg_target = find_rtg_target(task);
+#endif
+
+ sd = rcu_dereference(per_cpu(sd_asym_cpucapacity, 0));
+ if (!sd) {
+ rcu_read_unlock();
+ return -1;
+ }
+
+ sg = sd->groups;
+ do {
+ if (!cpumask_intersects(lowest_mask, sched_group_span(sg)))
+ continue;
+
+ if (boosted) {
+ if (cpumask_test_cpu(rd->max_cap_orig_cpu,
+ sched_group_span(sg))) {
+ sg_target = sg;
+ break;
+ }
+ }
+
+ cpu = group_first_cpu(sg);
+#ifdef CONFIG_SCHED_RTG
+ /* honor the rtg tasks */
+ if (rtg_target) {
+ if (cpumask_test_cpu(cpu, rtg_target)) {
+ sg_target = sg;
+ break;
+ }
+
+ /* active LB or big_task favor cpus with more capacity */
+ if (task->state == TASK_RUNNING || boosted) {
+ if (capacity_orig_of(cpu) >
+ capacity_orig_of(cpumask_any(rtg_target))) {
+ sg_target = sg;
+ break;
+ }
+
+ sg_backup = sg;
+ continue;
+ }
+ }
+#endif
+ /*
+ * 1. add margin to support task migration
+ * 2. if task_util is high then all cpus, make sure the
+ * sg_backup with the most powerful cpus is selected
+ */
+ if (!rt_task_fits_capacity(task, cpu)) {
+ sg_backup = sg;
+ continue;
+ }
+
+ /* support task boost */
+ cpu_capacity = capacity_orig_of(cpu);
+ if (boosted_tutil > cpu_capacity) {
+ sg_backup = sg;
+ continue;
+ }
+
+ /* sg_target: select the sg with smaller capacity */
+ if (cpu_capacity < target_capacity) {
+ target_capacity = cpu_capacity;
+ sg_target = sg;
+ }
+ } while (sg = sg->next, sg != sd->groups);
+
+ if (!sg_target)
+ sg_target = sg_backup;
+
+ if (sg_target) {
+ cpumask_and(&search_cpu, lowest_mask, sched_group_span(sg_target));
+ cpumask_copy(&backup_search_cpu, lowest_mask);
+ cpumask_andnot(&backup_search_cpu, &backup_search_cpu, &search_cpu);
+ } else {
+ cpumask_copy(&search_cpu, lowest_mask);
+ cpumask_clear(&backup_search_cpu);
+ }
+
+retry:
+ cpu = cpumask_first(&search_cpu);
+ do {
+ trace_sched_find_cas_cpu_each(task, cpu, target_cpu,
+ cpu_isolated(cpu),
+ idle_cpu(cpu), boosted_tutil, cpu_util(cpu),
+ capacity_orig_of(cpu));
+
+ if (cpu_isolated(cpu))
+ continue;
+
+ if (!cpumask_test_cpu(cpu, task->cpus_ptr))
+ continue;
+
+ /* find best cpu with smallest max_capacity */
+ if (target_cpu != -1 &&
+ capacity_orig_of(cpu) > capacity_orig_of(target_cpu))
+ continue;
+
+ util = cpu_util(cpu);
+
+ /* Find the least loaded CPU */
+ if (util > target_cpu_util)
+ continue;
+
+ /*
+ * If the preivous CPU has same load, keep it as
+ * target_cpu
+ */
+ if (target_cpu_util == util && target_cpu == prev_cpu)
+ continue;
+
+ /*
+ * If candidate CPU is the previous CPU, select it.
+ * If all above conditions are same, select the least
+ * cumulative window demand CPU.
+ */
+ target_cpu_util = util;
+ target_cpu = cpu;
+ } while ((cpu = cpumask_next(cpu, &search_cpu)) < nr_cpu_ids);
+
+ if (target_cpu != -1 && cpumask_test_cpu(target_cpu, lowest_mask)) {
+ goto done;
+ } else if (!cpumask_empty(&backup_search_cpu)) {
+ cpumask_copy(&search_cpu, &backup_search_cpu);
+ cpumask_clear(&backup_search_cpu);
+ goto retry;
+ }
+
+done:
+ trace_sched_find_cas_cpu(task, lowest_mask, boosted_tutil, prev_cpu, target_cpu);
+ rcu_read_unlock();
+ return target_cpu;
+}
+#endif
+
static DEFINE_PER_CPU(cpumask_var_t, local_cpu_mask);
static int find_lowest_rq(struct task_struct *task)
@@ -1718,6 +1886,9 @@ static int find_lowest_rq(struct task_struct *task)
int this_cpu = smp_processor_id();
int cpu = task_cpu(task);
int ret;
+#ifdef CONFIG_SCHED_RT_CAS
+ int cas_cpu;
+#endif
/* Make sure the mask is initialized first */
if (unlikely(!lowest_mask))
@@ -1744,6 +1915,12 @@ static int find_lowest_rq(struct task_struct *task)
if (!ret)
return -1; /* No targets found */
+#ifdef CONFIG_SCHED_RT_CAS
+ cas_cpu = find_cas_cpu(sd, task, lowest_mask);
+ if (cas_cpu != -1)
+ return cas_cpu;
+#endif
+
/*
* At this point we have built a mask of CPUs representing the
* lowest priority tasks in the system. Now we want to elect
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 09ad491bed45..e4c65d96185e 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -87,6 +87,10 @@
struct rq;
struct cpuidle_state;
+#ifdef CONFIG_SCHED_RT_CAS
+extern unsigned long uclamp_task_util(struct task_struct *p);
+#endif
+
#ifdef CONFIG_SCHED_WALT
extern unsigned int sched_ravg_window;
extern unsigned int walt_cpu_util_freq_divisor;
@@ -893,6 +897,9 @@ struct root_domain {
* CPUs of the rd. Protected by RCU.
*/
struct perf_domain __rcu *pd;
+#ifdef CONFIG_SCHED_RT_CAS
+ int max_cap_orig_cpu;
+#endif
};
extern void init_defrootdomain(void);
@@ -2582,6 +2589,11 @@ unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long
util,
return clamp(util, min_util, max_util);
}
+static inline bool uclamp_boosted(struct task_struct *p)
+{
+ return uclamp_eff_value(p, UCLAMP_MIN) > 0;
+}
+
/*
* When uclamp is compiled in, the aggregation at rq level is 'turned off'
* by default in the fast path and only gets turned on once userspace performs
@@ -2602,6 +2614,11 @@ unsigned long uclamp_rq_util_with(struct rq *rq, unsigned long
util,
return util;
}
+static inline bool uclamp_boosted(struct task_struct *p)
+{
+ return false;
+}
+
static inline bool uclamp_is_used(void)
{
return false;
diff --git a/kernel/sched/topology.c b/kernel/sched/topology.c
index b30b62f0d683..9191e5daaa3c 100644
--- a/kernel/sched/topology.c
+++ b/kernel/sched/topology.c
@@ -522,6 +522,10 @@ static int init_rootdomain(struct root_domain *rd)
if (cpupri_init(&rd->cpupri) != 0)
goto free_cpudl;
+
+#ifdef CONFIG_SCHED_RT_CAS
+ rd->max_cap_orig_cpu = -1;
+#endif
return 0;
free_cpudl:
@@ -2121,9 +2125,19 @@ build_sched_domains(const struct cpumask *cpu_map, struct
sched_domain_attr *att
/* Attach the domains */
rcu_read_lock();
for_each_cpu(i, cpu_map) {
+#ifdef CONFIG_SCHED_RT_CAS
+ int max_cpu = READ_ONCE(d.rd->max_cap_orig_cpu);
+#endif
+
rq = cpu_rq(i);
sd = *per_cpu_ptr(d.sd, i);
+#ifdef CONFIG_SCHED_RT_CAS
+ if (max_cpu < 0 || arch_scale_cpu_capacity(i) >
+ arch_scale_cpu_capacity(max_cpu))
+ WRITE_ONCE(d.rd->max_cap_orig_cpu, i);
+#endif
+
/* Use READ_ONCE()/WRITE_ONCE() to avoid load/store tearing: */
if (rq->cpu_capacity_orig > READ_ONCE(d.rd->max_cpu_capacity))
WRITE_ONCE(d.rd->max_cpu_capacity, rq->cpu_capacity_orig);
diff --git a/kernel/sysctl.c b/kernel/sysctl.c
index d5fef7aba276..e34d6937594c 100644
--- a/kernel/sysctl.c
+++ b/kernel/sysctl.c
@@ -1659,6 +1659,15 @@ static struct ctl_table kern_table[] = {
.mode = 0644,
.proc_handler = proc_dointvec,
},
+#ifdef CONFIG_SCHED_RT_CAS
+ {
+ .procname = "sched_enable_rt_cas",
+ .data = &sysctl_sched_enable_rt_cas,
+ .maxlen = sizeof(unsigned int),
+ .mode = 0644,
+ .proc_handler = proc_dointvec,
+ },
+#endif
#ifdef CONFIG_SCHED_RT_ACTIVE_LB
{
.procname = "sched_enable_rt_active_lb",
--
2.25.1
811
days inactive
811
days old
2 comments
1 participants
participants (1)
-
gc1202