diff options
Diffstat (limited to 'SOURCES/cachy-bore.patch')
| -rw-r--r-- | SOURCES/cachy-bore.patch | 1046 |
1 files changed, 554 insertions, 492 deletions
diff --git a/SOURCES/cachy-bore.patch b/SOURCES/cachy-bore.patch index 7421807..f15d6da 100644 --- a/SOURCES/cachy-bore.patch +++ b/SOURCES/cachy-bore.patch @@ -1,24 +1,53 @@ -From 3816495f5635104fae1dda21b743f750c2914196 Mon Sep 17 00:00:00 2001 -From: Eric Naim <dnaim@proton.me> -Date: Sat, 3 Aug 2024 15:23:30 +0700 -Subject: [PATCH] bore +From 2328e7500823151cb2f119b847454e63e9da0f64 Mon Sep 17 00:00:00 2001 +From: Masahito S <firelzrd@gmail.com> +Date: Thu, 3 Oct 2024 17:33:57 +0900 +Subject: [PATCH] linux6.11.y-bore5.6.0 --- - include/linux/sched.h | 10 ++ - init/Kconfig | 17 +++ - kernel/Kconfig.hz | 16 ++ - kernel/sched/core.c | 143 ++++++++++++++++++ - kernel/sched/debug.c | 60 +++++++- - kernel/sched/fair.c | 322 ++++++++++++++++++++++++++++++++++++++-- - kernel/sched/features.h | 22 ++- - kernel/sched/sched.h | 7 + - 8 files changed, 583 insertions(+), 14 deletions(-) + include/linux/sched.h | 20 +- + include/linux/sched/bore.h | 37 ++++ + init/Kconfig | 17 ++ + kernel/Kconfig.hz | 17 ++ + kernel/fork.c | 5 + + kernel/sched/Makefile | 1 + + kernel/sched/bore.c | 380 +++++++++++++++++++++++++++++++++++++ + kernel/sched/core.c | 7 + + kernel/sched/debug.c | 60 +++++- + kernel/sched/fair.c | 89 ++++++++- + kernel/sched/features.h | 4 + + kernel/sched/sched.h | 7 + + 12 files changed, 639 insertions(+), 5 deletions(-) + create mode 100644 include/linux/sched/bore.h + create mode 100644 kernel/sched/bore.c diff --git a/include/linux/sched.h b/include/linux/sched.h -index 76214d7c8..9f65d367b 100644 +index f8d150343d..2481cf0125 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h -@@ -547,6 +547,16 @@ struct sched_entity { +@@ -535,6 +535,14 @@ struct sched_statistics { + #endif /* CONFIG_SCHEDSTATS */ + } ____cacheline_aligned; + ++#ifdef CONFIG_SCHED_BORE ++struct sched_burst_cache { ++ u8 score; ++ u32 count; ++ u64 timestamp; ++}; ++#endif // CONFIG_SCHED_BORE ++ + struct sched_entity { + /* For load-balancing: */ + struct load_weight load; +@@ -543,12 +551,22 @@ struct sched_entity { + u64 min_vruntime; + + struct list_head group_node; +- unsigned int on_rq; ++ unsigned char on_rq; ++ unsigned char rel_deadline; + + u64 exec_start; u64 sum_exec_runtime; u64 prev_sum_exec_runtime; u64 vruntime; @@ -28,18 +57,60 @@ index 76214d7c8..9f65d367b 100644 + u8 curr_burst_penalty; + u8 burst_penalty; + u8 burst_score; -+ u8 child_burst; -+ u32 child_burst_cnt; -+ u64 child_burst_last_cached; ++ struct sched_burst_cache child_burst; ++ struct sched_burst_cache group_burst; +#endif // CONFIG_SCHED_BORE s64 vlag; u64 slice; +diff --git a/include/linux/sched/bore.h b/include/linux/sched/bore.h +new file mode 100644 +index 0000000000..12a613a94f +--- /dev/null ++++ b/include/linux/sched/bore.h +@@ -0,0 +1,37 @@ ++ ++#include <linux/sched.h> ++#include <linux/sched/cputime.h> ++ ++#ifndef _LINUX_SCHED_BORE_H ++#define _LINUX_SCHED_BORE_H ++ ++#ifdef CONFIG_SCHED_BORE ++extern u8 __read_mostly sched_bore; ++extern u8 __read_mostly sched_burst_exclude_kthreads; ++extern u8 __read_mostly sched_burst_smoothness_long; ++extern u8 __read_mostly sched_burst_smoothness_short; ++extern u8 __read_mostly sched_burst_fork_atavistic; ++extern u8 __read_mostly sched_burst_parity_threshold; ++extern u8 __read_mostly sched_burst_penalty_offset; ++extern uint __read_mostly sched_burst_penalty_scale; ++extern uint __read_mostly sched_burst_cache_lifetime; ++extern uint __read_mostly sched_deadline_boost_mask; ++ ++extern void update_burst_score(struct sched_entity *se); ++extern void update_burst_penalty(struct sched_entity *se); ++ ++extern void restart_burst(struct sched_entity *se); ++extern void restart_burst_rescale_deadline(struct sched_entity *se); ++ ++extern int sched_bore_update_handler(const struct ctl_table *table, int write, ++ void __user *buffer, size_t *lenp, loff_t *ppos); ++ ++extern void sched_clone_bore( ++ struct task_struct *p, struct task_struct *parent, u64 clone_flags); ++ ++extern void init_task_bore(struct task_struct *p); ++ ++extern void reweight_entity( ++ struct cfs_rq *cfs_rq, struct sched_entity *se, unsigned long weight); ++#endif // CONFIG_SCHED_BORE ++#endif // _LINUX_SCHED_BORE_H diff --git a/init/Kconfig b/init/Kconfig -index febdea2af..171b5d995 100644 +index 5783a0b875..b648ed538c 100644 --- a/init/Kconfig +++ b/init/Kconfig -@@ -1283,6 +1283,23 @@ config CHECKPOINT_RESTORE +@@ -1297,6 +1297,23 @@ config CHECKPOINT_RESTORE If unsure, say N here. @@ -64,10 +135,10 @@ index febdea2af..171b5d995 100644 bool "Automatic process group scheduling" select CGROUPS diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz -index 38ef6d068..5f6eecd1e 100644 +index 38ef6d0688..253c566b59 100644 --- a/kernel/Kconfig.hz +++ b/kernel/Kconfig.hz -@@ -55,5 +55,21 @@ config HZ +@@ -55,5 +55,22 @@ config HZ default 300 if HZ_300 default 1000 if HZ_1000 @@ -78,7 +149,8 @@ index 38ef6d068..5f6eecd1e 100644 + The BORE Scheduler automatically calculates the optimal base + slice for the configured HZ using the following equation: + -+ base_slice_ns = max(min_base_slice_ns, 1000000000/HZ) ++ base_slice_ns = ++ 1000000000/HZ * DIV_ROUNDUP(min_base_slice_ns, 1000000000/HZ) + + This option sets the default lower bound limit of the base slice + to prevent the loss of task throughput due to overscheduling. @@ -89,32 +161,188 @@ index 38ef6d068..5f6eecd1e 100644 + config SCHED_HRTICK def_bool HIGH_RES_TIMERS -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index ebf21373f..5d1c97612 100644 ---- a/kernel/sched/core.c -+++ b/kernel/sched/core.c -@@ -4512,6 +4512,138 @@ int wake_up_state(struct task_struct *p, unsigned int state) - return try_to_wake_up(p, state, 0); - } +diff --git a/kernel/fork.c b/kernel/fork.c +index cc760491f2..179b884da3 100644 +--- a/kernel/fork.c ++++ b/kernel/fork.c +@@ -111,6 +111,8 @@ + #include <asm/cacheflush.h> + #include <asm/tlbflush.h> + ++#include <linux/sched/bore.h> ++ + #include <trace/events/sched.h> + + #define CREATE_TRACE_POINTS +@@ -2344,6 +2346,9 @@ __latent_entropy struct task_struct *copy_process( + retval = sched_fork(clone_flags, p); + if (retval) + goto bad_fork_cleanup_policy; ++#ifdef CONFIG_SCHED_BORE ++ sched_clone_bore(p, current, clone_flags); ++#endif // CONFIG_SCHED_BORE + retval = perf_event_init_task(p, clone_flags); + if (retval) +diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile +index 976092b7bd..293aad6754 100644 +--- a/kernel/sched/Makefile ++++ b/kernel/sched/Makefile +@@ -32,3 +32,4 @@ obj-y += core.o + obj-y += fair.o + obj-y += build_policy.o + obj-y += build_utility.o ++obj-y += bore.o +diff --git a/kernel/sched/bore.c b/kernel/sched/bore.c +new file mode 100644 +index 0000000000..62a0191a32 +--- /dev/null ++++ b/kernel/sched/bore.c +@@ -0,0 +1,380 @@ ++/* ++ * Burst-Oriented Response Enhancer (BORE) CPU Scheduler ++ * Copyright (C) 2021-2024 Masahito Suzuki <firelzrd@gmail.com> ++ */ ++#include <linux/cpuset.h> ++#include <linux/sched/bore.h> ++#include "sched.h" ++ +#ifdef CONFIG_SCHED_BORE -+extern u8 sched_burst_fork_atavistic; -+extern uint sched_burst_cache_lifetime; -+ -+static void __init sched_init_bore(void) { -+ init_task.se.burst_time = 0; -+ init_task.se.prev_burst_penalty = 0; -+ init_task.se.curr_burst_penalty = 0; -+ init_task.se.burst_penalty = 0; -+ init_task.se.burst_score = 0; -+ init_task.se.child_burst_last_cached = 0; ++u8 __read_mostly sched_bore = 1; ++u8 __read_mostly sched_burst_exclude_kthreads = 1; ++u8 __read_mostly sched_burst_smoothness_long = 1; ++u8 __read_mostly sched_burst_smoothness_short = 0; ++u8 __read_mostly sched_burst_fork_atavistic = 2; ++u8 __read_mostly sched_burst_parity_threshold = 2; ++u8 __read_mostly sched_burst_penalty_offset = 24; ++uint __read_mostly sched_burst_penalty_scale = 1280; ++uint __read_mostly sched_burst_cache_lifetime = 60000000; ++uint __read_mostly sched_deadline_boost_mask = ENQUEUE_INITIAL ++ | ENQUEUE_WAKEUP; ++static int __maybe_unused sixty_four = 64; ++static int __maybe_unused maxval_u8 = 255; ++static int __maybe_unused maxval_12_bits = 4095; ++ ++#define MAX_BURST_PENALTY (39U <<2) ++ ++static inline u32 log2plus1_u64_u32f8(u64 v) { ++ u32 integral = fls64(v); ++ u8 fractional = v << (64 - integral) >> 55; ++ return integral << 8 | fractional; +} + -+inline void sched_fork_bore(struct task_struct *p) { -+ p->se.burst_time = 0; -+ p->se.curr_burst_penalty = 0; -+ p->se.burst_score = 0; -+ p->se.child_burst_last_cached = 0; ++static inline u32 calc_burst_penalty(u64 burst_time) { ++ u32 greed, tolerance, penalty, scaled_penalty; ++ ++ greed = log2plus1_u64_u32f8(burst_time); ++ tolerance = sched_burst_penalty_offset << 8; ++ penalty = max(0, (s32)(greed - tolerance)); ++ scaled_penalty = penalty * sched_burst_penalty_scale >> 16; ++ ++ return min(MAX_BURST_PENALTY, scaled_penalty); ++} ++ ++static inline u64 __scale_slice(u64 delta, u8 score) ++{return mul_u64_u32_shr(delta, sched_prio_to_wmult[score], 22);} ++ ++static inline u64 __unscale_slice(u64 delta, u8 score) ++{return mul_u64_u32_shr(delta, sched_prio_to_weight[score], 10);} ++ ++static void reweight_task_by_prio(struct task_struct *p, int prio) { ++ struct sched_entity *se = &p->se; ++ unsigned long weight = scale_load(sched_prio_to_weight[prio]); ++ ++ reweight_entity(cfs_rq_of(se), se, weight); ++ se->load.inv_weight = sched_prio_to_wmult[prio]; ++} ++ ++static inline u8 effective_prio(struct task_struct *p) { ++ u8 prio = p->static_prio - MAX_RT_PRIO; ++ if (likely(sched_bore)) ++ prio += p->se.burst_score; ++ return min(39, prio); ++} ++ ++void update_burst_score(struct sched_entity *se) { ++ if (!entity_is_task(se)) return; ++ struct task_struct *p = task_of(se); ++ u8 prev_prio = effective_prio(p); ++ ++ u8 burst_score = 0; ++ if (!((p->flags & PF_KTHREAD) && likely(sched_burst_exclude_kthreads))) ++ burst_score = se->burst_penalty >> 2; ++ se->burst_score = burst_score; ++ ++ u8 new_prio = effective_prio(p); ++ if (new_prio != prev_prio) ++ reweight_task_by_prio(p, new_prio); ++} ++ ++void update_burst_penalty(struct sched_entity *se) { ++ se->curr_burst_penalty = calc_burst_penalty(se->burst_time); ++ se->burst_penalty = max(se->prev_burst_penalty, se->curr_burst_penalty); ++ update_burst_score(se); ++} ++ ++static inline u32 binary_smooth(u32 new, u32 old) { ++ int increment = new - old; ++ return (0 <= increment)? ++ old + ( increment >> (int)sched_burst_smoothness_long): ++ old - (-increment >> (int)sched_burst_smoothness_short); ++} ++ ++static void revolve_burst_penalty(struct sched_entity *se) { ++ se->prev_burst_penalty = ++ binary_smooth(se->curr_burst_penalty, se->prev_burst_penalty); ++ se->burst_time = 0; ++ se->curr_burst_penalty = 0; ++} ++ ++inline void restart_burst(struct sched_entity *se) { ++ revolve_burst_penalty(se); ++ se->burst_penalty = se->prev_burst_penalty; ++ update_burst_score(se); ++} ++ ++void restart_burst_rescale_deadline(struct sched_entity *se) { ++ s64 vscaled, wremain, vremain = se->deadline - se->vruntime; ++ struct task_struct *p = task_of(se); ++ u8 prev_prio = effective_prio(p); ++ restart_burst(se); ++ u8 new_prio = effective_prio(p); ++ if (prev_prio > new_prio) { ++ wremain = __unscale_slice(abs(vremain), prev_prio); ++ vscaled = __scale_slice(wremain, new_prio); ++ if (unlikely(vremain < 0)) ++ vscaled = -vscaled; ++ se->deadline = se->vruntime + vscaled; ++ } ++} ++ ++static void reset_task_weights_bore(void) { ++ struct task_struct *task; ++ struct rq *rq; ++ struct rq_flags rf; ++ ++ write_lock_irq(&tasklist_lock); ++ for_each_process(task) { ++ rq = task_rq(task); ++ rq_lock_irqsave(rq, &rf); ++ reweight_task_by_prio(task, effective_prio(task)); ++ rq_unlock_irqrestore(rq, &rf); ++ } ++ write_unlock_irq(&tasklist_lock); ++} ++ ++int sched_bore_update_handler(const struct ctl_table *table, int write, ++ void __user *buffer, size_t *lenp, loff_t *ppos) { ++ int ret = proc_dou8vec_minmax(table, write, buffer, lenp, ppos); ++ if (ret || !write) ++ return ret; ++ ++ reset_task_weights_bore(); ++ ++ return 0; +} + +static u32 count_child_tasks(struct task_struct *p) { @@ -124,52 +352,45 @@ index ebf21373f..5d1c97612 100644 + return cnt; +} + -+static inline bool task_is_inheritable(struct task_struct *p) { -+ return (p->sched_class == &fair_sched_class); -+} ++static inline bool task_is_bore_eligible(struct task_struct *p) ++{return p->sched_class == &fair_sched_class;} + -+static inline bool child_burst_cache_expired(struct task_struct *p, u64 now) { -+ u64 expiration_time = -+ p->se.child_burst_last_cached + sched_burst_cache_lifetime; -+ return ((s64)(expiration_time - now) < 0); -+} ++static inline bool burst_cache_expired(struct sched_burst_cache *bc, u64 now) ++{return (s64)(bc->timestamp + sched_burst_cache_lifetime - now) < 0;} + -+static void __update_child_burst_cache( -+ struct task_struct *p, u32 cnt, u32 sum, u64 now) { -+ u8 avg = 0; -+ if (cnt) avg = sum / cnt; -+ p->se.child_burst = max(avg, p->se.burst_penalty); -+ p->se.child_burst_cnt = cnt; -+ p->se.child_burst_last_cached = now; ++static void update_burst_cache(struct sched_burst_cache *bc, ++ struct task_struct *p, u32 cnt, u32 sum, u64 now) { ++ u8 avg = cnt ? sum / cnt : 0; ++ bc->score = max(avg, p->se.burst_penalty); ++ bc->count = cnt; ++ bc->timestamp = now; +} + +static inline void update_child_burst_direct(struct task_struct *p, u64 now) { ++ u32 cnt = 0, sum = 0; + struct task_struct *child; -+ u32 cnt = 0; -+ u32 sum = 0; + + list_for_each_entry(child, &p->children, sibling) { -+ if (!task_is_inheritable(child)) continue; ++ if (!task_is_bore_eligible(child)) continue; + cnt++; + sum += child->se.burst_penalty; + } + -+ __update_child_burst_cache(p, cnt, sum, now); ++ update_burst_cache(&p->se.child_burst, p, cnt, sum, now); +} + -+static inline u8 __inherit_burst_direct(struct task_struct *p, u64 now) { -+ struct task_struct *parent = p->real_parent; -+ if (child_burst_cache_expired(parent, now)) ++static inline u8 inherit_burst_direct(struct task_struct *p, u64 now) { ++ struct task_struct *parent = p; ++ if (burst_cache_expired(&parent->se.child_burst, now)) + update_child_burst_direct(parent, now); + -+ return parent->se.child_burst; ++ return parent->se.child_burst.score; +} + +static void update_child_burst_topological( + struct task_struct *p, u64 now, u32 depth, u32 *acnt, u32 *asum) { ++ u32 cnt = 0, dcnt = 0, sum = 0; + struct task_struct *child, *dec; -+ u32 cnt = 0, dcnt = 0; -+ u32 sum = 0; + + list_for_each_entry(child, &p->children, sibling) { + dec = child; @@ -177,95 +398,214 @@ index ebf21373f..5d1c97612 100644 + dec = list_first_entry(&dec->children, struct task_struct, sibling); + + if (!dcnt || !depth) { -+ if (!task_is_inheritable(dec)) continue; ++ if (!task_is_bore_eligible(dec)) continue; + cnt++; + sum += dec->se.burst_penalty; + continue; + } -+ if (!child_burst_cache_expired(dec, now)) { -+ cnt += dec->se.child_burst_cnt; -+ sum += (u32)dec->se.child_burst * dec->se.child_burst_cnt; ++ if (!burst_cache_expired(&dec->se.child_burst, now)) { ++ cnt += dec->se.child_burst.count; ++ sum += (u32)dec->se.child_burst.score * dec->se.child_burst.count; + continue; + } + update_child_burst_topological(dec, now, depth - 1, &cnt, &sum); + } + -+ __update_child_burst_cache(p, cnt, sum, now); ++ update_burst_cache(&p->se.child_burst, p, cnt, sum, now); + *acnt += cnt; + *asum += sum; +} + -+static inline u8 __inherit_burst_topological(struct task_struct *p, u64 now) { -+ struct task_struct *anc = p->real_parent; ++static inline u8 inherit_burst_topological(struct task_struct *p, u64 now) { ++ struct task_struct *anc = p; + u32 cnt = 0, sum = 0; + + while (anc->real_parent != anc && count_child_tasks(anc) == 1) + anc = anc->real_parent; + -+ if (child_burst_cache_expired(anc, now)) ++ if (burst_cache_expired(&anc->se.child_burst, now)) + update_child_burst_topological( + anc, now, sched_burst_fork_atavistic - 1, &cnt, &sum); + -+ return anc->se.child_burst; ++ return anc->se.child_burst.score; +} + -+static inline void inherit_burst(struct task_struct *p) { -+ u8 burst_cache; -+ u64 now = ktime_get_ns(); ++static inline void update_tg_burst(struct task_struct *p, u64 now) { ++ struct task_struct *task; ++ u32 cnt = 0, sum = 0; + ++ for_each_thread(p, task) { ++ if (!task_is_bore_eligible(task)) continue; ++ cnt++; ++ sum += task->se.burst_penalty; ++ } ++ ++ update_burst_cache(&p->se.group_burst, p, cnt, sum, now); ++} ++ ++static inline u8 inherit_burst_tg(struct task_struct *p, u64 now) { ++ struct task_struct *parent = p->group_leader; ++ if (burst_cache_expired(&parent->se.group_burst, now)) ++ update_tg_burst(parent, now); ++ ++ return parent->se.group_burst.score; ++} ++ ++void sched_clone_bore( ++ struct task_struct *p, struct task_struct *parent, u64 clone_flags) { ++ if (!task_is_bore_eligible(p)) return; ++ ++ u64 now = ktime_get_ns(); + read_lock(&tasklist_lock); -+ burst_cache = likely(sched_burst_fork_atavistic)? -+ __inherit_burst_topological(p, now): -+ __inherit_burst_direct(p, now); ++ u8 penalty = (clone_flags & CLONE_THREAD) ? ++ inherit_burst_tg(parent, now) : ++ likely(sched_burst_fork_atavistic) ? ++ inherit_burst_topological(parent, now): ++ inherit_burst_direct(parent, now); + read_unlock(&tasklist_lock); + -+ p->se.prev_burst_penalty = max(p->se.prev_burst_penalty, burst_cache); ++ struct sched_entity *se = &p->se; ++ revolve_burst_penalty(se); ++ se->burst_penalty = se->prev_burst_penalty = ++ max(se->prev_burst_penalty, penalty); ++ se->child_burst.timestamp = 0; ++ se->group_burst.timestamp = 0; +} + -+static void sched_post_fork_bore(struct task_struct *p) { -+ if (p->sched_class == &fair_sched_class) -+ inherit_burst(p); -+ p->se.burst_penalty = p->se.prev_burst_penalty; ++void init_task_bore(struct task_struct *p) { ++ p->se.burst_time = 0; ++ p->se.prev_burst_penalty = 0; ++ p->se.curr_burst_penalty = 0; ++ p->se.burst_penalty = 0; ++ p->se.burst_score = 0; ++ memset(&p->se.child_burst, 0, sizeof(struct sched_burst_cache)); ++ memset(&p->se.group_burst, 0, sizeof(struct sched_burst_cache)); +} -+#endif // CONFIG_SCHED_BORE + - /* - * Perform scheduler related setup for a newly forked process p. - * p is forked by current. -@@ -4528,6 +4660,9 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p) - p->se.prev_sum_exec_runtime = 0; - p->se.nr_migrations = 0; - p->se.vruntime = 0; -+#ifdef CONFIG_SCHED_BORE -+ sched_fork_bore(p); ++#ifdef CONFIG_SYSCTL ++static struct ctl_table sched_bore_sysctls[] = { ++ { ++ .procname = "sched_bore", ++ .data = &sched_bore, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = sched_bore_update_handler, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_ONE, ++ }, ++ { ++ .procname = "sched_burst_exclude_kthreads", ++ .data = &sched_burst_exclude_kthreads, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_ONE, ++ }, ++ { ++ .procname = "sched_burst_smoothness_long", ++ .data = &sched_burst_smoothness_long, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_ONE, ++ }, ++ { ++ .procname = "sched_burst_smoothness_short", ++ .data = &sched_burst_smoothness_short, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_ONE, ++ }, ++ { ++ .procname = "sched_burst_fork_atavistic", ++ .data = &sched_burst_fork_atavistic, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = SYSCTL_THREE, ++ }, ++ { ++ .procname = "sched_burst_parity_threshold", ++ .data = &sched_burst_parity_threshold, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = &maxval_u8, ++ }, ++ { ++ .procname = "sched_burst_penalty_offset", ++ .data = &sched_burst_penalty_offset, ++ .maxlen = sizeof(u8), ++ .mode = 0644, ++ .proc_handler = proc_dou8vec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = &sixty_four, ++ }, ++ { ++ .procname = "sched_burst_penalty_scale", ++ .data = &sched_burst_penalty_scale, ++ .maxlen = sizeof(uint), ++ .mode = 0644, ++ .proc_handler = proc_douintvec_minmax, ++ .extra1 = SYSCTL_ZERO, ++ .extra2 = &maxval_12_bits, ++ }, ++ { ++ .procname = "sched_burst_cache_lifetime", ++ .data = &sched_burst_cache_lifetime, ++ .maxlen = sizeof(uint), ++ .mode = 0644, ++ .proc_handler = proc_douintvec, ++ }, ++ { ++ .procname = "sched_deadline_boost_mask", ++ .data = &sched_deadline_boost_mask, ++ .maxlen = sizeof(uint), ++ .mode = 0644, ++ .proc_handler = proc_douintvec, ++ }, ++}; ++ ++static int __init sched_bore_sysctl_init(void) { ++ register_sysctl_init("kernel", sched_bore_sysctls); ++ return 0; ++} ++late_initcall(sched_bore_sysctl_init); ++#endif // CONFIG_SYSCTL +#endif // CONFIG_SCHED_BORE - p->se.vlag = 0; - p->se.slice = sysctl_sched_base_slice; - INIT_LIST_HEAD(&p->se.group_node); -@@ -4843,6 +4978,9 @@ void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs) +diff --git a/kernel/sched/core.c b/kernel/sched/core.c +index f3951e4a55..9264e542c9 100644 +--- a/kernel/sched/core.c ++++ b/kernel/sched/core.c +@@ -97,6 +97,8 @@ + #include "../../io_uring/io-wq.h" + #include "../smpboot.h" - void sched_post_fork(struct task_struct *p) - { -+#ifdef CONFIG_SCHED_BORE -+ sched_post_fork_bore(p); -+#endif // CONFIG_SCHED_BORE - uclamp_post_fork(p); - } ++#include <linux/sched/bore.h> ++ + EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpu); + EXPORT_TRACEPOINT_SYMBOL_GPL(ipi_send_cpumask); -@@ -9930,6 +10068,11 @@ void __init sched_init(void) +@@ -8197,6 +8199,11 @@ void __init sched_init(void) BUG_ON(&dl_sched_class != &stop_sched_class + 1); #endif +#ifdef CONFIG_SCHED_BORE -+ sched_init_bore(); -+ printk(KERN_INFO "BORE (Burst-Oriented Response Enhancer) CPU Scheduler modification 5.2.8 by Masahito Suzuki"); ++ printk(KERN_INFO "BORE (Burst-Oriented Response Enhancer) CPU Scheduler modification 5.6.0 by Masahito Suzuki"); ++ init_task_bore(&init_task); +#endif // CONFIG_SCHED_BORE + wait_bit_init(); #ifdef CONFIG_FAIR_GROUP_SCHED diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c -index c1eb9a1af..e2da8d773 100644 +index c1eb9a1afd..e2da8d7738 100644 --- a/kernel/sched/debug.c +++ b/kernel/sched/debug.c @@ -167,7 +167,52 @@ static const struct file_operations sched_feat_fops = { @@ -372,21 +712,20 @@ index c1eb9a1af..e2da8d773 100644 P(se.avg.runnable_sum); P(se.avg.util_sum); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c -index 483c137b9..4c8d7fbd5 100644 +index 9057584ec0..465d2626ee 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c -@@ -19,6 +19,9 @@ +@@ -55,6 +55,8 @@ + #include "stats.h" + #include "autogroup.h" + ++#include <linux/sched/bore.h> ++ + /* + * The initial- and re-scaling of tunables is configurable * - * Adaptive scheduling granularity, math enhancements by Peter Zijlstra - * Copyright (C) 2007 Red Hat, Inc., Peter Zijlstra -+ * -+ * Burst-Oriented Response Enhancer (BORE) CPU Scheduler -+ * Copyright (C) 2021-2024 Masahito Suzuki <firelzrd@gmail.com> - */ - #include <linux/energy_model.h> - #include <linux/mmap_lock.h> -@@ -64,20 +67,146 @@ - * SCHED_TUNABLESCALING_LOG - scaled logarithmical, *1+ilog(ncpus) +@@ -64,17 +66,29 @@ + * SCHED_TUNABLESCALING_LOG - scaled logarithmically, *1+ilog(ncpus) * SCHED_TUNABLESCALING_LINEAR - scaled linear, *ncpus * - * (default SCHED_TUNABLESCALING_LOG = *(1+ilog(ncpus)) @@ -417,231 +756,26 @@ index 483c137b9..4c8d7fbd5 100644 const_debug unsigned int sysctl_sched_migration_cost = 500000UL; -+#ifdef CONFIG_SCHED_BORE -+u8 __read_mostly sched_bore = 1; -+u8 __read_mostly sched_burst_exclude_kthreads = 1; -+u8 __read_mostly sched_burst_smoothness_long = 1; -+u8 __read_mostly sched_burst_smoothness_short = 0; -+u8 __read_mostly sched_burst_fork_atavistic = 2; -+u8 __read_mostly sched_burst_penalty_offset = 22; -+uint __read_mostly sched_burst_penalty_scale = 1280; -+uint __read_mostly sched_burst_cache_lifetime = 60000000; -+uint __read_mostly sched_deadline_boost_mask = ENQUEUE_INITIAL -+ | ENQUEUE_WAKEUP; -+uint __read_mostly sched_deadline_preserve_mask = ENQUEUE_RESTORE -+ | ENQUEUE_MIGRATED; -+static int __maybe_unused sixty_four = 64; -+static int __maybe_unused maxval_12_bits = 4095; -+ -+#define MAX_BURST_PENALTY (39U <<2) -+ -+static inline u32 log2plus1_u64_u32f8(u64 v) { -+ u32 msb = fls64(v); -+ s32 excess_bits = msb - 9; -+ u8 fractional = (0 <= excess_bits)? v >> excess_bits: v << -excess_bits; -+ return msb << 8 | fractional; -+} -+ -+static inline u32 calc_burst_penalty(u64 burst_time) { -+ u32 greed, tolerance, penalty, scaled_penalty; -+ -+ greed = log2plus1_u64_u32f8(burst_time); -+ tolerance = sched_burst_penalty_offset << 8; -+ penalty = max(0, (s32)greed - (s32)tolerance); -+ scaled_penalty = penalty * sched_burst_penalty_scale >> 16; -+ -+ return min(MAX_BURST_PENALTY, scaled_penalty); -+} -+ -+static inline u64 scale_slice(u64 delta, struct sched_entity *se) { -+ return mul_u64_u32_shr(delta, sched_prio_to_wmult[se->burst_score], 22); -+} -+ -+static inline u64 __unscale_slice(u64 delta, u8 score) { -+ return mul_u64_u32_shr(delta, sched_prio_to_weight[score], 10); -+} -+ -+static inline u64 unscale_slice(u64 delta, struct sched_entity *se) { -+ return __unscale_slice(delta, se->burst_score); -+} -+ -+static void reweight_entity( -+ struct cfs_rq *cfs_rq, struct sched_entity *se, unsigned long weight); -+ -+static void renice_task(struct task_struct *p, int prio) -+{ -+ struct sched_entity *se = &p->se; -+ struct cfs_rq *cfs_rq = cfs_rq_of(se); -+ struct load_weight *load = &se->load; -+ unsigned long weight = scale_load(sched_prio_to_weight[prio]); -+ -+ reweight_entity(cfs_rq, se, weight); -+ load->inv_weight = sched_prio_to_wmult[prio]; -+} -+ -+static void update_burst_score(struct sched_entity *se) { -+ if (!entity_is_task(se)) return; -+ struct task_struct *p = task_of(se); -+ u8 prio = p->static_prio - MAX_RT_PRIO; -+ u8 prev_prio = min(39, prio + se->burst_score); -+ -+ u8 burst_score = 0; -+ if (!(sched_burst_exclude_kthreads && (p->flags & PF_KTHREAD))) -+ burst_score = se->burst_penalty >> 2; -+ -+ se->burst_score = burst_score; -+ -+ u8 new_prio = min(39, prio + se->burst_score); -+ if (new_prio != prev_prio) -+ renice_task(p, new_prio); -+} -+ -+static void update_burst_penalty(struct sched_entity *se) { -+ se->curr_burst_penalty = calc_burst_penalty(se->burst_time); -+ se->burst_penalty = max(se->prev_burst_penalty, se->curr_burst_penalty); -+ update_burst_score(se); -+} -+ -+static inline u32 binary_smooth(u32 new, u32 old) { -+ int increment = new - old; -+ return (0 <= increment)? -+ old + ( increment >> (int)sched_burst_smoothness_long): -+ old - (-increment >> (int)sched_burst_smoothness_short); -+} -+ -+static void restart_burst(struct sched_entity *se) { -+ se->burst_penalty = se->prev_burst_penalty = -+ binary_smooth(se->curr_burst_penalty, se->prev_burst_penalty); -+ se->curr_burst_penalty = 0; -+ se->burst_time = 0; -+ update_burst_score(se); -+} -+ -+static void restart_burst_rescale_deadline(struct sched_entity *se) { -+ s64 vscaled, wremain, vremain = se->deadline - se->vruntime; -+ u8 prev_score = se->burst_score; -+ restart_burst(se); -+ if (prev_score > se->burst_score) { -+ wremain = __unscale_slice(abs(vremain), prev_score); -+ vscaled = scale_slice(wremain, se); -+ if (unlikely(vremain < 0)) -+ vscaled = -vscaled; -+ se->deadline = se->vruntime + vscaled; -+ } -+} -+#endif // CONFIG_SCHED_BORE -+ - static int __init setup_sched_thermal_decay_shift(char *str) - { - pr_warn("Ignoring the deprecated sched_thermal_decay_shift= option\n"); -@@ -131,6 +260,92 @@ static unsigned int sysctl_numa_balancing_promote_rate_limit = 65536; - - #ifdef CONFIG_SYSCTL - static struct ctl_table sched_fair_sysctls[] = { -+#ifdef CONFIG_SCHED_BORE -+ { -+ .procname = "sched_bore", -+ .data = &sched_bore, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ONE, -+ .extra2 = SYSCTL_ONE, -+ }, -+ { -+ .procname = "sched_burst_exclude_kthreads", -+ .data = &sched_burst_exclude_kthreads, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE, -+ }, -+ { -+ .procname = "sched_burst_smoothness_long", -+ .data = &sched_burst_smoothness_long, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE, -+ }, -+ { -+ .procname = "sched_burst_smoothness_short", -+ .data = &sched_burst_smoothness_short, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_ONE, -+ }, -+ { -+ .procname = "sched_burst_fork_atavistic", -+ .data = &sched_burst_fork_atavistic, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = SYSCTL_THREE, -+ }, -+ { -+ .procname = "sched_burst_penalty_offset", -+ .data = &sched_burst_penalty_offset, -+ .maxlen = sizeof(u8), -+ .mode = 0644, -+ .proc_handler = proc_dou8vec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = &sixty_four, -+ }, -+ { -+ .procname = "sched_burst_penalty_scale", -+ .data = &sched_burst_penalty_scale, -+ .maxlen = sizeof(uint), -+ .mode = 0644, -+ .proc_handler = proc_douintvec_minmax, -+ .extra1 = SYSCTL_ZERO, -+ .extra2 = &maxval_12_bits, -+ }, -+ { -+ .procname = "sched_burst_cache_lifetime", -+ .data = &sched_burst_cache_lifetime, -+ .maxlen = sizeof(uint), -+ .mode = 0644, -+ .proc_handler = proc_douintvec, -+ }, -+ { -+ .procname = "sched_deadline_boost_mask", -+ .data = &sched_deadline_boost_mask, -+ .maxlen = sizeof(uint), -+ .mode = 0644, -+ .proc_handler = proc_douintvec, -+ }, -+ { -+ .procname = "sched_deadline_preserve_mask", -+ .data = &sched_deadline_preserve_mask, -+ .maxlen = sizeof(uint), -+ .mode = 0644, -+ .proc_handler = proc_douintvec, -+ }, -+#endif // CONFIG_SCHED_BORE - #ifdef CONFIG_CFS_BANDWIDTH - { - .procname = "sched_cfs_bandwidth_slice_us", -@@ -188,6 +403,13 @@ static inline void update_load_set(struct load_weight *lw, unsigned long w) +@@ -188,6 +202,18 @@ static inline void update_load_set(struct load_weight *lw, unsigned long w) * * This idea comes from the SD scheduler of Con Kolivas: */ +#ifdef CONFIG_SCHED_BORE +static void update_sysctl(void) { -+ sysctl_sched_base_slice = -+ max(sysctl_sched_min_base_slice, configured_sched_base_slice); ++ unsigned int base_slice = configured_sched_base_slice; ++ unsigned int min_base_slice = sysctl_sched_min_base_slice; ++ ++ if (min_base_slice) ++ base_slice *= DIV_ROUND_UP(min_base_slice, base_slice); ++ ++ sysctl_sched_base_slice = base_slice; +} +void sched_update_min_base_slice(void) { update_sysctl(); } +#else // !CONFIG_SCHED_BORE static unsigned int get_update_sysctl_factor(void) { unsigned int cpus = min_t(unsigned int, num_online_cpus(), 8); -@@ -218,6 +440,7 @@ static void update_sysctl(void) +@@ -218,6 +244,7 @@ static void update_sysctl(void) SET_SYSCTL(sched_base_slice); #undef SET_SYSCTL } @@ -649,93 +783,28 @@ index 483c137b9..4c8d7fbd5 100644 void __init sched_init_granularity(void) { -@@ -695,6 +918,9 @@ static s64 entity_lag(u64 avruntime, struct sched_entity *se) +@@ -695,6 +722,9 @@ static s64 entity_lag(u64 avruntime, struct sched_entity *se) vlag = avruntime - se->vruntime; limit = calc_delta_fair(max_t(u64, 2*se->slice, TICK_NSEC), se); +#ifdef CONFIG_SCHED_BORE -+ limit >>= 1; ++ limit >>= !!sched_bore; +#endif // CONFIG_SCHED_BORE return clamp(vlag, -limit, limit); } -@@ -855,6 +1081,39 @@ struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) - return __node_2_se(left); - } - -+static inline bool pick_curr(struct cfs_rq *cfs_rq, -+ struct sched_entity *curr, struct sched_entity *wakee) -+{ -+ /* -+ * Nothing to preserve... -+ */ -+ if (!curr || !sched_feat(RESPECT_SLICE)) -+ return false; -+ -+ /* -+ * Allow preemption at the 0-lag point -- even if not all of the slice -+ * is consumed. Note: placement of positive lag can push V left and render -+ * @curr instantly ineligible irrespective the time on-cpu. -+ */ -+ if (sched_feat(RUN_TO_PARITY) && !entity_eligible(cfs_rq, curr)) -+ return false; -+ -+ /* -+ * Don't preserve @curr when the @wakee has a shorter slice and earlier -+ * deadline. IOW, explicitly allow preemption. -+ */ -+ if (sched_feat(PREEMPT_SHORT) && wakee && -+ wakee->slice < curr->slice && -+ (s64)(wakee->deadline - curr->deadline) < 0) -+ return false; -+ -+ /* -+ * Preserve @curr to allow it to finish its first slice. -+ * See the HACK in set_next_entity(). -+ */ -+ return curr->vlag == curr->deadline; -+} -+ - /* - * Earliest Eligible Virtual Deadline First - * -@@ -874,28 +1133,27 @@ struct sched_entity *__pick_first_entity(struct cfs_rq *cfs_rq) - * - * Which allows tree pruning through eligibility. - */ --static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) -+static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq, struct sched_entity *wakee) - { - struct rb_node *node = cfs_rq->tasks_timeline.rb_root.rb_node; - struct sched_entity *se = __pick_first_entity(cfs_rq); - struct sched_entity *curr = cfs_rq->curr; - struct sched_entity *best = NULL; - -+ if (curr && !curr->on_rq) -+ curr = NULL; -+ - /* - * We can safely skip eligibility check if there is only one entity - * in this cfs_rq, saving some cycles. - */ - if (cfs_rq->nr_running == 1) -- return curr && curr->on_rq ? curr : se; -- -- if (curr && (!curr->on_rq || !entity_eligible(cfs_rq, curr))) -- curr = NULL; -+ return curr ?: se; - - /* -- * Once selected, run a task until it either becomes non-eligible or -- * until it gets a new slice. See the HACK in set_next_entity(). -+ * Preserve @curr to let it finish its slice. +@@ -896,6 +926,10 @@ static struct sched_entity *pick_eevdf(struct cfs_rq *cfs_rq) + * until it gets a new slice. See the HACK in set_next_entity(). */ -- if (sched_feat(RUN_TO_PARITY) && curr && curr->vlag == curr->deadline) -+ if (pick_curr(cfs_rq, curr, wakee)) + if (sched_feat(RUN_TO_PARITY) && curr && curr->vlag == curr->deadline) ++#ifdef CONFIG_SCHED_BORE ++ if (!(likely(sched_bore) && likely(sched_burst_parity_threshold) && ++ sched_burst_parity_threshold < cfs_rq->nr_running)) ++#endif // CONFIG_SCHED_BORE return curr; /* Pick the leftmost entity if it's eligible */ -@@ -954,6 +1212,7 @@ struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) +@@ -954,6 +988,7 @@ struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) * Scheduling class statistics methods: */ #ifdef CONFIG_SMP @@ -743,7 +812,7 @@ index 483c137b9..4c8d7fbd5 100644 int sched_update_scaling(void) { unsigned int factor = get_update_sysctl_factor(); -@@ -965,6 +1224,7 @@ int sched_update_scaling(void) +@@ -965,6 +1000,7 @@ int sched_update_scaling(void) return 0; } @@ -751,33 +820,27 @@ index 483c137b9..4c8d7fbd5 100644 #endif #endif -@@ -1165,7 +1425,13 @@ static void update_curr(struct cfs_rq *cfs_rq) +@@ -1165,6 +1201,10 @@ static void update_curr(struct cfs_rq *cfs_rq) if (unlikely(delta_exec <= 0)) return; +#ifdef CONFIG_SCHED_BORE + curr->burst_time += delta_exec; + update_burst_penalty(curr); -+ curr->vruntime += max(1ULL, calc_delta_fair(delta_exec, curr)); -+#else // !CONFIG_SCHED_BORE - curr->vruntime += calc_delta_fair(delta_exec, curr); +#endif // CONFIG_SCHED_BORE + curr->vruntime += calc_delta_fair(delta_exec, curr); update_deadline(cfs_rq, curr); update_min_vruntime(cfs_rq); +@@ -3782,7 +3822,7 @@ static void reweight_eevdf(struct sched_entity *se, u64 avruntime, + se->deadline = avruntime + vslice; + } -@@ -5179,6 +5445,11 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) - s64 lag = 0; - - se->slice = sysctl_sched_base_slice; -+#ifdef CONFIG_SCHED_BORE -+ if (flags & ~sched_deadline_boost_mask & sched_deadline_preserve_mask) -+ vslice = se->deadline - se->vruntime; -+ else -+#endif // CONFIG_SCHED_BORE - vslice = calc_delta_fair(se->slice, se); - - /* -@@ -5189,6 +5460,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) +-static void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, ++void reweight_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, + unsigned long weight) + { + bool curr = cfs_rq->curr == se; +@@ -5189,6 +5229,9 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) * * EEVDF: placement strategy #1 / #2 */ @@ -787,28 +850,44 @@ index 483c137b9..4c8d7fbd5 100644 if (sched_feat(PLACE_LAG) && cfs_rq->nr_running) { struct sched_entity *curr = cfs_rq->curr; unsigned long load; -@@ -5264,7 +5538,11 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) - * on average, halfway through their slice, as such start tasks - * off with half a slice to ease into the competition. - */ -+#if !defined(CONFIG_SCHED_BORE) - if (sched_feat(PLACE_DEADLINE_INITIAL) && (flags & ENQUEUE_INITIAL)) -+#else // CONFIG_SCHED_BORE -+ if (flags & sched_deadline_boost_mask) -+#endif // CONFIG_SCHED_BORE - vslice /= 2; +@@ -5259,6 +5302,16 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) + + se->vruntime = vruntime - lag; ++ if (sched_feat(PLACE_REL_DEADLINE) && se->rel_deadline) { ++ se->deadline += se->vruntime; ++ se->rel_deadline = 0; ++ return; ++ } ++#ifdef CONFIG_SCHED_BORE ++ else if (likely(sched_bore)) ++ vslice >>= !!(flags & sched_deadline_boost_mask); ++ else ++#endif // CONFIG_SCHED_BORE /* -@@ -5478,7 +5756,7 @@ pick_next_entity(struct cfs_rq *cfs_rq) - cfs_rq->next && entity_eligible(cfs_rq, cfs_rq->next)) - return cfs_rq->next; + * When joining the competition; the existing tasks will be, + * on average, halfway through their slice, as such start tasks +@@ -5368,6 +5421,7 @@ static __always_inline void return_cfs_rq_runtime(struct cfs_rq *cfs_rq); + static void + dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) + { ++ bool sleep = flags & DEQUEUE_SLEEP; + int action = UPDATE_TG; -- return pick_eevdf(cfs_rq); -+ return pick_eevdf(cfs_rq, NULL); - } + if (entity_is_task(se) && task_on_rq_migrating(task_of(se))) +@@ -5395,6 +5449,11 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int flags) + clear_buddies(cfs_rq, se); - static bool check_cfs_rq_runtime(struct cfs_rq *cfs_rq); -@@ -6846,6 +7124,14 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) + update_entity_lag(cfs_rq, se); ++ if (sched_feat(PLACE_REL_DEADLINE) && !sleep) { ++ se->deadline -= se->vruntime; ++ se->rel_deadline = 1; ++ } ++ + if (se != cfs_rq->curr) + __dequeue_entity(cfs_rq, se); + se->on_rq = 0; +@@ -6846,6 +6905,14 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) bool was_sched_idle = sched_idle_rq(rq); util_est_dequeue(&rq->cfs, p); @@ -823,16 +902,7 @@ index 483c137b9..4c8d7fbd5 100644 for_each_sched_entity(se) { cfs_rq = cfs_rq_of(se); -@@ -8414,7 +8700,7 @@ static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int - /* - * XXX pick_eevdf(cfs_rq) != se ? - */ -- if (pick_eevdf(cfs_rq) == pse) -+ if (pick_eevdf(cfs_rq, pse) == pse) - goto preempt; - - return; -@@ -8632,16 +8918,25 @@ static void yield_task_fair(struct rq *rq) +@@ -8632,16 +8699,25 @@ static void yield_task_fair(struct rq *rq) /* * Are we the only task in the tree? */ @@ -858,7 +928,7 @@ index 483c137b9..4c8d7fbd5 100644 /* * Tell update_rq_clock() that we've just updated, * so we don't do microscopic update in schedule() -@@ -12709,6 +13004,9 @@ static void task_fork_fair(struct task_struct *p) +@@ -12716,6 +12792,9 @@ static void task_fork_fair(struct task_struct *p) curr = cfs_rq->curr; if (curr) update_curr(cfs_rq); @@ -868,46 +938,38 @@ index 483c137b9..4c8d7fbd5 100644 place_entity(cfs_rq, se, ENQUEUE_INITIAL); rq_unlock(rq, &rf); } +@@ -12828,6 +12907,10 @@ static void attach_task_cfs_rq(struct task_struct *p) + + static void switched_from_fair(struct rq *rq, struct task_struct *p) + { ++ p->se.rel_deadline = 0; ++#ifdef CONFIG_SCHED_BORE ++ init_task_bore(p); ++#endif // CONFIG_SCHED_BORE + detach_task_cfs_rq(p); + } + diff --git a/kernel/sched/features.h b/kernel/sched/features.h -index 143f55df8..3aad8900c 100644 +index 143f55df89..e97b7b68bd 100644 --- a/kernel/sched/features.h +++ b/kernel/sched/features.h -@@ -5,8 +5,28 @@ - * sleep+wake cycles. EEVDF placement strategy #1, #2 if disabled. +@@ -6,6 +6,10 @@ */ SCHED_FEAT(PLACE_LAG, true) -+/* -+ * Give new tasks half a slice to ease into the competition. -+ */ -+#if !defined(CONFIG_SCHED_BORE) SCHED_FEAT(PLACE_DEADLINE_INITIAL, true) --SCHED_FEAT(RUN_TO_PARITY, true) -+#endif // CONFIG_SCHED_BORE -+/* -+ * Inhibit (wakeup) preemption until the current task has exhausted its slice. -+ */ -+#ifdef CONFIG_SCHED_BORE -+SCHED_FEAT(RESPECT_SLICE, false) -+#else // !CONFIG_SCHED_BORE -+SCHED_FEAT(RESPECT_SLICE, true) -+#endif // CONFIG_SCHED_BORE +/* -+ * Relax RESPECT_SLICE to allow preemption once current has reached 0-lag. ++ * Preserve relative virtual deadline on 'migration'. + */ -+SCHED_FEAT(RUN_TO_PARITY, false) -+/* -+ * Allow tasks with a shorter slice to disregard RESPECT_SLICE -+ */ -+SCHED_FEAT(PREEMPT_SHORT, true) ++SCHED_FEAT(PLACE_REL_DEADLINE, true) + SCHED_FEAT(RUN_TO_PARITY, true) /* - * Prefer to schedule the task we woke last (assuming it failed diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h -index 38aeedd8a..aa0ae3fb9 100644 +index 4c36cc6803..cc18ad228e 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h -@@ -1969,7 +1969,11 @@ static inline void dirty_sched_domain_sysctl(int cpu) - } +@@ -1984,7 +1984,11 @@ static inline void update_sched_domain_debugfs(void) { } + static inline void dirty_sched_domain_sysctl(int cpu) { } #endif +#ifdef CONFIG_SCHED_BORE @@ -918,7 +980,7 @@ index 38aeedd8a..aa0ae3fb9 100644 static inline const struct cpumask *task_user_cpus(struct task_struct *p) { -@@ -2554,6 +2558,9 @@ extern const_debug unsigned int sysctl_sched_nr_migrate; +@@ -2601,6 +2605,9 @@ extern const_debug unsigned int sysctl_sched_nr_migrate; extern const_debug unsigned int sysctl_sched_migration_cost; extern unsigned int sysctl_sched_base_slice; @@ -929,5 +991,5 @@ index 38aeedd8a..aa0ae3fb9 100644 #ifdef CONFIG_SCHED_DEBUG extern int sysctl_resched_latency_warn_ms; -- -2.46.0 +2.34.1 |