aboutsummaryrefslogtreecommitdiff
path: root/SOURCES/scx-kernel.patch
diff options
context:
space:
mode:
Diffstat (limited to 'SOURCES/scx-kernel.patch')
-rw-r--r--SOURCES/scx-kernel.patch15332
1 files changed, 15332 insertions, 0 deletions
diff --git a/SOURCES/scx-kernel.patch b/SOURCES/scx-kernel.patch
new file mode 100644
index 0000000..7ff03ff
--- /dev/null
+++ b/SOURCES/scx-kernel.patch
@@ -0,0 +1,15332 @@
+From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
+From: Jan200101 <sentrycraft123@gmail.com>
+Date: Wed, 17 Jul 2024 14:30:16 +0200
+Subject: [PATCH] scx
+
+Signed-off-by: Jan200101 <sentrycraft123@gmail.com>
+---
+ Documentation/scheduler/index.rst | 1 +
+ Documentation/scheduler/sched-ext.rst | 316 +
+ MAINTAINERS | 13 +
+ include/asm-generic/vmlinux.lds.h | 1 +
+ include/linux/cgroup.h | 4 +-
+ include/linux/sched.h | 5 +
+ include/linux/sched/ext.h | 206 +
+ include/linux/sched/task.h | 3 +-
+ include/trace/events/sched_ext.h | 32 +
+ include/uapi/linux/sched.h | 1 +
+ init/init_task.c | 12 +
+ kernel/Kconfig.preempt | 26 +-
+ kernel/fork.c | 17 +-
+ kernel/sched/build_policy.c | 10 +
+ kernel/sched/core.c | 250 +-
+ kernel/sched/cpufreq_schedutil.c | 50 +-
+ kernel/sched/debug.c | 3 +
+ kernel/sched/ext.c | 6537 +++++++++++++++++
+ kernel/sched/ext.h | 69 +
+ kernel/sched/fair.c | 27 +-
+ kernel/sched/idle.c | 2 +
+ kernel/sched/sched.h | 171 +-
+ lib/dump_stack.c | 1 +
+ tools/Makefile | 10 +-
+ tools/sched_ext/.gitignore | 2 +
+ tools/sched_ext/Makefile | 246 +
+ tools/sched_ext/README.md | 258 +
+ .../sched_ext/include/bpf-compat/gnu/stubs.h | 11 +
+ tools/sched_ext/include/scx/common.bpf.h | 401 +
+ tools/sched_ext/include/scx/common.h | 75 +
+ tools/sched_ext/include/scx/compat.bpf.h | 28 +
+ tools/sched_ext/include/scx/compat.h | 187 +
+ tools/sched_ext/include/scx/user_exit_info.h | 111 +
+ tools/sched_ext/scx_central.bpf.c | 361 +
+ tools/sched_ext/scx_central.c | 135 +
+ tools/sched_ext/scx_qmap.bpf.c | 706 ++
+ tools/sched_ext/scx_qmap.c | 144 +
+ tools/sched_ext/scx_show_state.py | 39 +
+ tools/sched_ext/scx_simple.bpf.c | 156 +
+ tools/sched_ext/scx_simple.c | 107 +
+ tools/testing/selftests/sched_ext/.gitignore | 6 +
+ tools/testing/selftests/sched_ext/Makefile | 218 +
+ tools/testing/selftests/sched_ext/config | 9 +
+ .../selftests/sched_ext/create_dsq.bpf.c | 58 +
+ .../testing/selftests/sched_ext/create_dsq.c | 57 +
+ .../sched_ext/ddsp_bogus_dsq_fail.bpf.c | 42 +
+ .../selftests/sched_ext/ddsp_bogus_dsq_fail.c | 57 +
+ .../sched_ext/ddsp_vtimelocal_fail.bpf.c | 39 +
+ .../sched_ext/ddsp_vtimelocal_fail.c | 56 +
+ .../selftests/sched_ext/dsp_local_on.bpf.c | 65 +
+ .../selftests/sched_ext/dsp_local_on.c | 58 +
+ .../sched_ext/enq_last_no_enq_fails.bpf.c | 21 +
+ .../sched_ext/enq_last_no_enq_fails.c | 60 +
+ .../sched_ext/enq_select_cpu_fails.bpf.c | 43 +
+ .../sched_ext/enq_select_cpu_fails.c | 61 +
+ tools/testing/selftests/sched_ext/exit.bpf.c | 84 +
+ tools/testing/selftests/sched_ext/exit.c | 55 +
+ tools/testing/selftests/sched_ext/exit_test.h | 20 +
+ .../testing/selftests/sched_ext/hotplug.bpf.c | 61 +
+ tools/testing/selftests/sched_ext/hotplug.c | 168 +
+ .../selftests/sched_ext/hotplug_test.h | 15 +
+ .../sched_ext/init_enable_count.bpf.c | 53 +
+ .../selftests/sched_ext/init_enable_count.c | 166 +
+ .../testing/selftests/sched_ext/maximal.bpf.c | 132 +
+ tools/testing/selftests/sched_ext/maximal.c | 51 +
+ .../selftests/sched_ext/maybe_null.bpf.c | 36 +
+ .../testing/selftests/sched_ext/maybe_null.c | 49 +
+ .../sched_ext/maybe_null_fail_dsp.bpf.c | 25 +
+ .../sched_ext/maybe_null_fail_yld.bpf.c | 28 +
+ .../testing/selftests/sched_ext/minimal.bpf.c | 21 +
+ tools/testing/selftests/sched_ext/minimal.c | 58 +
+ .../selftests/sched_ext/prog_run.bpf.c | 32 +
+ tools/testing/selftests/sched_ext/prog_run.c | 78 +
+ .../testing/selftests/sched_ext/reload_loop.c | 75 +
+ tools/testing/selftests/sched_ext/runner.c | 201 +
+ tools/testing/selftests/sched_ext/scx_test.h | 131 +
+ .../selftests/sched_ext/select_cpu_dfl.bpf.c | 40 +
+ .../selftests/sched_ext/select_cpu_dfl.c | 72 +
+ .../sched_ext/select_cpu_dfl_nodispatch.bpf.c | 89 +
+ .../sched_ext/select_cpu_dfl_nodispatch.c | 72 +
+ .../sched_ext/select_cpu_dispatch.bpf.c | 41 +
+ .../selftests/sched_ext/select_cpu_dispatch.c | 70 +
+ .../select_cpu_dispatch_bad_dsq.bpf.c | 37 +
+ .../sched_ext/select_cpu_dispatch_bad_dsq.c | 56 +
+ .../select_cpu_dispatch_dbl_dsp.bpf.c | 38 +
+ .../sched_ext/select_cpu_dispatch_dbl_dsp.c | 56 +
+ .../sched_ext/select_cpu_vtime.bpf.c | 92 +
+ .../selftests/sched_ext/select_cpu_vtime.c | 59 +
+ .../selftests/sched_ext/test_example.c | 49 +
+ tools/testing/selftests/sched_ext/util.c | 71 +
+ tools/testing/selftests/sched_ext/util.h | 13 +
+ 91 files changed, 13860 insertions(+), 118 deletions(-)
+ create mode 100644 Documentation/scheduler/sched-ext.rst
+ create mode 100644 include/linux/sched/ext.h
+ create mode 100644 include/trace/events/sched_ext.h
+ create mode 100644 kernel/sched/ext.c
+ create mode 100644 kernel/sched/ext.h
+ create mode 100644 tools/sched_ext/.gitignore
+ create mode 100644 tools/sched_ext/Makefile
+ create mode 100644 tools/sched_ext/README.md
+ create mode 100644 tools/sched_ext/include/bpf-compat/gnu/stubs.h
+ create mode 100644 tools/sched_ext/include/scx/common.bpf.h
+ create mode 100644 tools/sched_ext/include/scx/common.h
+ create mode 100644 tools/sched_ext/include/scx/compat.bpf.h
+ create mode 100644 tools/sched_ext/include/scx/compat.h
+ create mode 100644 tools/sched_ext/include/scx/user_exit_info.h
+ create mode 100644 tools/sched_ext/scx_central.bpf.c
+ create mode 100644 tools/sched_ext/scx_central.c
+ create mode 100644 tools/sched_ext/scx_qmap.bpf.c
+ create mode 100644 tools/sched_ext/scx_qmap.c
+ create mode 100644 tools/sched_ext/scx_show_state.py
+ create mode 100644 tools/sched_ext/scx_simple.bpf.c
+ create mode 100644 tools/sched_ext/scx_simple.c
+ create mode 100644 tools/testing/selftests/sched_ext/.gitignore
+ create mode 100644 tools/testing/selftests/sched_ext/Makefile
+ create mode 100644 tools/testing/selftests/sched_ext/config
+ create mode 100644 tools/testing/selftests/sched_ext/create_dsq.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/create_dsq.c
+ create mode 100644 tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.c
+ create mode 100644 tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.c
+ create mode 100644 tools/testing/selftests/sched_ext/dsp_local_on.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/dsp_local_on.c
+ create mode 100644 tools/testing/selftests/sched_ext/enq_last_no_enq_fails.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/enq_last_no_enq_fails.c
+ create mode 100644 tools/testing/selftests/sched_ext/enq_select_cpu_fails.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/enq_select_cpu_fails.c
+ create mode 100644 tools/testing/selftests/sched_ext/exit.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/exit.c
+ create mode 100644 tools/testing/selftests/sched_ext/exit_test.h
+ create mode 100644 tools/testing/selftests/sched_ext/hotplug.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/hotplug.c
+ create mode 100644 tools/testing/selftests/sched_ext/hotplug_test.h
+ create mode 100644 tools/testing/selftests/sched_ext/init_enable_count.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/init_enable_count.c
+ create mode 100644 tools/testing/selftests/sched_ext/maximal.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/maximal.c
+ create mode 100644 tools/testing/selftests/sched_ext/maybe_null.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/maybe_null.c
+ create mode 100644 tools/testing/selftests/sched_ext/maybe_null_fail_dsp.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/maybe_null_fail_yld.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/minimal.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/minimal.c
+ create mode 100644 tools/testing/selftests/sched_ext/prog_run.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/prog_run.c
+ create mode 100644 tools/testing/selftests/sched_ext/reload_loop.c
+ create mode 100644 tools/testing/selftests/sched_ext/runner.c
+ create mode 100644 tools/testing/selftests/sched_ext/scx_test.h
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_dfl.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_dfl.c
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.c
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_dispatch.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_dispatch.c
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.c
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.c
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_vtime.bpf.c
+ create mode 100644 tools/testing/selftests/sched_ext/select_cpu_vtime.c
+ create mode 100644 tools/testing/selftests/sched_ext/test_example.c
+ create mode 100644 tools/testing/selftests/sched_ext/util.c
+ create mode 100644 tools/testing/selftests/sched_ext/util.h
+
+diff --git a/Documentation/scheduler/index.rst b/Documentation/scheduler/index.rst
+index 43bd8a145b7a..0611dc3dda8e 100644
+--- a/Documentation/scheduler/index.rst
++++ b/Documentation/scheduler/index.rst
+@@ -20,6 +20,7 @@ Scheduler
+ sched-nice-design
+ sched-rt-group
+ sched-stats
++ sched-ext
+ sched-debug
+
+ text_files
+diff --git a/Documentation/scheduler/sched-ext.rst b/Documentation/scheduler/sched-ext.rst
+new file mode 100644
+index 000000000000..a707d2181a77
+--- /dev/null
++++ b/Documentation/scheduler/sched-ext.rst
+@@ -0,0 +1,316 @@
++==========================
++Extensible Scheduler Class
++==========================
++
++sched_ext is a scheduler class whose behavior can be defined by a set of BPF
++programs - the BPF scheduler.
++
++* sched_ext exports a full scheduling interface so that any scheduling
++ algorithm can be implemented on top.
++
++* The BPF scheduler can group CPUs however it sees fit and schedule them
++ together, as tasks aren't tied to specific CPUs at the time of wakeup.
++
++* The BPF scheduler can be turned on and off dynamically anytime.
++
++* The system integrity is maintained no matter what the BPF scheduler does.
++ The default scheduling behavior is restored anytime an error is detected,
++ a runnable task stalls, or on invoking the SysRq key sequence
++ :kbd:`SysRq-S`.
++
++* When the BPF scheduler triggers an error, debug information is dumped to
++ aid debugging. The debug dump is passed to and printed out by the
++ scheduler binary. The debug dump can also be accessed through the
++ `sched_ext_dump` tracepoint. The SysRq key sequence :kbd:`SysRq-D`
++ triggers a debug dump. This doesn't terminate the BPF scheduler and can
++ only be read through the tracepoint.
++
++Switching to and from sched_ext
++===============================
++
++``CONFIG_SCHED_CLASS_EXT`` is the config option to enable sched_ext and
++``tools/sched_ext`` contains the example schedulers. The following config
++options should be enabled to use sched_ext:
++
++.. code-block:: none
++
++ CONFIG_BPF=y
++ CONFIG_SCHED_CLASS_EXT=y
++ CONFIG_BPF_SYSCALL=y
++ CONFIG_BPF_JIT=y
++ CONFIG_DEBUG_INFO_BTF=y
++ CONFIG_BPF_JIT_ALWAYS_ON=y
++ CONFIG_BPF_JIT_DEFAULT_ON=y
++ CONFIG_PAHOLE_HAS_SPLIT_BTF=y
++ CONFIG_PAHOLE_HAS_BTF_TAG=y
++
++sched_ext is used only when the BPF scheduler is loaded and running.
++
++If a task explicitly sets its scheduling policy to ``SCHED_EXT``, it will be
++treated as ``SCHED_NORMAL`` and scheduled by CFS until the BPF scheduler is
++loaded.
++
++When the BPF scheduler is loaded and ``SCX_OPS_SWITCH_PARTIAL`` is not set
++in ``ops->flags``, all ``SCHED_NORMAL``, ``SCHED_BATCH``, ``SCHED_IDLE``, and
++``SCHED_EXT`` tasks are scheduled by sched_ext.
++
++However, when the BPF scheduler is loaded and ``SCX_OPS_SWITCH_PARTIAL`` is
++set in ``ops->flags``, only tasks with the ``SCHED_EXT`` policy are scheduled
++by sched_ext, while tasks with ``SCHED_NORMAL``, ``SCHED_BATCH`` and
++``SCHED_IDLE`` policies are scheduled by CFS.
++
++Terminating the sched_ext scheduler program, triggering :kbd:`SysRq-S`, or
++detection of any internal error including stalled runnable tasks aborts the
++BPF scheduler and reverts all tasks back to CFS.
++
++.. code-block:: none
++
++ # make -j16 -C tools/sched_ext
++ # tools/sched_ext/scx_simple
++ local=0 global=3
++ local=5 global=24
++ local=9 global=44
++ local=13 global=56
++ local=17 global=72
++ ^CEXIT: BPF scheduler unregistered
++
++The current status of the BPF scheduler can be determined as follows:
++
++.. code-block:: none
++
++ # cat /sys/kernel/sched_ext/state
++ enabled
++ # cat /sys/kernel/sched_ext/root/ops
++ simple
++
++``tools/sched_ext/scx_show_state.py`` is a drgn script which shows more
++detailed information:
++
++.. code-block:: none
++
++ # tools/sched_ext/scx_show_state.py
++ ops : simple
++ enabled : 1
++ switching_all : 1
++ switched_all : 1
++ enable_state : enabled (2)
++ bypass_depth : 0
++ nr_rejected : 0
++
++If ``CONFIG_SCHED_DEBUG`` is set, whether a given task is on sched_ext can
++be determined as follows:
++
++.. code-block:: none
++
++ # grep ext /proc/self/sched
++ ext.enabled : 1
++
++The Basics
++==========
++
++Userspace can implement an arbitrary BPF scheduler by loading a set of BPF
++programs that implement ``struct sched_ext_ops``. The only mandatory field
++is ``ops.name`` which must be a valid BPF object name. All operations are
++optional. The following modified excerpt is from
++``tools/sched_ext/scx_simple.bpf.c`` showing a minimal global FIFO scheduler.
++
++.. code-block:: c
++
++ /*
++ * Decide which CPU a task should be migrated to before being
++ * enqueued (either at wakeup, fork time, or exec time). If an
++ * idle core is found by the default ops.select_cpu() implementation,
++ * then dispatch the task directly to SCX_DSQ_LOCAL and skip the
++ * ops.enqueue() callback.
++ *
++ * Note that this implementation has exactly the same behavior as the
++ * default ops.select_cpu implementation. The behavior of the scheduler
++ * would be exactly same if the implementation just didn't define the
++ * simple_select_cpu() struct_ops prog.
++ */
++ s32 BPF_STRUCT_OPS(simple_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++ {
++ s32 cpu;
++ /* Need to initialize or the BPF verifier will reject the program */
++ bool direct = false;
++
++ cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &direct);
++
++ if (direct)
++ scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
++
++ return cpu;
++ }
++
++ /*
++ * Do a direct dispatch of a task to the global DSQ. This ops.enqueue()
++ * callback will only be invoked if we failed to find a core to dispatch
++ * to in ops.select_cpu() above.
++ *
++ * Note that this implementation has exactly the same behavior as the
++ * default ops.enqueue implementation, which just dispatches the task
++ * to SCX_DSQ_GLOBAL. The behavior of the scheduler would be exactly same
++ * if the implementation just didn't define the simple_enqueue struct_ops
++ * prog.
++ */
++ void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags)
++ {
++ scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
++ }
++
++ s32 BPF_STRUCT_OPS_SLEEPABLE(simple_init)
++ {
++ /*
++ * By default, all SCHED_EXT, SCHED_OTHER, SCHED_IDLE, and
++ * SCHED_BATCH tasks should use sched_ext.
++ */
++ return 0;
++ }
++
++ void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei)
++ {
++ exit_type = ei->type;
++ }
++
++ SEC(".struct_ops")
++ struct sched_ext_ops simple_ops = {
++ .select_cpu = (void *)simple_select_cpu,
++ .enqueue = (void *)simple_enqueue,
++ .init = (void *)simple_init,
++ .exit = (void *)simple_exit,
++ .name = "simple",
++ };
++
++Dispatch Queues
++---------------
++
++To match the impedance between the scheduler core and the BPF scheduler,
++sched_ext uses DSQs (dispatch queues) which can operate as both a FIFO and a
++priority queue. By default, there is one global FIFO (``SCX_DSQ_GLOBAL``),
++and one local dsq per CPU (``SCX_DSQ_LOCAL``). The BPF scheduler can manage
++an arbitrary number of dsq's using ``scx_bpf_create_dsq()`` and
++``scx_bpf_destroy_dsq()``.
++
++A CPU always executes a task from its local DSQ. A task is "dispatched" to a
++DSQ. A non-local DSQ is "consumed" to transfer a task to the consuming CPU's
++local DSQ.
++
++When a CPU is looking for the next task to run, if the local DSQ is not
++empty, the first task is picked. Otherwise, the CPU tries to consume the
++global DSQ. If that doesn't yield a runnable task either, ``ops.dispatch()``
++is invoked.
++
++Scheduling Cycle
++----------------
++
++The following briefly shows how a waking task is scheduled and executed.
++
++1. When a task is waking up, ``ops.select_cpu()`` is the first operation
++ invoked. This serves two purposes. First, CPU selection optimization
++ hint. Second, waking up the selected CPU if idle.
++
++ The CPU selected by ``ops.select_cpu()`` is an optimization hint and not
++ binding. The actual decision is made at the last step of scheduling.
++ However, there is a small performance gain if the CPU
++ ``ops.select_cpu()`` returns matches the CPU the task eventually runs on.
++
++ A side-effect of selecting a CPU is waking it up from idle. While a BPF
++ scheduler can wake up any cpu using the ``scx_bpf_kick_cpu()`` helper,
++ using ``ops.select_cpu()`` judiciously can be simpler and more efficient.
++
++ A task can be immediately dispatched to a DSQ from ``ops.select_cpu()`` by
++ calling ``scx_bpf_dispatch()``. If the task is dispatched to
++ ``SCX_DSQ_LOCAL`` from ``ops.select_cpu()``, it will be dispatched to the
++ local DSQ of whichever CPU is returned from ``ops.select_cpu()``.
++ Additionally, dispatching directly from ``ops.select_cpu()`` will cause the
++ ``ops.enqueue()`` callback to be skipped.
++
++ Note that the scheduler core will ignore an invalid CPU selection, for
++ example, if it's outside the allowed cpumask of the task.
++
++2. Once the target CPU is selected, ``ops.enqueue()`` is invoked (unless the
++ task was dispatched directly from ``ops.select_cpu()``). ``ops.enqueue()``
++ can make one of the following decisions:
++
++ * Immediately dispatch the task to either the global or local DSQ by
++ calling ``scx_bpf_dispatch()`` with ``SCX_DSQ_GLOBAL`` or
++ ``SCX_DSQ_LOCAL``, respectively.
++
++ * Immediately dispatch the task to a custom DSQ by calling
++ ``scx_bpf_dispatch()`` with a DSQ ID which is smaller than 2^63.
++
++ * Queue the task on the BPF side.
++
++3. When a CPU is ready to schedule, it first looks at its local DSQ. If
++ empty, it then looks at the global DSQ. If there still isn't a task to
++ run, ``ops.dispatch()`` is invoked which can use the following two
++ functions to populate the local DSQ.
++
++ * ``scx_bpf_dispatch()`` dispatches a task to a DSQ. Any target DSQ can
++ be used - ``SCX_DSQ_LOCAL``, ``SCX_DSQ_LOCAL_ON | cpu``,
++ ``SCX_DSQ_GLOBAL`` or a custom DSQ. While ``scx_bpf_dispatch()``
++ currently can't be called with BPF locks held, this is being worked on
++ and will be supported. ``scx_bpf_dispatch()`` schedules dispatching
++ rather than performing them immediately. There can be up to
++ ``ops.dispatch_max_batch`` pending tasks.
++
++ * ``scx_bpf_consume()`` tranfers a task from the specified non-local DSQ
++ to the dispatching DSQ. This function cannot be called with any BPF
++ locks held. ``scx_bpf_consume()`` flushes the pending dispatched tasks
++ before trying to consume the specified DSQ.
++
++4. After ``ops.dispatch()`` returns, if there are tasks in the local DSQ,
++ the CPU runs the first one. If empty, the following steps are taken:
++
++ * Try to consume the global DSQ. If successful, run the task.
++
++ * If ``ops.dispatch()`` has dispatched any tasks, retry #3.
++
++ * If the previous task is an SCX task and still runnable, keep executing
++ it (see ``SCX_OPS_ENQ_LAST``).
++
++ * Go idle.
++
++Note that the BPF scheduler can always choose to dispatch tasks immediately
++in ``ops.enqueue()`` as illustrated in the above simple example. If only the
++built-in DSQs are used, there is no need to implement ``ops.dispatch()`` as
++a task is never queued on the BPF scheduler and both the local and global
++DSQs are consumed automatically.
++
++``scx_bpf_dispatch()`` queues the task on the FIFO of the target DSQ. Use
++``scx_bpf_dispatch_vtime()`` for the priority queue. Internal DSQs such as
++``SCX_DSQ_LOCAL`` and ``SCX_DSQ_GLOBAL`` do not support priority-queue
++dispatching, and must be dispatched to with ``scx_bpf_dispatch()``. See the
++function documentation and usage in ``tools/sched_ext/scx_simple.bpf.c`` for
++more information.
++
++Where to Look
++=============
++
++* ``include/linux/sched/ext.h`` defines the core data structures, ops table
++ and constants.
++
++* ``kernel/sched/ext.c`` contains sched_ext core implementation and helpers.
++ The functions prefixed with ``scx_bpf_`` can be called from the BPF
++ scheduler.
++
++* ``tools/sched_ext/`` hosts example BPF scheduler implementations.
++
++ * ``scx_simple[.bpf].c``: Minimal global FIFO scheduler example using a
++ custom DSQ.
++
++ * ``scx_qmap[.bpf].c``: A multi-level FIFO scheduler supporting five
++ levels of priority implemented with ``BPF_MAP_TYPE_QUEUE``.
++
++ABI Instability
++===============
++
++The APIs provided by sched_ext to BPF schedulers programs have no stability
++guarantees. This includes the ops table callbacks and constants defined in
++``include/linux/sched/ext.h``, as well as the ``scx_bpf_`` kfuncs defined in
++``kernel/sched/ext.c``.
++
++While we will attempt to provide a relatively stable API surface when
++possible, they are subject to change without warning between kernel
++versions.
+diff --git a/MAINTAINERS b/MAINTAINERS
+index 3121709d99e3..bf3b77e96dc4 100644
+--- a/MAINTAINERS
++++ b/MAINTAINERS
+@@ -19623,6 +19623,19 @@ F: include/linux/wait.h
+ F: include/uapi/linux/sched.h
+ F: kernel/sched/
+
++SCHEDULER - SCHED_EXT
++R: Tejun Heo <tj@kernel.org>
++R: David Vernet <void@manifault.com>
++L: linux-kernel@vger.kernel.org
++S: Maintained
++W: https://github.com/sched-ext/scx
++T: git://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git
++F: include/linux/sched/ext.h
++F: kernel/sched/ext.h
++F: kernel/sched/ext.c
++F: tools/sched_ext/
++F: tools/testing/selftests/sched_ext
++
+ SCSI LIBSAS SUBSYSTEM
+ R: John Garry <john.g.garry@oracle.com>
+ R: Jason Yan <yanaijie@huawei.com>
+diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
+index f7749d0f2562..05bfe4acba1d 100644
+--- a/include/asm-generic/vmlinux.lds.h
++++ b/include/asm-generic/vmlinux.lds.h
+@@ -131,6 +131,7 @@
+ *(__dl_sched_class) \
+ *(__rt_sched_class) \
+ *(__fair_sched_class) \
++ *(__ext_sched_class) \
+ *(__idle_sched_class) \
+ __sched_class_lowest = .;
+
+diff --git a/include/linux/cgroup.h b/include/linux/cgroup.h
+index 34aaf0e87def..bcebf8096e91 100644
+--- a/include/linux/cgroup.h
++++ b/include/linux/cgroup.h
+@@ -29,8 +29,6 @@
+
+ struct kernel_clone_args;
+
+-#ifdef CONFIG_CGROUPS
+-
+ /*
+ * All weight knobs on the default hierarchy should use the following min,
+ * default and max values. The default value is the logarithmic center of
+@@ -40,6 +38,8 @@ struct kernel_clone_args;
+ #define CGROUP_WEIGHT_DFL 100
+ #define CGROUP_WEIGHT_MAX 10000
+
++#ifdef CONFIG_CGROUPS
++
+ enum {
+ CSS_TASK_ITER_PROCS = (1U << 0), /* walk only threadgroup leaders */
+ CSS_TASK_ITER_THREADED = (1U << 1), /* walk all threaded css_sets in the domain */
+diff --git a/include/linux/sched.h b/include/linux/sched.h
+index 3c2abbc587b4..dc07eb0d3290 100644
+--- a/include/linux/sched.h
++++ b/include/linux/sched.h
+@@ -80,6 +80,8 @@ struct task_group;
+ struct task_struct;
+ struct user_event_mm;
+
++#include <linux/sched/ext.h>
++
+ /*
+ * Task state bitmask. NOTE! These bits are also
+ * encoded in fs/proc/array.c: get_task_state().
+@@ -798,6 +800,9 @@ struct task_struct {
+ struct sched_rt_entity rt;
+ struct sched_dl_entity dl;
+ struct sched_dl_entity *dl_server;
++#ifdef CONFIG_SCHED_CLASS_EXT
++ struct sched_ext_entity scx;
++#endif
+ const struct sched_class *sched_class;
+
+ #ifdef CONFIG_SCHED_CORE
+diff --git a/include/linux/sched/ext.h b/include/linux/sched/ext.h
+new file mode 100644
+index 000000000000..593d2f4909dd
+--- /dev/null
++++ b/include/linux/sched/ext.h
+@@ -0,0 +1,206 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
++ *
++ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
++ */
++#ifndef _LINUX_SCHED_EXT_H
++#define _LINUX_SCHED_EXT_H
++
++#ifdef CONFIG_SCHED_CLASS_EXT
++
++#include <linux/llist.h>
++#include <linux/rhashtable-types.h>
++
++enum scx_public_consts {
++ SCX_OPS_NAME_LEN = 128,
++
++ SCX_SLICE_DFL = 20 * 1000000, /* 20ms */
++ SCX_SLICE_INF = U64_MAX, /* infinite, implies nohz */
++};
++
++/*
++ * DSQ (dispatch queue) IDs are 64bit of the format:
++ *
++ * Bits: [63] [62 .. 0]
++ * [ B] [ ID ]
++ *
++ * B: 1 for IDs for built-in DSQs, 0 for ops-created user DSQs
++ * ID: 63 bit ID
++ *
++ * Built-in IDs:
++ *
++ * Bits: [63] [62] [61..32] [31 .. 0]
++ * [ 1] [ L] [ R ] [ V ]
++ *
++ * 1: 1 for built-in DSQs.
++ * L: 1 for LOCAL_ON DSQ IDs, 0 for others
++ * V: For LOCAL_ON DSQ IDs, a CPU number. For others, a pre-defined value.
++ */
++enum scx_dsq_id_flags {
++ SCX_DSQ_FLAG_BUILTIN = 1LLU << 63,
++ SCX_DSQ_FLAG_LOCAL_ON = 1LLU << 62,
++
++ SCX_DSQ_INVALID = SCX_DSQ_FLAG_BUILTIN | 0,
++ SCX_DSQ_GLOBAL = SCX_DSQ_FLAG_BUILTIN | 1,
++ SCX_DSQ_LOCAL = SCX_DSQ_FLAG_BUILTIN | 2,
++ SCX_DSQ_LOCAL_ON = SCX_DSQ_FLAG_BUILTIN | SCX_DSQ_FLAG_LOCAL_ON,
++ SCX_DSQ_LOCAL_CPU_MASK = 0xffffffffLLU,
++};
++
++/*
++ * A dispatch queue (DSQ) can be either a FIFO or p->scx.dsq_vtime ordered
++ * queue. A built-in DSQ is always a FIFO. The built-in local DSQs are used to
++ * buffer between the scheduler core and the BPF scheduler. See the
++ * documentation for more details.
++ */
++struct scx_dispatch_q {
++ raw_spinlock_t lock;
++ struct list_head list; /* tasks in dispatch order */
++ struct rb_root priq; /* used to order by p->scx.dsq_vtime */
++ u32 nr;
++ u32 seq; /* used by BPF iter */
++ u64 id;
++ struct rhash_head hash_node;
++ struct llist_node free_node;
++ struct rcu_head rcu;
++};
++
++/* scx_entity.flags */
++enum scx_ent_flags {
++ SCX_TASK_QUEUED = 1 << 0, /* on ext runqueue */
++ SCX_TASK_BAL_KEEP = 1 << 1, /* balance decided to keep current */
++ SCX_TASK_RESET_RUNNABLE_AT = 1 << 2, /* runnable_at should be reset */
++ SCX_TASK_DEQD_FOR_SLEEP = 1 << 3, /* last dequeue was for SLEEP */
++
++ SCX_TASK_STATE_SHIFT = 8, /* bit 8 and 9 are used to carry scx_task_state */
++ SCX_TASK_STATE_BITS = 2,
++ SCX_TASK_STATE_MASK = ((1 << SCX_TASK_STATE_BITS) - 1) << SCX_TASK_STATE_SHIFT,
++
++ SCX_TASK_CURSOR = 1 << 31, /* iteration cursor, not a task */
++};
++
++/* scx_entity.flags & SCX_TASK_STATE_MASK */
++enum scx_task_state {
++ SCX_TASK_NONE, /* ops.init_task() not called yet */
++ SCX_TASK_INIT, /* ops.init_task() succeeded, but task can be cancelled */
++ SCX_TASK_READY, /* fully initialized, but not in sched_ext */
++ SCX_TASK_ENABLED, /* fully initialized and in sched_ext */
++
++ SCX_TASK_NR_STATES,
++};
++
++/* scx_entity.dsq_flags */
++enum scx_ent_dsq_flags {
++ SCX_TASK_DSQ_ON_PRIQ = 1 << 0, /* task is queued on the priority queue of a dsq */
++};
++
++/*
++ * Mask bits for scx_entity.kf_mask. Not all kfuncs can be called from
++ * everywhere and the following bits track which kfunc sets are currently
++ * allowed for %current. This simple per-task tracking works because SCX ops
++ * nest in a limited way. BPF will likely implement a way to allow and disallow
++ * kfuncs depending on the calling context which will replace this manual
++ * mechanism. See scx_kf_allow().
++ */
++enum scx_kf_mask {
++ SCX_KF_UNLOCKED = 0, /* not sleepable, not rq locked */
++ /* all non-sleepables may be nested inside SLEEPABLE */
++ SCX_KF_SLEEPABLE = 1 << 0, /* sleepable init operations */
++ /* ENQUEUE and DISPATCH may be nested inside CPU_RELEASE */
++ SCX_KF_CPU_RELEASE = 1 << 1, /* ops.cpu_release() */
++ /* ops.dequeue (in REST) may be nested inside DISPATCH */
++ SCX_KF_DISPATCH = 1 << 2, /* ops.dispatch() */
++ SCX_KF_ENQUEUE = 1 << 3, /* ops.enqueue() and ops.select_cpu() */
++ SCX_KF_SELECT_CPU = 1 << 4, /* ops.select_cpu() */
++ SCX_KF_REST = 1 << 5, /* other rq-locked operations */
++
++ __SCX_KF_RQ_LOCKED = SCX_KF_CPU_RELEASE | SCX_KF_DISPATCH |
++ SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU | SCX_KF_REST,
++ __SCX_KF_TERMINAL = SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU | SCX_KF_REST,
++};
++
++struct scx_dsq_list_node {
++ struct list_head node;
++ bool is_bpf_iter_cursor;
++};
++
++/*
++ * The following is embedded in task_struct and contains all fields necessary
++ * for a task to be scheduled by SCX.
++ */
++struct sched_ext_entity {
++ struct scx_dispatch_q *dsq;
++ struct scx_dsq_list_node dsq_list; /* dispatch order */
++ struct rb_node dsq_priq; /* p->scx.dsq_vtime order */
++ u32 dsq_seq;
++ u32 dsq_flags; /* protected by DSQ lock */
++ u32 flags; /* protected by rq lock */
++ u32 weight;
++ s32 sticky_cpu;
++ s32 holding_cpu;
++ u32 kf_mask; /* see scx_kf_mask above */
++ struct task_struct *kf_tasks[2]; /* see SCX_CALL_OP_TASK() */
++ atomic_long_t ops_state;
++
++ struct list_head runnable_node; /* rq->scx.runnable_list */
++ unsigned long runnable_at;
++
++#ifdef CONFIG_SCHED_CORE
++ u64 core_sched_at; /* see scx_prio_less() */
++#endif
++ u64 ddsp_dsq_id;
++ u64 ddsp_enq_flags;
++
++ /* BPF scheduler modifiable fields */
++
++ /*
++ * Runtime budget in nsecs. This is usually set through
++ * scx_bpf_dispatch() but can also be modified directly by the BPF
++ * scheduler. Automatically decreased by SCX as the task executes. On
++ * depletion, a scheduling event is triggered.
++ *
++ * This value is cleared to zero if the task is preempted by
++ * %SCX_KICK_PREEMPT and shouldn't be used to determine how long the
++ * task ran. Use p->se.sum_exec_runtime instead.
++ */
++ u64 slice;
++
++ /*
++ * Used to order tasks when dispatching to the vtime-ordered priority
++ * queue of a dsq. This is usually set through scx_bpf_dispatch_vtime()
++ * but can also be modified directly by the BPF scheduler. Modifying it
++ * while a task is queued on a dsq may mangle the ordering and is not
++ * recommended.
++ */
++ u64 dsq_vtime;
++
++ /*
++ * If set, reject future sched_setscheduler(2) calls updating the policy
++ * to %SCHED_EXT with -%EACCES.
++ *
++ * If set from ops.init_task() and the task's policy is already
++ * %SCHED_EXT, which can happen while the BPF scheduler is being loaded
++ * or by inhering the parent's policy during fork, the task's policy is
++ * rejected and forcefully reverted to %SCHED_NORMAL. The number of
++ * such events are reported through /sys/kernel/debug/sched_ext::nr_rejected.
++ */
++ bool disallow; /* reject switching into SCX */
++
++ /* cold fields */
++ /* must be the last field, see init_scx_entity() */
++ struct list_head tasks_node;
++};
++
++void sched_ext_free(struct task_struct *p);
++void print_scx_info(const char *log_lvl, struct task_struct *p);
++
++#else /* !CONFIG_SCHED_CLASS_EXT */
++
++static inline void sched_ext_free(struct task_struct *p) {}
++static inline void print_scx_info(const char *log_lvl, struct task_struct *p) {}
++
++#endif /* CONFIG_SCHED_CLASS_EXT */
++#endif /* _LINUX_SCHED_EXT_H */
+diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h
+index d362aacf9f89..4df2f9055587 100644
+--- a/include/linux/sched/task.h
++++ b/include/linux/sched/task.h
+@@ -63,7 +63,8 @@ extern asmlinkage void schedule_tail(struct task_struct *prev);
+ extern void init_idle(struct task_struct *idle, int cpu);
+
+ extern int sched_fork(unsigned long clone_flags, struct task_struct *p);
+-extern void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs);
++extern int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs);
++extern void sched_cancel_fork(struct task_struct *p);
+ extern void sched_post_fork(struct task_struct *p);
+ extern void sched_dead(struct task_struct *p);
+
+diff --git a/include/trace/events/sched_ext.h b/include/trace/events/sched_ext.h
+new file mode 100644
+index 000000000000..fe19da7315a9
+--- /dev/null
++++ b/include/trace/events/sched_ext.h
+@@ -0,0 +1,32 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++#undef TRACE_SYSTEM
++#define TRACE_SYSTEM sched_ext
++
++#if !defined(_TRACE_SCHED_EXT_H) || defined(TRACE_HEADER_MULTI_READ)
++#define _TRACE_SCHED_EXT_H
++
++#include <linux/tracepoint.h>
++
++TRACE_EVENT(sched_ext_dump,
++
++ TP_PROTO(const char *line),
++
++ TP_ARGS(line),
++
++ TP_STRUCT__entry(
++ __string(line, line)
++ ),
++
++ TP_fast_assign(
++ __assign_str(line, line);
++ ),
++
++ TP_printk("%s",
++ __get_str(line)
++ )
++);
++
++#endif /* _TRACE_SCHED_EXT_H */
++
++/* This part must be outside protection */
++#include <trace/define_trace.h>
+diff --git a/include/uapi/linux/sched.h b/include/uapi/linux/sched.h
+index 3bac0a8ceab2..359a14cc76a4 100644
+--- a/include/uapi/linux/sched.h
++++ b/include/uapi/linux/sched.h
+@@ -118,6 +118,7 @@ struct clone_args {
+ /* SCHED_ISO: reserved but not implemented yet */
+ #define SCHED_IDLE 5
+ #define SCHED_DEADLINE 6
++#define SCHED_EXT 7
+
+ /* Can be ORed in to make sure the process is reverted back to SCHED_NORMAL on fork */
+ #define SCHED_RESET_ON_FORK 0x40000000
+diff --git a/init/init_task.c b/init/init_task.c
+index 4daee6d761c8..ce882dbd2635 100644
+--- a/init/init_task.c
++++ b/init/init_task.c
+@@ -6,6 +6,7 @@
+ #include <linux/sched/sysctl.h>
+ #include <linux/sched/rt.h>
+ #include <linux/sched/task.h>
++#include <linux/sched/ext.h>
+ #include <linux/init.h>
+ #include <linux/fs.h>
+ #include <linux/mm.h>
+@@ -97,6 +98,17 @@ struct task_struct init_task __aligned(L1_CACHE_BYTES) = {
+ #endif
+ #ifdef CONFIG_CGROUP_SCHED
+ .sched_task_group = &root_task_group,
++#endif
++#ifdef CONFIG_SCHED_CLASS_EXT
++ .scx = {
++ .dsq_list.node = LIST_HEAD_INIT(init_task.scx.dsq_list.node),
++ .sticky_cpu = -1,
++ .holding_cpu = -1,
++ .runnable_node = LIST_HEAD_INIT(init_task.scx.runnable_node),
++ .runnable_at = INITIAL_JIFFIES,
++ .ddsp_dsq_id = SCX_DSQ_INVALID,
++ .slice = SCX_SLICE_DFL,
++ },
+ #endif
+ .ptraced = LIST_HEAD_INIT(init_task.ptraced),
+ .ptrace_entry = LIST_HEAD_INIT(init_task.ptrace_entry),
+diff --git a/kernel/Kconfig.preempt b/kernel/Kconfig.preempt
+index c2f1fd95a821..f3d140c3acc1 100644
+--- a/kernel/Kconfig.preempt
++++ b/kernel/Kconfig.preempt
+@@ -133,4 +133,28 @@ config SCHED_CORE
+ which is the likely usage by Linux distributions, there should
+ be no measurable impact on performance.
+
+-
++config SCHED_CLASS_EXT
++ bool "Extensible Scheduling Class"
++ depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF
++ help
++ This option enables a new scheduler class sched_ext (SCX), which
++ allows scheduling policies to be implemented as BPF programs to
++ achieve the following:
++
++ - Ease of experimentation and exploration: Enabling rapid
++ iteration of new scheduling policies.
++ - Customization: Building application-specific schedulers which
++ implement policies that are not applicable to general-purpose
++ schedulers.
++ - Rapid scheduler deployments: Non-disruptive swap outs of
++ scheduling policies in production environments.
++
++ sched_ext leverages BPF struct_ops feature to define a structure
++ which exports function callbacks and flags to BPF programs that
++ wish to implement scheduling policies. The struct_ops structure
++ exported by sched_ext is struct sched_ext_ops, and is conceptually
++ similar to struct sched_class.
++
++ For more information:
++ Documentation/scheduler/sched-ext.rst
++ https://github.com/sched-ext/scx
+diff --git a/kernel/fork.c b/kernel/fork.c
+index aebb3e6c96dc..5d1f9de254d6 100644
+--- a/kernel/fork.c
++++ b/kernel/fork.c
+@@ -23,6 +23,7 @@
+ #include <linux/sched/task.h>
+ #include <linux/sched/task_stack.h>
+ #include <linux/sched/cputime.h>
++#include <linux/sched/ext.h>
+ #include <linux/seq_file.h>
+ #include <linux/rtmutex.h>
+ #include <linux/init.h>
+@@ -971,6 +972,7 @@ void __put_task_struct(struct task_struct *tsk)
+ WARN_ON(refcount_read(&tsk->usage));
+ WARN_ON(tsk == current);
+
++ sched_ext_free(tsk);
+ io_uring_free(tsk);
+ cgroup_free(tsk);
+ task_numa_free(tsk, true);
+@@ -2363,7 +2365,7 @@ __latent_entropy struct task_struct *copy_process(
+
+ retval = perf_event_init_task(p, clone_flags);
+ if (retval)
+- goto bad_fork_cleanup_policy;
++ goto bad_fork_sched_cancel_fork;
+ retval = audit_alloc(p);
+ if (retval)
+ goto bad_fork_cleanup_perf;
+@@ -2496,7 +2498,9 @@ __latent_entropy struct task_struct *copy_process(
+ * cgroup specific, it unconditionally needs to place the task on a
+ * runqueue.
+ */
+- sched_cgroup_fork(p, args);
++ retval = sched_cgroup_fork(p, args);
++ if (retval)
++ goto bad_fork_cancel_cgroup;
+
+ /*
+ * From this point on we must avoid any synchronous user-space
+@@ -2542,13 +2546,13 @@ __latent_entropy struct task_struct *copy_process(
+ /* Don't start children in a dying pid namespace */
+ if (unlikely(!(ns_of_pid(pid)->pid_allocated & PIDNS_ADDING))) {
+ retval = -ENOMEM;
+- goto bad_fork_cancel_cgroup;
++ goto bad_fork_core_free;
+ }
+
+ /* Let kill terminate clone/fork in the middle */
+ if (fatal_signal_pending(current)) {
+ retval = -EINTR;
+- goto bad_fork_cancel_cgroup;
++ goto bad_fork_core_free;
+ }
+
+ /* No more failure paths after this point. */
+@@ -2622,10 +2626,11 @@ __latent_entropy struct task_struct *copy_process(
+
+ return p;
+
+-bad_fork_cancel_cgroup:
++bad_fork_core_free:
+ sched_core_free(p);
+ spin_unlock(&current->sighand->siglock);
+ write_unlock_irq(&tasklist_lock);
++bad_fork_cancel_cgroup:
+ cgroup_cancel_fork(p, args);
+ bad_fork_put_pidfd:
+ if (clone_flags & CLONE_PIDFD) {
+@@ -2664,6 +2669,8 @@ __latent_entropy struct task_struct *copy_process(
+ audit_free(p);
+ bad_fork_cleanup_perf:
+ perf_event_free_task(p);
++bad_fork_sched_cancel_fork:
++ sched_cancel_fork(p);
+ bad_fork_cleanup_policy:
+ lockdep_free_task(p);
+ #ifdef CONFIG_NUMA
+diff --git a/kernel/sched/build_policy.c b/kernel/sched/build_policy.c
+index d9dc9ab3773f..e7d539bb721e 100644
+--- a/kernel/sched/build_policy.c
++++ b/kernel/sched/build_policy.c
+@@ -16,18 +16,25 @@
+ #include <linux/sched/clock.h>
+ #include <linux/sched/cputime.h>
+ #include <linux/sched/hotplug.h>
++#include <linux/sched/isolation.h>
+ #include <linux/sched/posix-timers.h>
+ #include <linux/sched/rt.h>
+
+ #include <linux/cpuidle.h>
+ #include <linux/jiffies.h>
++#include <linux/kobject.h>
+ #include <linux/livepatch.h>
++#include <linux/pm.h>
+ #include <linux/psi.h>
++#include <linux/rhashtable.h>
++#include <linux/seq_buf.h>
+ #include <linux/seqlock_api.h>
+ #include <linux/slab.h>
+ #include <linux/suspend.h>
+ #include <linux/tsacct_kern.h>
+ #include <linux/vtime.h>
++#include <linux/sysrq.h>
++#include <linux/percpu-rwsem.h>
+
+ #include <uapi/linux/sched/types.h>
+
+@@ -52,3 +59,6 @@
+ #include "cputime.c"
+ #include "deadline.c"
+
++#ifdef CONFIG_SCHED_CLASS_EXT
++# include "ext.c"
++#endif
+diff --git a/kernel/sched/core.c b/kernel/sched/core.c
+index d211d40a2edc..e5a6766b3a45 100644
+--- a/kernel/sched/core.c
++++ b/kernel/sched/core.c
+@@ -168,7 +168,10 @@ static inline int __task_prio(const struct task_struct *p)
+ if (p->sched_class == &idle_sched_class)
+ return MAX_RT_PRIO + NICE_WIDTH; /* 140 */
+
+- return MAX_RT_PRIO + MAX_NICE; /* 120, squash fair */
++ if (task_on_scx(p))
++ return MAX_RT_PRIO + MAX_NICE + 1; /* 120, squash ext */
++
++ return MAX_RT_PRIO + MAX_NICE; /* 119, squash fair */
+ }
+
+ /*
+@@ -197,6 +200,11 @@ static inline bool prio_less(const struct task_struct *a,
+ if (pa == MAX_RT_PRIO + MAX_NICE) /* fair */
+ return cfs_prio_less(a, b, in_fi);
+
++#ifdef CONFIG_SCHED_CLASS_EXT
++ if (pa == MAX_RT_PRIO + MAX_NICE + 1) /* ext */
++ return scx_prio_less(a, b, in_fi);
++#endif
++
+ return false;
+ }
+
+@@ -1255,11 +1263,14 @@ bool sched_can_stop_tick(struct rq *rq)
+ return true;
+
+ /*
+- * If there are no DL,RR/FIFO tasks, there must only be CFS tasks left;
+- * if there's more than one we need the tick for involuntary
+- * preemption.
++ * If there are no DL,RR/FIFO tasks, there must only be CFS or SCX tasks
++ * left. For CFS, if there's more than one we need the tick for
++ * involuntary preemption. For SCX, ask.
+ */
+- if (rq->nr_running > 1)
++ if (!scx_switched_all() && rq->nr_running > 1)
++ return false;
++
++ if (scx_enabled() && !scx_can_stop_tick(rq))
+ return false;
+
+ /*
+@@ -1327,27 +1338,24 @@ int tg_nop(struct task_group *tg, void *data)
+ static void set_load_weight(struct task_struct *p, bool update_load)
+ {
+ int prio = p->static_prio - MAX_RT_PRIO;
+- struct load_weight *load = &p->se.load;
++ struct load_weight lw;
+
+- /*
+- * SCHED_IDLE tasks get minimal weight:
+- */
+ if (task_has_idle_policy(p)) {
+- load->weight = scale_load(WEIGHT_IDLEPRIO);
+- load->inv_weight = WMULT_IDLEPRIO;
+- return;
++ lw.weight = scale_load(WEIGHT_IDLEPRIO);
++ lw.inv_weight = WMULT_IDLEPRIO;
++ } else {
++ lw.weight = scale_load(sched_prio_to_weight[prio]);
++ lw.inv_weight = sched_prio_to_wmult[prio];
+ }
+
+ /*
+ * SCHED_OTHER tasks have to update their load when changing their
+ * weight
+ */
+- if (update_load && p->sched_class == &fair_sched_class) {
+- reweight_task(p, prio);
+- } else {
+- load->weight = scale_load(sched_prio_to_weight[prio]);
+- load->inv_weight = sched_prio_to_wmult[prio];
+- }
++ if (update_load && p->sched_class->reweight_task)
++ p->sched_class->reweight_task(task_rq(p), p, &lw);
++ else
++ p->se.load = lw;
+ }
+
+ #ifdef CONFIG_UCLAMP_TASK
+@@ -2214,6 +2222,17 @@ inline int task_curr(const struct task_struct *p)
+ return cpu_curr(task_cpu(p)) == p;
+ }
+
++/*
++ * ->switching_to() is called with the pi_lock and rq_lock held and must not
++ * mess with locking.
++ */
++void check_class_changing(struct rq *rq, struct task_struct *p,
++ const struct sched_class *prev_class)
++{
++ if (prev_class != p->sched_class && p->sched_class->switching_to)
++ p->sched_class->switching_to(rq, p);
++}
++
+ /*
+ * switched_from, switched_to and prio_changed must _NOT_ drop rq->lock,
+ * use the balance_callback list if you want balancing.
+@@ -2221,9 +2240,9 @@ inline int task_curr(const struct task_struct *p)
+ * this means any call to check_class_changed() must be followed by a call to
+ * balance_callback().
+ */
+-static inline void check_class_changed(struct rq *rq, struct task_struct *p,
+- const struct sched_class *prev_class,
+- int oldprio)
++void check_class_changed(struct rq *rq, struct task_struct *p,
++ const struct sched_class *prev_class,
++ int oldprio)
+ {
+ if (prev_class != p->sched_class) {
+ if (prev_class->switched_from)
+@@ -3986,6 +4005,15 @@ bool cpus_share_resources(int this_cpu, int that_cpu)
+
+ static inline bool ttwu_queue_cond(struct task_struct *p, int cpu)
+ {
++ /*
++ * The BPF scheduler may depend on select_task_rq() being invoked during
++ * wakeups. In addition, @p may end up executing on a different CPU
++ * regardless of what happens in the wakeup path making the ttwu_queue
++ * optimization less meaningful. Skip if on SCX.
++ */
++ if (task_on_scx(p))
++ return false;
++
+ /*
+ * Do not complicate things with the async wake_list while the CPU is
+ * in hotplug state.
+@@ -4553,6 +4581,10 @@ static void __sched_fork(unsigned long clone_flags, struct task_struct *p)
+ p->rt.on_rq = 0;
+ p->rt.on_list = 0;
+
++#ifdef CONFIG_SCHED_CLASS_EXT
++ init_scx_entity(&p->scx);
++#endif
++
+ #ifdef CONFIG_PREEMPT_NOTIFIERS
+ INIT_HLIST_HEAD(&p->preempt_notifiers);
+ #endif
+@@ -4794,10 +4826,18 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
+
+ if (dl_prio(p->prio))
+ return -EAGAIN;
+- else if (rt_prio(p->prio))
++
++ scx_pre_fork(p);
++
++ if (rt_prio(p->prio)) {
+ p->sched_class = &rt_sched_class;
+- else
++#ifdef CONFIG_SCHED_CLASS_EXT
++ } else if (task_should_scx(p)) {
++ p->sched_class = &ext_sched_class;
++#endif
++ } else {
+ p->sched_class = &fair_sched_class;
++ }
+
+ init_entity_runnable_average(&p->se);
+
+@@ -4817,7 +4857,7 @@ int sched_fork(unsigned long clone_flags, struct task_struct *p)
+ return 0;
+ }
+
+-void sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
++int sched_cgroup_fork(struct task_struct *p, struct kernel_clone_args *kargs)
+ {
+ unsigned long flags;
+
+@@ -4979,6 +4979,13 @@
+ if (p->sched_class->task_fork)
+ p->sched_class->task_fork(p);
+ raw_spin_unlock_irqrestore(&p->pi_lock, flags);
++
++ return scx_fork(p);
++}
++
++void sched_cancel_fork(struct task_struct *p)
++{
++ scx_cancel_fork(p);
+ }
+
+ void sched_post_fork(struct task_struct *p)
+@@ -4987,6 +4994,7 @@
+ sched_post_fork_bore(p);
+ #endif // CONFIG_SCHED_BORE
+ uclamp_post_fork(p);
++ scx_post_fork(p);
+ }
+
+ unsigned long to_ratio(u64 period, u64 runtime)
+@@ -5687,6 +5735,7 @@ void scheduler_tick(void)
+ calc_global_load_tick(rq);
+ sched_core_tick(rq);
+ task_tick_mm_cid(rq, curr);
++ scx_tick(rq);
+
+ rq_unlock(rq, &rf);
+
+@@ -5699,8 +5748,10 @@ void scheduler_tick(void)
+ wq_worker_tick(curr);
+
+ #ifdef CONFIG_SMP
+- rq->idle_balance = idle_cpu(cpu);
+- trigger_load_balance(rq);
++ if (!scx_switched_all()) {
++ rq->idle_balance = idle_cpu(cpu);
++ trigger_load_balance(rq);
++ }
+ #endif
+ }
+
+@@ -5991,7 +6042,19 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev,
+ struct rq_flags *rf)
+ {
+ #ifdef CONFIG_SMP
++ const struct sched_class *start_class = prev->sched_class;
+ const struct sched_class *class;
++
++#ifdef CONFIG_SCHED_CLASS_EXT
++ /*
++ * SCX requires a balance() call before every pick_next_task() including
++ * when waking up from SCHED_IDLE. If @start_class is below SCX, start
++ * from SCX instead.
++ */
++ if (sched_class_above(&ext_sched_class, start_class))
++ start_class = &ext_sched_class;
++#endif
++
+ /*
+ * We must do the balancing pass before put_prev_task(), such
+ * that when we release the rq->lock the task is in the same
+@@ -6000,7 +6063,7 @@ static void put_prev_task_balance(struct rq *rq, struct task_struct *prev,
+ * We can terminate the balance pass as soon as we know there is
+ * a runnable task of @class priority or higher.
+ */
+- for_class_range(class, prev->sched_class, &idle_sched_class) {
++ for_active_class_range(class, start_class, &idle_sched_class) {
+ if (class->balance(rq, prev, rf))
+ break;
+ }
+@@ -6018,6 +6081,9 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
+ const struct sched_class *class;
+ struct task_struct *p;
+
++ if (scx_enabled())
++ goto restart;
++
+ /*
+ * Optimization: we know that if all tasks are in the fair class we can
+ * call that function directly, but only if the @prev task wasn't of a
+@@ -6058,10 +6124,15 @@ __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf)
+ if (prev->dl_server)
+ prev->dl_server = NULL;
+
+- for_each_class(class) {
++ for_each_active_class(class) {
+ p = class->pick_next_task(rq);
+- if (p)
++ if (p) {
++ const struct sched_class *prev_class = prev->sched_class;
++
++ if (class != prev_class && prev_class->switch_class)
++ prev_class->switch_class(rq, p);
+ return p;
++ }
+ }
+
+ BUG(); /* The idle class should always have a runnable task. */
+@@ -6091,7 +6162,7 @@ static inline struct task_struct *pick_task(struct rq *rq)
+ const struct sched_class *class;
+ struct task_struct *p;
+
+- for_each_class(class) {
++ for_each_active_class(class) {
+ p = class->pick_task(rq);
+ if (p)
+ return p;
+@@ -7081,12 +7152,16 @@ int default_wake_function(wait_queue_entry_t *curr, unsigned mode, int wake_flag
+ }
+ EXPORT_SYMBOL(default_wake_function);
+
+-static void __setscheduler_prio(struct task_struct *p, int prio)
++void __setscheduler_prio(struct task_struct *p, int prio)
+ {
+ if (dl_prio(prio))
+ p->sched_class = &dl_sched_class;
+ else if (rt_prio(prio))
+ p->sched_class = &rt_sched_class;
++#ifdef CONFIG_SCHED_CLASS_EXT
++ else if (task_should_scx(p))
++ p->sched_class = &ext_sched_class;
++#endif
+ else
+ p->sched_class = &fair_sched_class;
+
+@@ -7247,6 +7322,7 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task)
+ }
+
+ __setscheduler_prio(p, prio);
++ check_class_changing(rq, p, prev_class);
+
+ if (queued)
+ enqueue_task(rq, p, queue_flag);
+@@ -7468,6 +7544,25 @@ int sched_core_idle_cpu(int cpu)
+ #endif
+
+ #ifdef CONFIG_SMP
++/*
++ * Load avg and utiliztion metrics need to be updated periodically and before
++ * consumption. This function updates the metrics for all subsystems except for
++ * the fair class. @rq must be locked and have its clock updated.
++ */
++bool update_other_load_avgs(struct rq *rq)
++{
++ u64 now = rq_clock_pelt(rq);
++ const struct sched_class *curr_class = rq->curr->sched_class;
++ unsigned long hw_pressure = arch_scale_thermal_pressure(cpu_of(rq));
++
++ lockdep_assert_rq_held(rq);
++
++ return update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) |
++ update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) |
++ update_thermal_load_avg(now, rq, hw_pressure) |
++ update_irq_load_avg(rq, 0);
++}
++
+ /*
+ * This function computes an effective utilization for the given CPU, to be
+ * used for frequency selection given the linear relation: f = u * f_max.
+@@ -7790,6 +7885,10 @@ static int __sched_setscheduler(struct task_struct *p,
+ goto unlock;
+ }
+
++ retval = scx_check_setscheduler(p, policy);
++ if (retval)
++ goto unlock;
++
+ /*
+ * If not changing anything there's no need to proceed further,
+ * but store a possible modification of reset_on_fork.
+@@ -7892,6 +7991,7 @@ static int __sched_setscheduler(struct task_struct *p,
+ __setscheduler_prio(p, newprio);
+ }
+ __setscheduler_uclamp(p, attr);
++ check_class_changing(rq, p, prev_class);
+
+ if (queued) {
+ /*
+@@ -9067,6 +9167,7 @@ SYSCALL_DEFINE1(sched_get_priority_max, int, policy)
+ case SCHED_NORMAL:
+ case SCHED_BATCH:
+ case SCHED_IDLE:
++ case SCHED_EXT:
+ ret = 0;
+ break;
+ }
+@@ -9094,6 +9195,7 @@ SYSCALL_DEFINE1(sched_get_priority_min, int, policy)
+ case SCHED_NORMAL:
+ case SCHED_BATCH:
+ case SCHED_IDLE:
++ case SCHED_EXT:
+ ret = 0;
+ }
+ return ret;
+@@ -9189,6 +9291,7 @@ void sched_show_task(struct task_struct *p)
+
+ print_worker_info(KERN_INFO, p);
+ print_stop_info(KERN_INFO, p);
++ print_scx_info(KERN_INFO, p);
+ show_stack(p, NULL, KERN_INFO);
+ put_task_stack(p);
+ }
+@@ -9681,6 +9784,8 @@ int sched_cpu_activate(unsigned int cpu)
+ cpuset_cpu_active();
+ }
+
++ scx_rq_activate(rq);
++
+ /*
+ * Put the rq online, if not already. This happens:
+ *
+@@ -9741,6 +9846,8 @@ int sched_cpu_deactivate(unsigned int cpu)
+ }
+ rq_unlock_irqrestore(rq, &rf);
+
++ scx_rq_deactivate(rq);
++
+ #ifdef CONFIG_SCHED_SMT
+ /*
+ * When going down, decrement the number of cores with SMT present.
+@@ -10062,11 +10062,15 @@
+ int i;
+
+ /* Make sure the linker didn't screw up */
+- BUG_ON(&idle_sched_class != &fair_sched_class + 1 ||
+- &fair_sched_class != &rt_sched_class + 1 ||
+- &rt_sched_class != &dl_sched_class + 1);
+ #ifdef CONFIG_SMP
+- BUG_ON(&dl_sched_class != &stop_sched_class + 1);
++ BUG_ON(!sched_class_above(&stop_sched_class, &dl_sched_class));
++#endif
++ BUG_ON(!sched_class_above(&dl_sched_class, &rt_sched_class));
++ BUG_ON(!sched_class_above(&rt_sched_class, &fair_sched_class));
++ BUG_ON(!sched_class_above(&fair_sched_class, &idle_sched_class));
++#ifdef CONFIG_SCHED_CLASS_EXT
++ BUG_ON(!sched_class_above(&fair_sched_class, &ext_sched_class));
++ BUG_ON(!sched_class_above(&ext_sched_class, &idle_sched_class));
+ #endif
+
+ #ifdef CONFIG_SCHED_BORE
+@@ -10097,6 +10208,7 @@ void __init sched_init(void)
+ balance_push_set(smp_processor_id(), false);
+ #endif
+ init_sched_fair_class();
++ init_sched_ext_class();
+
+ psi_init();
+
+@@ -10523,11 +10635,6 @@ void sched_move_task(struct task_struct *tsk)
+ }
+ }
+
+-static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
+-{
+- return css ? container_of(css, struct task_group, css) : NULL;
+-}
+-
+ static struct cgroup_subsys_state *
+ cpu_cgroup_css_alloc(struct cgroup_subsys_state *parent_css)
+ {
+@@ -11294,29 +11401,27 @@ static int cpu_local_stat_show(struct seq_file *sf,
+ }
+
+ #ifdef CONFIG_FAIR_GROUP_SCHED
++
++static unsigned long tg_weight(struct task_group *tg)
++{
++ return scale_load_down(tg->shares);
++}
++
+ static u64 cpu_weight_read_u64(struct cgroup_subsys_state *css,
+ struct cftype *cft)
+ {
+- struct task_group *tg = css_tg(css);
+- u64 weight = scale_load_down(tg->shares);
+-
+- return DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024);
++ return sched_weight_to_cgroup(tg_weight(css_tg(css)));
+ }
+
+ static int cpu_weight_write_u64(struct cgroup_subsys_state *css,
+- struct cftype *cft, u64 weight)
++ struct cftype *cft, u64 cgrp_weight)
+ {
+- /*
+- * cgroup weight knobs should use the common MIN, DFL and MAX
+- * values which are 1, 100 and 10000 respectively. While it loses
+- * a bit of range on both ends, it maps pretty well onto the shares
+- * value used by scheduler and the round-trip conversions preserve
+- * the original value over the entire range.
+- */
+- if (weight < CGROUP_WEIGHT_MIN || weight > CGROUP_WEIGHT_MAX)
++ unsigned long weight;
++
++ if (cgrp_weight < CGROUP_WEIGHT_MIN || cgrp_weight > CGROUP_WEIGHT_MAX)
+ return -ERANGE;
+
+- weight = DIV_ROUND_CLOSEST_ULL(weight * 1024, CGROUP_WEIGHT_DFL);
++ weight = sched_weight_from_cgroup(cgrp_weight);
+
+ return sched_group_set_shares(css_tg(css), scale_load(weight));
+ }
+@@ -11324,7 +11429,7 @@ static int cpu_weight_write_u64(struct cgroup_subsys_state *css,
+ static s64 cpu_weight_nice_read_s64(struct cgroup_subsys_state *css,
+ struct cftype *cft)
+ {
+- unsigned long weight = scale_load_down(css_tg(css)->shares);
++ unsigned long weight = tg_weight(css_tg(css));
+ int last_delta = INT_MAX;
+ int prio, delta;
+
+@@ -12065,3 +12170,38 @@ void sched_mm_cid_fork(struct task_struct *t)
+ t->mm_cid_active = 1;
+ }
+ #endif
++
++#ifdef CONFIG_SCHED_CLASS_EXT
++void sched_deq_and_put_task(struct task_struct *p, int queue_flags,
++ struct sched_enq_and_set_ctx *ctx)
++{
++ struct rq *rq = task_rq(p);
++
++ lockdep_assert_rq_held(rq);
++
++ *ctx = (struct sched_enq_and_set_ctx){
++ .p = p,
++ .queue_flags = queue_flags,
++ .queued = task_on_rq_queued(p),
++ .running = task_current(rq, p),
++ };
++
++ update_rq_clock(rq);
++ if (ctx->queued)
++ dequeue_task(rq, p, queue_flags | DEQUEUE_NOCLOCK);
++ if (ctx->running)
++ put_prev_task(rq, p);
++}
++
++void sched_enq_and_set_task(struct sched_enq_and_set_ctx *ctx)
++{
++ struct rq *rq = task_rq(ctx->p);
++
++ lockdep_assert_rq_held(rq);
++
++ if (ctx->queued)
++ enqueue_task(rq, ctx->p, ctx->queue_flags | ENQUEUE_NOCLOCK);
++ if (ctx->running)
++ set_next_task(rq, ctx->p);
++}
++#endif /* CONFIG_SCHED_CLASS_EXT */
+diff --git a/kernel/sched/cpufreq_schedutil.c b/kernel/sched/cpufreq_schedutil.c
+index eece6244f9d2..e683e5d08daa 100644
+--- a/kernel/sched/cpufreq_schedutil.c
++++ b/kernel/sched/cpufreq_schedutil.c
+@@ -197,8 +197,10 @@ unsigned long sugov_effective_cpu_perf(int cpu, unsigned long actual,
+
+ static void sugov_get_util(struct sugov_cpu *sg_cpu, unsigned long boost)
+ {
+- unsigned long min, max, util = cpu_util_cfs_boost(sg_cpu->cpu);
++ unsigned long min, max, util = scx_cpuperf_target(sg_cpu->cpu);
+
++ if (!scx_switched_all())
++ util += cpu_util_cfs_boost(sg_cpu->cpu);
+ util = effective_cpu_util(sg_cpu->cpu, util, &min, &max);
+ util = max(util, boost);
+ sg_cpu->bw_min = min;
+@@ -325,16 +327,35 @@ static unsigned long sugov_iowait_apply(struct sugov_cpu *sg_cpu, u64 time,
+ }
+
+ #ifdef CONFIG_NO_HZ_COMMON
+-static bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu)
++static bool sugov_hold_freq(struct sugov_cpu *sg_cpu)
+ {
+- unsigned long idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);
+- bool ret = idle_calls == sg_cpu->saved_idle_calls;
++ unsigned long idle_calls;
++ bool ret;
++
++ /*
++ * The heuristics in this function is for the fair class. For SCX, the
++ * performance target comes directly from the BPF scheduler. Let's just
++ * follow it.
++ */
++ if (scx_switched_all())
++ return false;
++
++ /* if capped by uclamp_max, always update to be in compliance */
++ if (uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)))
++ return false;
++
++ /*
++ * Maintain the frequency if the CPU has not been idle recently, as
++ * reduction is likely to be premature.
++ */
++ idle_calls = tick_nohz_get_idle_calls_cpu(sg_cpu->cpu);
++ ret = idle_calls == sg_cpu->saved_idle_calls;
+
+ sg_cpu->saved_idle_calls = idle_calls;
+ return ret;
+ }
+ #else
+-static inline bool sugov_cpu_is_busy(struct sugov_cpu *sg_cpu) { return false; }
++static inline bool sugov_hold_freq(struct sugov_cpu *sg_cpu) { return false; }
+ #endif /* CONFIG_NO_HZ_COMMON */
+
+ /*
+@@ -382,14 +403,8 @@ static void sugov_update_single_freq(struct update_util_data *hook, u64 time,
+ return;
+
+ next_f = get_next_freq(sg_policy, sg_cpu->util, max_cap);
+- /*
+- * Do not reduce the frequency if the CPU has not been idle
+- * recently, as the reduction is likely to be premature then.
+- *
+- * Except when the rq is capped by uclamp_max.
+- */
+- if (!uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)) &&
+- sugov_cpu_is_busy(sg_cpu) && next_f < sg_policy->next_freq &&
++
++ if (sugov_hold_freq(sg_cpu) && next_f < sg_policy->next_freq &&
+ !sg_policy->need_freq_update) {
+ next_f = sg_policy->next_freq;
+
+@@ -436,14 +451,7 @@ static void sugov_update_single_perf(struct update_util_data *hook, u64 time,
+ if (!sugov_update_single_common(sg_cpu, time, max_cap, flags))
+ return;
+
+- /*
+- * Do not reduce the target performance level if the CPU has not been
+- * idle recently, as the reduction is likely to be premature then.
+- *
+- * Except when the rq is capped by uclamp_max.
+- */
+- if (!uclamp_rq_is_capped(cpu_rq(sg_cpu->cpu)) &&
+- sugov_cpu_is_busy(sg_cpu) && sg_cpu->util < prev_util)
++ if (sugov_hold_freq(sg_cpu) && sg_cpu->util < prev_util)
+ sg_cpu->util = prev_util;
+
+ cpufreq_driver_adjust_perf(sg_cpu->cpu, sg_cpu->bw_min,
+diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
+index 8d5d98a5834d..6f306e1c9c3e 100644
+--- a/kernel/sched/debug.c
++++ b/kernel/sched/debug.c
+@@ -1089,6 +1089,9 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
+ P(dl.runtime);
+ P(dl.deadline);
+ }
++#ifdef CONFIG_SCHED_CLASS_EXT
++ __PS("ext.enabled", task_on_scx(p));
++#endif
+ #undef PN_SCHEDSTAT
+ #undef P_SCHEDSTAT
+
+diff --git a/kernel/sched/ext.c b/kernel/sched/ext.c
+new file mode 100644
+index 000000000000..b9fd7b7d4a86
+--- /dev/null
++++ b/kernel/sched/ext.c
+@@ -0,0 +1,6537 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
++ *
++ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
++ */
++#define SCX_OP_IDX(op) (offsetof(struct sched_ext_ops, op) / sizeof(void (*)(void)))
++
++enum scx_consts {
++ SCX_DSP_DFL_MAX_BATCH = 32,
++ SCX_DSP_MAX_LOOPS = 32,
++ SCX_WATCHDOG_MAX_TIMEOUT = 30 * HZ,
++
++ SCX_EXIT_BT_LEN = 64,
++ SCX_EXIT_MSG_LEN = 1024,
++ SCX_EXIT_DUMP_DFL_LEN = 32768,
++
++ SCX_CPUPERF_ONE = SCHED_CAPACITY_SCALE,
++};
++
++enum scx_exit_kind {
++ SCX_EXIT_NONE,
++ SCX_EXIT_DONE,
++
++ SCX_EXIT_UNREG = 64, /* user-space initiated unregistration */
++ SCX_EXIT_UNREG_BPF, /* BPF-initiated unregistration */
++ SCX_EXIT_UNREG_KERN, /* kernel-initiated unregistration */
++ SCX_EXIT_SYSRQ, /* requested by 'S' sysrq */
++
++ SCX_EXIT_ERROR = 1024, /* runtime error, error msg contains details */
++ SCX_EXIT_ERROR_BPF, /* ERROR but triggered through scx_bpf_error() */
++ SCX_EXIT_ERROR_STALL, /* watchdog detected stalled runnable tasks */
++};
++
++/*
++ * An exit code can be specified when exiting with scx_bpf_exit() or
++ * scx_ops_exit(), corresponding to exit_kind UNREG_BPF and UNREG_KERN
++ * respectively. The codes are 64bit of the format:
++ *
++ * Bits: [63 .. 48 47 .. 32 31 .. 0]
++ * [ SYS ACT ] [ SYS RSN ] [ USR ]
++ *
++ * SYS ACT: System-defined exit actions
++ * SYS RSN: System-defined exit reasons
++ * USR : User-defined exit codes and reasons
++ *
++ * Using the above, users may communicate intention and context by ORing system
++ * actions and/or system reasons with a user-defined exit code.
++ */
++enum scx_exit_code {
++ /* Reasons */
++ SCX_ECODE_RSN_HOTPLUG = 1LLU << 32,
++
++ /* Actions */
++ SCX_ECODE_ACT_RESTART = 1LLU << 48,
++};
++
++/*
++ * scx_exit_info is passed to ops.exit() to describe why the BPF scheduler is
++ * being disabled.
++ */
++struct scx_exit_info {
++ /* %SCX_EXIT_* - broad category of the exit reason */
++ enum scx_exit_kind kind;
++
++ /* exit code if gracefully exiting */
++ s64 exit_code;
++
++ /* textual representation of the above */
++ const char *reason;
++
++ /* backtrace if exiting due to an error */
++ unsigned long *bt;
++ u32 bt_len;
++
++ /* informational message */
++ char *msg;
++
++ /* debug dump */
++ char *dump;
++};
++
++/* sched_ext_ops.flags */
++enum scx_ops_flags {
++ /*
++ * Keep built-in idle tracking even if ops.update_idle() is implemented.
++ */
++ SCX_OPS_KEEP_BUILTIN_IDLE = 1LLU << 0,
++
++ /*
++ * By default, if there are no other task to run on the CPU, ext core
++ * keeps running the current task even after its slice expires. If this
++ * flag is specified, such tasks are passed to ops.enqueue() with
++ * %SCX_ENQ_LAST. See the comment above %SCX_ENQ_LAST for more info.
++ */
++ SCX_OPS_ENQ_LAST = 1LLU << 1,
++
++ /*
++ * An exiting task may schedule after PF_EXITING is set. In such cases,
++ * bpf_task_from_pid() may not be able to find the task and if the BPF
++ * scheduler depends on pid lookup for dispatching, the task will be
++ * lost leading to various issues including RCU grace period stalls.
++ *
++ * To mask this problem, by default, unhashed tasks are automatically
++ * dispatched to the local DSQ on enqueue. If the BPF scheduler doesn't
++ * depend on pid lookups and wants to handle these tasks directly, the
++ * following flag can be used.
++ */
++ SCX_OPS_ENQ_EXITING = 1LLU << 2,
++
++ /*
++ * If set, only tasks with policy set to SCHED_EXT are attached to
++ * sched_ext. If clear, SCHED_NORMAL tasks are also included.
++ */
++ SCX_OPS_SWITCH_PARTIAL = 1LLU << 3,
++
++ SCX_OPS_ALL_FLAGS = SCX_OPS_KEEP_BUILTIN_IDLE |
++ SCX_OPS_ENQ_LAST |
++ SCX_OPS_ENQ_EXITING |
++ SCX_OPS_SWITCH_PARTIAL,
++};
++
++/* argument container for ops.init_task() */
++struct scx_init_task_args {
++ /*
++ * Set if ops.init_task() is being invoked on the fork path, as opposed
++ * to the scheduler transition path.
++ */
++ bool fork;
++};
++
++/* argument container for ops.exit_task() */
++struct scx_exit_task_args {
++ /* Whether the task exited before running on sched_ext. */
++ bool cancelled;
++};
++
++enum scx_cpu_preempt_reason {
++ /* next task is being scheduled by &sched_class_rt */
++ SCX_CPU_PREEMPT_RT,
++ /* next task is being scheduled by &sched_class_dl */
++ SCX_CPU_PREEMPT_DL,
++ /* next task is being scheduled by &sched_class_stop */
++ SCX_CPU_PREEMPT_STOP,
++ /* unknown reason for SCX being preempted */
++ SCX_CPU_PREEMPT_UNKNOWN,
++};
++
++/*
++ * Argument container for ops->cpu_acquire(). Currently empty, but may be
++ * expanded in the future.
++ */
++struct scx_cpu_acquire_args {};
++
++/* argument container for ops->cpu_release() */
++struct scx_cpu_release_args {
++ /* the reason the CPU was preempted */
++ enum scx_cpu_preempt_reason reason;
++
++ /* the task that's going to be scheduled on the CPU */
++ struct task_struct *task;
++};
++
++/*
++ * Informational context provided to dump operations.
++ */
++struct scx_dump_ctx {
++ enum scx_exit_kind kind;
++ s64 exit_code;
++ const char *reason;
++ u64 at_ns;
++ u64 at_jiffies;
++};
++
++/**
++ * struct sched_ext_ops - Operation table for BPF scheduler implementation
++ *
++ * Userland can implement an arbitrary scheduling policy by implementing and
++ * loading operations in this table.
++ */
++struct sched_ext_ops {
++ /**
++ * select_cpu - Pick the target CPU for a task which is being woken up
++ * @p: task being woken up
++ * @prev_cpu: the cpu @p was on before sleeping
++ * @wake_flags: SCX_WAKE_*
++ *
++ * Decision made here isn't final. @p may be moved to any CPU while it
++ * is getting dispatched for execution later. However, as @p is not on
++ * the rq at this point, getting the eventual execution CPU right here
++ * saves a small bit of overhead down the line.
++ *
++ * If an idle CPU is returned, the CPU is kicked and will try to
++ * dispatch. While an explicit custom mechanism can be added,
++ * select_cpu() serves as the default way to wake up idle CPUs.
++ *
++ * @p may be dispatched directly by calling scx_bpf_dispatch(). If @p
++ * is dispatched, the ops.enqueue() callback will be skipped. Finally,
++ * if @p is dispatched to SCX_DSQ_LOCAL, it will be dispatched to the
++ * local DSQ of whatever CPU is returned by this callback.
++ */
++ s32 (*select_cpu)(struct task_struct *p, s32 prev_cpu, u64 wake_flags);
++
++ /**
++ * enqueue - Enqueue a task on the BPF scheduler
++ * @p: task being enqueued
++ * @enq_flags: %SCX_ENQ_*
++ *
++ * @p is ready to run. Dispatch directly by calling scx_bpf_dispatch()
++ * or enqueue on the BPF scheduler. If not directly dispatched, the bpf
++ * scheduler owns @p and if it fails to dispatch @p, the task will
++ * stall.
++ *
++ * If @p was dispatched from ops.select_cpu(), this callback is
++ * skipped.
++ */
++ void (*enqueue)(struct task_struct *p, u64 enq_flags);
++
++ /**
++ * dequeue - Remove a task from the BPF scheduler
++ * @p: task being dequeued
++ * @deq_flags: %SCX_DEQ_*
++ *
++ * Remove @p from the BPF scheduler. This is usually called to isolate
++ * the task while updating its scheduling properties (e.g. priority).
++ *
++ * The ext core keeps track of whether the BPF side owns a given task or
++ * not and can gracefully ignore spurious dispatches from BPF side,
++ * which makes it safe to not implement this method. However, depending
++ * on the scheduling logic, this can lead to confusing behaviors - e.g.
++ * scheduling position not being updated across a priority change.
++ */
++ void (*dequeue)(struct task_struct *p, u64 deq_flags);
++
++ /**
++ * dispatch - Dispatch tasks from the BPF scheduler and/or consume DSQs
++ * @cpu: CPU to dispatch tasks for
++ * @prev: previous task being switched out
++ *
++ * Called when a CPU's local dsq is empty. The operation should dispatch
++ * one or more tasks from the BPF scheduler into the DSQs using
++ * scx_bpf_dispatch() and/or consume user DSQs into the local DSQ using
++ * scx_bpf_consume().
++ *
++ * The maximum number of times scx_bpf_dispatch() can be called without
++ * an intervening scx_bpf_consume() is specified by
++ * ops.dispatch_max_batch. See the comments on top of the two functions
++ * for more details.
++ *
++ * When not %NULL, @prev is an SCX task with its slice depleted. If
++ * @prev is still runnable as indicated by set %SCX_TASK_QUEUED in
++ * @prev->scx.flags, it is not enqueued yet and will be enqueued after
++ * ops.dispatch() returns. To keep executing @prev, return without
++ * dispatching or consuming any tasks. Also see %SCX_OPS_ENQ_LAST.
++ */
++ void (*dispatch)(s32 cpu, struct task_struct *prev);
++
++ /**
++ * tick - Periodic tick
++ * @p: task running currently
++ *
++ * This operation is called every 1/HZ seconds on CPUs which are
++ * executing an SCX task. Setting @p->scx.slice to 0 will trigger an
++ * immediate dispatch cycle on the CPU.
++ */
++ void (*tick)(struct task_struct *p);
++
++ /**
++ * runnable - A task is becoming runnable on its associated CPU
++ * @p: task becoming runnable
++ * @enq_flags: %SCX_ENQ_*
++ *
++ * This and the following three functions can be used to track a task's
++ * execution state transitions. A task becomes ->runnable() on a CPU,
++ * and then goes through one or more ->running() and ->stopping() pairs
++ * as it runs on the CPU, and eventually becomes ->quiescent() when it's
++ * done running on the CPU.
++ *
++ * @p is becoming runnable on the CPU because it's
++ *
++ * - waking up (%SCX_ENQ_WAKEUP)
++ * - being moved from another CPU
++ * - being restored after temporarily taken off the queue for an
++ * attribute change.
++ *
++ * This and ->enqueue() are related but not coupled. This operation
++ * notifies @p's state transition and may not be followed by ->enqueue()
++ * e.g. when @p is being dispatched to a remote CPU, or when @p is
++ * being enqueued on a CPU experiencing a hotplug event. Likewise, a
++ * task may be ->enqueue()'d without being preceded by this operation
++ * e.g. after exhausting its slice.
++ */
++ void (*runnable)(struct task_struct *p, u64 enq_flags);
++
++ /**
++ * running - A task is starting to run on its associated CPU
++ * @p: task starting to run
++ *
++ * See ->runnable() for explanation on the task state notifiers.
++ */
++ void (*running)(struct task_struct *p);
++
++ /**
++ * stopping - A task is stopping execution
++ * @p: task stopping to run
++ * @runnable: is task @p still runnable?
++ *
++ * See ->runnable() for explanation on the task state notifiers. If
++ * !@runnable, ->quiescent() will be invoked after this operation
++ * returns.
++ */
++ void (*stopping)(struct task_struct *p, bool runnable);
++
++ /**
++ * quiescent - A task is becoming not runnable on its associated CPU
++ * @p: task becoming not runnable
++ * @deq_flags: %SCX_DEQ_*
++ *
++ * See ->runnable() for explanation on the task state notifiers.
++ *
++ * @p is becoming quiescent on the CPU because it's
++ *
++ * - sleeping (%SCX_DEQ_SLEEP)
++ * - being moved to another CPU
++ * - being temporarily taken off the queue for an attribute change
++ * (%SCX_DEQ_SAVE)
++ *
++ * This and ->dequeue() are related but not coupled. This operation
++ * notifies @p's state transition and may not be preceded by ->dequeue()
++ * e.g. when @p is being dispatched to a remote CPU.
++ */
++ void (*quiescent)(struct task_struct *p, u64 deq_flags);
++
++ /**
++ * yield - Yield CPU
++ * @from: yielding task
++ * @to: optional yield target task
++ *
++ * If @to is NULL, @from is yielding the CPU to other runnable tasks.
++ * The BPF scheduler should ensure that other available tasks are
++ * dispatched before the yielding task. Return value is ignored in this
++ * case.
++ *
++ * If @to is not-NULL, @from wants to yield the CPU to @to. If the bpf
++ * scheduler can implement the request, return %true; otherwise, %false.
++ */
++ bool (*yield)(struct task_struct *from, struct task_struct *to);
++
++ /**
++ * core_sched_before - Task ordering for core-sched
++ * @a: task A
++ * @b: task B
++ *
++ * Used by core-sched to determine the ordering between two tasks. See
++ * Documentation/admin-guide/hw-vuln/core-scheduling.rst for details on
++ * core-sched.
++ *
++ * Both @a and @b are runnable and may or may not currently be queued on
++ * the BPF scheduler. Should return %true if @a should run before @b.
++ * %false if there's no required ordering or @b should run before @a.
++ *
++ * If not specified, the default is ordering them according to when they
++ * became runnable.
++ */
++ bool (*core_sched_before)(struct task_struct *a, struct task_struct *b);
++
++ /**
++ * set_weight - Set task weight
++ * @p: task to set weight for
++ * @weight: new weight [1..10000]
++ *
++ * Update @p's weight to @weight.
++ */
++ void (*set_weight)(struct task_struct *p, u32 weight);
++
++ /**
++ * set_cpumask - Set CPU affinity
++ * @p: task to set CPU affinity for
++ * @cpumask: cpumask of cpus that @p can run on
++ *
++ * Update @p's CPU affinity to @cpumask.
++ */
++ void (*set_cpumask)(struct task_struct *p,
++ const struct cpumask *cpumask);
++
++ /**
++ * update_idle - Update the idle state of a CPU
++ * @cpu: CPU to udpate the idle state for
++ * @idle: whether entering or exiting the idle state
++ *
++ * This operation is called when @rq's CPU goes or leaves the idle
++ * state. By default, implementing this operation disables the built-in
++ * idle CPU tracking and the following helpers become unavailable:
++ *
++ * - scx_bpf_select_cpu_dfl()
++ * - scx_bpf_test_and_clear_cpu_idle()
++ * - scx_bpf_pick_idle_cpu()
++ *
++ * The user also must implement ops.select_cpu() as the default
++ * implementation relies on scx_bpf_select_cpu_dfl().
++ *
++ * Specify the %SCX_OPS_KEEP_BUILTIN_IDLE flag to keep the built-in idle
++ * tracking.
++ */
++ void (*update_idle)(s32 cpu, bool idle);
++
++ /**
++ * cpu_acquire - A CPU is becoming available to the BPF scheduler
++ * @cpu: The CPU being acquired by the BPF scheduler.
++ * @args: Acquire arguments, see the struct definition.
++ *
++ * A CPU that was previously released from the BPF scheduler is now once
++ * again under its control.
++ */
++ void (*cpu_acquire)(s32 cpu, struct scx_cpu_acquire_args *args);
++
++ /**
++ * cpu_release - A CPU is taken away from the BPF scheduler
++ * @cpu: The CPU being released by the BPF scheduler.
++ * @args: Release arguments, see the struct definition.
++ *
++ * The specified CPU is no longer under the control of the BPF
++ * scheduler. This could be because it was preempted by a higher
++ * priority sched_class, though there may be other reasons as well. The
++ * caller should consult @args->reason to determine the cause.
++ */
++ void (*cpu_release)(s32 cpu, struct scx_cpu_release_args *args);
++
++ /**
++ * init_task - Initialize a task to run in a BPF scheduler
++ * @p: task to initialize for BPF scheduling
++ * @args: init arguments, see the struct definition
++ *
++ * Either we're loading a BPF scheduler or a new task is being forked.
++ * Initialize @p for BPF scheduling. This operation may block and can
++ * be used for allocations, and is called exactly once for a task.
++ *
++ * Return 0 for success, -errno for failure. An error return while
++ * loading will abort loading of the BPF scheduler. During a fork, it
++ * will abort that specific fork.
++ */
++ s32 (*init_task)(struct task_struct *p, struct scx_init_task_args *args);
++
++ /**
++ * exit_task - Exit a previously-running task from the system
++ * @p: task to exit
++ *
++ * @p is exiting or the BPF scheduler is being unloaded. Perform any
++ * necessary cleanup for @p.
++ */
++ void (*exit_task)(struct task_struct *p, struct scx_exit_task_args *args);
++
++ /**
++ * enable - Enable BPF scheduling for a task
++ * @p: task to enable BPF scheduling for
++ *
++ * Enable @p for BPF scheduling. enable() is called on @p any time it
++ * enters SCX, and is always paired with a matching disable().
++ */
++ void (*enable)(struct task_struct *p);
++
++ /**
++ * disable - Disable BPF scheduling for a task
++ * @p: task to disable BPF scheduling for
++ *
++ * @p is exiting, leaving SCX or the BPF scheduler is being unloaded.
++ * Disable BPF scheduling for @p. A disable() call is always matched
++ * with a prior enable() call.
++ */
++ void (*disable)(struct task_struct *p);
++
++ /**
++ * dump - Dump BPF scheduler state on error
++ * @ctx: debug dump context
++ *
++ * Use scx_bpf_dump() to generate BPF scheduler specific debug dump.
++ */
++ void (*dump)(struct scx_dump_ctx *ctx);
++
++ /**
++ * dump_cpu - Dump BPF scheduler state for a CPU on error
++ * @ctx: debug dump context
++ * @cpu: CPU to generate debug dump for
++ * @idle: @cpu is currently idle without any runnable tasks
++ *
++ * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for
++ * @cpu. If @idle is %true and this operation doesn't produce any
++ * output, @cpu is skipped for dump.
++ */
++ void (*dump_cpu)(struct scx_dump_ctx *ctx, s32 cpu, bool idle);
++
++ /**
++ * dump_task - Dump BPF scheduler state for a runnable task on error
++ * @ctx: debug dump context
++ * @p: runnable task to generate debug dump for
++ *
++ * Use scx_bpf_dump() to generate BPF scheduler specific debug dump for
++ * @p.
++ */
++ void (*dump_task)(struct scx_dump_ctx *ctx, struct task_struct *p);
++
++ /*
++ * All online ops must come before ops.cpu_online().
++ */
++
++ /**
++ * cpu_online - A CPU became online
++ * @cpu: CPU which just came up
++ *
++ * @cpu just came online. @cpu will not call ops.enqueue() or
++ * ops.dispatch(), nor run tasks associated with other CPUs beforehand.
++ */
++ void (*cpu_online)(s32 cpu);
++
++ /**
++ * cpu_offline - A CPU is going offline
++ * @cpu: CPU which is going offline
++ *
++ * @cpu is going offline. @cpu will not call ops.enqueue() or
++ * ops.dispatch(), nor run tasks associated with other CPUs afterwards.
++ */
++ void (*cpu_offline)(s32 cpu);
++
++ /*
++ * All CPU hotplug ops must come before ops.init().
++ */
++
++ /**
++ * init - Initialize the BPF scheduler
++ */
++ s32 (*init)(void);
++
++ /**
++ * exit - Clean up after the BPF scheduler
++ * @info: Exit info
++ */
++ void (*exit)(struct scx_exit_info *info);
++
++ /**
++ * dispatch_max_batch - Max nr of tasks that dispatch() can dispatch
++ */
++ u32 dispatch_max_batch;
++
++ /**
++ * flags - %SCX_OPS_* flags
++ */
++ u64 flags;
++
++ /**
++ * timeout_ms - The maximum amount of time, in milliseconds, that a
++ * runnable task should be able to wait before being scheduled. The
++ * maximum timeout may not exceed the default timeout of 30 seconds.
++ *
++ * Defaults to the maximum allowed timeout value of 30 seconds.
++ */
++ u32 timeout_ms;
++
++ /**
++ * exit_dump_len - scx_exit_info.dump buffer length. If 0, the default
++ * value of 32768 is used.
++ */
++ u32 exit_dump_len;
++
++ /**
++ * hotplug_seq - A sequence number that may be set by the scheduler to
++ * detect when a hotplug event has occurred during the loading process.
++ * If 0, no detection occurs. Otherwise, the scheduler will fail to
++ * load if the sequence number does not match @scx_hotplug_seq on the
++ * enable path.
++ */
++ u64 hotplug_seq;
++
++ /**
++ * name - BPF scheduler's name
++ *
++ * Must be a non-zero valid BPF object name including only isalnum(),
++ * '_' and '.' chars. Shows up in kernel.sched_ext_ops sysctl while the
++ * BPF scheduler is enabled.
++ */
++ char name[SCX_OPS_NAME_LEN];
++};
++
++enum scx_opi {
++ SCX_OPI_BEGIN = 0,
++ SCX_OPI_NORMAL_BEGIN = 0,
++ SCX_OPI_NORMAL_END = SCX_OP_IDX(cpu_online),
++ SCX_OPI_CPU_HOTPLUG_BEGIN = SCX_OP_IDX(cpu_online),
++ SCX_OPI_CPU_HOTPLUG_END = SCX_OP_IDX(init),
++ SCX_OPI_END = SCX_OP_IDX(init),
++};
++
++enum scx_wake_flags {
++ /* expose select WF_* flags as enums */
++ SCX_WAKE_FORK = WF_FORK,
++ SCX_WAKE_TTWU = WF_TTWU,
++ SCX_WAKE_SYNC = WF_SYNC,
++};
++
++enum scx_enq_flags {
++ /* expose select ENQUEUE_* flags as enums */
++ SCX_ENQ_WAKEUP = ENQUEUE_WAKEUP,
++ SCX_ENQ_HEAD = ENQUEUE_HEAD,
++
++ /* high 32bits are SCX specific */
++
++ /*
++ * Set the following to trigger preemption when calling
++ * scx_bpf_dispatch() with a local dsq as the target. The slice of the
++ * current task is cleared to zero and the CPU is kicked into the
++ * scheduling path. Implies %SCX_ENQ_HEAD.
++ */
++ SCX_ENQ_PREEMPT = 1LLU << 32,
++
++ /*
++ * The task being enqueued was previously enqueued on the current CPU's
++ * %SCX_DSQ_LOCAL, but was removed from it in a call to the
++ * bpf_scx_reenqueue_local() kfunc. If bpf_scx_reenqueue_local() was
++ * invoked in a ->cpu_release() callback, and the task is again
++ * dispatched back to %SCX_LOCAL_DSQ by this current ->enqueue(), the
++ * task will not be scheduled on the CPU until at least the next invocation
++ * of the ->cpu_acquire() callback.
++ */
++ SCX_ENQ_REENQ = 1LLU << 40,
++
++ /*
++ * The task being enqueued is the only task available for the cpu. By
++ * default, ext core keeps executing such tasks but when
++ * %SCX_OPS_ENQ_LAST is specified, they're ops.enqueue()'d with the
++ * %SCX_ENQ_LAST flag set.
++ *
++ * If the BPF scheduler wants to continue executing the task,
++ * ops.enqueue() should dispatch the task to %SCX_DSQ_LOCAL immediately.
++ * If the task gets queued on a different dsq or the BPF side, the BPF
++ * scheduler is responsible for triggering a follow-up scheduling event.
++ * Otherwise, Execution may stall.
++ */
++ SCX_ENQ_LAST = 1LLU << 41,
++
++ /* high 8 bits are internal */
++ __SCX_ENQ_INTERNAL_MASK = 0xffLLU << 56,
++
++ SCX_ENQ_CLEAR_OPSS = 1LLU << 56,
++ SCX_ENQ_DSQ_PRIQ = 1LLU << 57,
++};
++
++enum scx_deq_flags {
++ /* expose select DEQUEUE_* flags as enums */
++ SCX_DEQ_SLEEP = DEQUEUE_SLEEP,
++
++ /* high 32bits are SCX specific */
++
++ /*
++ * The generic core-sched layer decided to execute the task even though
++ * it hasn't been dispatched yet. Dequeue from the BPF side.
++ */
++ SCX_DEQ_CORE_SCHED_EXEC = 1LLU << 32,
++};
++
++enum scx_pick_idle_cpu_flags {
++ SCX_PICK_IDLE_CORE = 1LLU << 0, /* pick a CPU whose SMT siblings are also idle */
++};
++
++enum scx_kick_flags {
++ /*
++ * Kick the target CPU if idle. Guarantees that the target CPU goes
++ * through at least one full scheduling cycle before going idle. If the
++ * target CPU can be determined to be currently not idle and going to go
++ * through a scheduling cycle before going idle, noop.
++ */
++ SCX_KICK_IDLE = 1LLU << 0,
++
++ /*
++ * Preempt the current task and execute the dispatch path. If the
++ * current task of the target CPU is an SCX task, its ->scx.slice is
++ * cleared to zero before the scheduling path is invoked so that the
++ * task expires and the dispatch path is invoked.
++ */
++ SCX_KICK_PREEMPT = 1LLU << 1,
++
++ /*
++ * Wait for the CPU to be rescheduled. The scx_bpf_kick_cpu() call will
++ * return after the target CPU finishes picking the next task.
++ */
++ SCX_KICK_WAIT = 1LLU << 2,
++};
++
++enum scx_ops_enable_state {
++ SCX_OPS_PREPPING,
++ SCX_OPS_ENABLING,
++ SCX_OPS_ENABLED,
++ SCX_OPS_DISABLING,
++ SCX_OPS_DISABLED,
++};
++
++static const char *scx_ops_enable_state_str[] = {
++ [SCX_OPS_PREPPING] = "prepping",
++ [SCX_OPS_ENABLING] = "enabling",
++ [SCX_OPS_ENABLED] = "enabled",
++ [SCX_OPS_DISABLING] = "disabling",
++ [SCX_OPS_DISABLED] = "disabled",
++};
++
++/*
++ * sched_ext_entity->ops_state
++ *
++ * Used to track the task ownership between the SCX core and the BPF scheduler.
++ * State transitions look as follows:
++ *
++ * NONE -> QUEUEING -> QUEUED -> DISPATCHING
++ * ^ | |
++ * | v v
++ * \-------------------------------/
++ *
++ * QUEUEING and DISPATCHING states can be waited upon. See wait_ops_state() call
++ * sites for explanations on the conditions being waited upon and why they are
++ * safe. Transitions out of them into NONE or QUEUED must store_release and the
++ * waiters should load_acquire.
++ *
++ * Tracking scx_ops_state enables sched_ext core to reliably determine whether
++ * any given task can be dispatched by the BPF scheduler at all times and thus
++ * relaxes the requirements on the BPF scheduler. This allows the BPF scheduler
++ * to try to dispatch any task anytime regardless of its state as the SCX core
++ * can safely reject invalid dispatches.
++ */
++enum scx_ops_state {
++ SCX_OPSS_NONE, /* owned by the SCX core */
++ SCX_OPSS_QUEUEING, /* in transit to the BPF scheduler */
++ SCX_OPSS_QUEUED, /* owned by the BPF scheduler */
++ SCX_OPSS_DISPATCHING, /* in transit back to the SCX core */
++
++ /*
++ * QSEQ brands each QUEUED instance so that, when dispatch races
++ * dequeue/requeue, the dispatcher can tell whether it still has a claim
++ * on the task being dispatched.
++ *
++ * As some 32bit archs can't do 64bit store_release/load_acquire,
++ * p->scx.ops_state is atomic_long_t which leaves 30 bits for QSEQ on
++ * 32bit machines. The dispatch race window QSEQ protects is very narrow
++ * and runs with IRQ disabled. 30 bits should be sufficient.
++ */
++ SCX_OPSS_QSEQ_SHIFT = 2,
++};
++
++/* Use macros to ensure that the type is unsigned long for the masks */
++#define SCX_OPSS_STATE_MASK ((1LU << SCX_OPSS_QSEQ_SHIFT) - 1)
++#define SCX_OPSS_QSEQ_MASK (~SCX_OPSS_STATE_MASK)
++
++/*
++ * During exit, a task may schedule after losing its PIDs. When disabling the
++ * BPF scheduler, we need to be able to iterate tasks in every state to
++ * guarantee system safety. Maintain a dedicated task list which contains every
++ * task between its fork and eventual free.
++ */
++static DEFINE_SPINLOCK(scx_tasks_lock);
++static LIST_HEAD(scx_tasks);
++
++/* ops enable/disable */
++static struct kthread_worker *scx_ops_helper;
++static DEFINE_MUTEX(scx_ops_enable_mutex);
++DEFINE_STATIC_KEY_FALSE(__scx_ops_enabled);
++DEFINE_STATIC_PERCPU_RWSEM(scx_fork_rwsem);
++static atomic_t scx_ops_enable_state_var = ATOMIC_INIT(SCX_OPS_DISABLED);
++static atomic_t scx_ops_bypass_depth = ATOMIC_INIT(0);
++static bool scx_switching_all;
++DEFINE_STATIC_KEY_FALSE(__scx_switched_all);
++
++static struct sched_ext_ops scx_ops;
++static bool scx_warned_zero_slice;
++
++static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_last);
++static DEFINE_STATIC_KEY_FALSE(scx_ops_enq_exiting);
++static DEFINE_STATIC_KEY_FALSE(scx_ops_cpu_preempt);
++static DEFINE_STATIC_KEY_FALSE(scx_builtin_idle_enabled);
++
++struct static_key_false scx_has_op[SCX_OPI_END] =
++ { [0 ... SCX_OPI_END-1] = STATIC_KEY_FALSE_INIT };
++
++static atomic_t scx_exit_kind = ATOMIC_INIT(SCX_EXIT_DONE);
++static struct scx_exit_info *scx_exit_info;
++
++static atomic_long_t scx_nr_rejected = ATOMIC_LONG_INIT(0);
++static atomic_long_t scx_hotplug_seq = ATOMIC_LONG_INIT(0);
++
++/*
++ * The maximum amount of time in jiffies that a task may be runnable without
++ * being scheduled on a CPU. If this timeout is exceeded, it will trigger
++ * scx_ops_error().
++ */
++static unsigned long scx_watchdog_timeout;
++
++/*
++ * The last time the delayed work was run. This delayed work relies on
++ * ksoftirqd being able to run to service timer interrupts, so it's possible
++ * that this work itself could get wedged. To account for this, we check that
++ * it's not stalled in the timer tick, and trigger an error if it is.
++ */
++static unsigned long scx_watchdog_timestamp = INITIAL_JIFFIES;
++
++static struct delayed_work scx_watchdog_work;
++
++/* idle tracking */
++#ifdef CONFIG_SMP
++#ifdef CONFIG_CPUMASK_OFFSTACK
++#define CL_ALIGNED_IF_ONSTACK
++#else
++#define CL_ALIGNED_IF_ONSTACK __cacheline_aligned_in_smp
++#endif
++
++static struct {
++ cpumask_var_t cpu;
++ cpumask_var_t smt;
++} idle_masks CL_ALIGNED_IF_ONSTACK;
++
++#endif /* CONFIG_SMP */
++
++/* for %SCX_KICK_WAIT */
++static unsigned long __percpu *scx_kick_cpus_pnt_seqs;
++
++/*
++ * Direct dispatch marker.
++ *
++ * Non-NULL values are used for direct dispatch from enqueue path. A valid
++ * pointer points to the task currently being enqueued. An ERR_PTR value is used
++ * to indicate that direct dispatch has already happened.
++ */
++static DEFINE_PER_CPU(struct task_struct *, direct_dispatch_task);
++
++/* dispatch queues */
++static struct scx_dispatch_q __cacheline_aligned_in_smp scx_dsq_global;
++
++static const struct rhashtable_params dsq_hash_params = {
++ .key_len = 8,
++ .key_offset = offsetof(struct scx_dispatch_q, id),
++ .head_offset = offsetof(struct scx_dispatch_q, hash_node),
++};
++
++static struct rhashtable dsq_hash;
++static LLIST_HEAD(dsqs_to_free);
++
++/* dispatch buf */
++struct scx_dsp_buf_ent {
++ struct task_struct *task;
++ unsigned long qseq;
++ u64 dsq_id;
++ u64 enq_flags;
++};
++
++static u32 scx_dsp_max_batch;
++
++struct scx_dsp_ctx {
++ struct rq *rq;
++ u32 cursor;
++ u32 nr_tasks;
++ struct scx_dsp_buf_ent buf[];
++};
++
++static struct scx_dsp_ctx __percpu *scx_dsp_ctx;
++
++/* string formatting from BPF */
++struct scx_bstr_buf {
++ u64 data[MAX_BPRINTF_VARARGS];
++ char line[SCX_EXIT_MSG_LEN];
++};
++
++static DEFINE_RAW_SPINLOCK(scx_exit_bstr_buf_lock);
++static struct scx_bstr_buf scx_exit_bstr_buf;
++
++/* ops debug dump */
++struct scx_dump_data {
++ s32 cpu;
++ bool first;
++ s32 cursor;
++ struct seq_buf *s;
++ const char *prefix;
++ struct scx_bstr_buf buf;
++};
++
++struct scx_dump_data scx_dump_data = {
++ .cpu = -1,
++};
++
++/* /sys/kernel/sched_ext interface */
++static struct kset *scx_kset;
++static struct kobject *scx_root_kobj;
++
++#define CREATE_TRACE_POINTS
++#include <trace/events/sched_ext.h>
++
++static void process_ddsp_deferred_locals(struct rq *rq);
++static void scx_bpf_kick_cpu(s32 cpu, u64 flags);
++static __printf(3, 4) void scx_ops_exit_kind(enum scx_exit_kind kind,
++ s64 exit_code,
++ const char *fmt, ...);
++
++#define scx_ops_error_kind(err, fmt, args...) \
++ scx_ops_exit_kind((err), 0, fmt, ##args)
++
++#define scx_ops_exit(code, fmt, args...) \
++ scx_ops_exit_kind(SCX_EXIT_UNREG_KERN, (code), fmt, ##args)
++
++#define scx_ops_error(fmt, args...) \
++ scx_ops_error_kind(SCX_EXIT_ERROR, fmt, ##args)
++
++#define SCX_HAS_OP(op) static_branch_likely(&scx_has_op[SCX_OP_IDX(op)])
++
++static long jiffies_delta_msecs(unsigned long at, unsigned long now)
++{
++ if (time_after(at, now))
++ return jiffies_to_msecs(at - now);
++ else
++ return -(long)jiffies_to_msecs(now - at);
++}
++
++/* if the highest set bit is N, return a mask with bits [N+1, 31] set */
++static u32 higher_bits(u32 flags)
++{
++ return ~((1 << fls(flags)) - 1);
++}
++
++/* return the mask with only the highest bit set */
++static u32 highest_bit(u32 flags)
++{
++ int bit = fls(flags);
++ return ((u64)1 << bit) >> 1;
++}
++
++static bool u32_before(u32 a, u32 b)
++{
++ return (s32)(a - b) < 0;
++}
++
++/*
++ * scx_kf_mask enforcement. Some kfuncs can only be called from specific SCX
++ * ops. When invoking SCX ops, SCX_CALL_OP[_RET]() should be used to indicate
++ * the allowed kfuncs and those kfuncs should use scx_kf_allowed() to check
++ * whether it's running from an allowed context.
++ *
++ * @mask is constant, always inline to cull the mask calculations.
++ */
++static __always_inline void scx_kf_allow(u32 mask)
++{
++ /* nesting is allowed only in increasing scx_kf_mask order */
++ WARN_ONCE((mask | higher_bits(mask)) & current->scx.kf_mask,
++ "invalid nesting current->scx.kf_mask=0x%x mask=0x%x\n",
++ current->scx.kf_mask, mask);
++ current->scx.kf_mask |= mask;
++ barrier();
++}
++
++static void scx_kf_disallow(u32 mask)
++{
++ barrier();
++ current->scx.kf_mask &= ~mask;
++}
++
++#define SCX_CALL_OP(mask, op, args...) \
++do { \
++ if (mask) { \
++ scx_kf_allow(mask); \
++ scx_ops.op(args); \
++ scx_kf_disallow(mask); \
++ } else { \
++ scx_ops.op(args); \
++ } \
++} while (0)
++
++#define SCX_CALL_OP_RET(mask, op, args...) \
++({ \
++ __typeof__(scx_ops.op(args)) __ret; \
++ if (mask) { \
++ scx_kf_allow(mask); \
++ __ret = scx_ops.op(args); \
++ scx_kf_disallow(mask); \
++ } else { \
++ __ret = scx_ops.op(args); \
++ } \
++ __ret; \
++})
++
++/*
++ * Some kfuncs are allowed only on the tasks that are subjects of the
++ * in-progress scx_ops operation for, e.g., locking guarantees. To enforce such
++ * restrictions, the following SCX_CALL_OP_*() variants should be used when
++ * invoking scx_ops operations that take task arguments. These can only be used
++ * for non-nesting operations due to the way the tasks are tracked.
++ *
++ * kfuncs which can only operate on such tasks can in turn use
++ * scx_kf_allowed_on_arg_tasks() to test whether the invocation is allowed on
++ * the specific task.
++ */
++#define SCX_CALL_OP_TASK(mask, op, task, args...) \
++do { \
++ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \
++ current->scx.kf_tasks[0] = task; \
++ SCX_CALL_OP(mask, op, task, ##args); \
++ current->scx.kf_tasks[0] = NULL; \
++} while (0)
++
++#define SCX_CALL_OP_TASK_RET(mask, op, task, args...) \
++({ \
++ __typeof__(scx_ops.op(task, ##args)) __ret; \
++ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \
++ current->scx.kf_tasks[0] = task; \
++ __ret = SCX_CALL_OP_RET(mask, op, task, ##args); \
++ current->scx.kf_tasks[0] = NULL; \
++ __ret; \
++})
++
++#define SCX_CALL_OP_2TASKS_RET(mask, op, task0, task1, args...) \
++({ \
++ __typeof__(scx_ops.op(task0, task1, ##args)) __ret; \
++ BUILD_BUG_ON((mask) & ~__SCX_KF_TERMINAL); \
++ current->scx.kf_tasks[0] = task0; \
++ current->scx.kf_tasks[1] = task1; \
++ __ret = SCX_CALL_OP_RET(mask, op, task0, task1, ##args); \
++ current->scx.kf_tasks[0] = NULL; \
++ current->scx.kf_tasks[1] = NULL; \
++ __ret; \
++})
++
++/* @mask is constant, always inline to cull unnecessary branches */
++static __always_inline bool scx_kf_allowed(u32 mask)
++{
++ if (unlikely(!(current->scx.kf_mask & mask))) {
++ scx_ops_error("kfunc with mask 0x%x called from an operation only allowing 0x%x",
++ mask, current->scx.kf_mask);
++ return false;
++ }
++
++ if (unlikely((mask & SCX_KF_SLEEPABLE) && in_interrupt())) {
++ scx_ops_error("sleepable kfunc called from non-sleepable context");
++ return false;
++ }
++
++ /*
++ * Enforce nesting boundaries. e.g. A kfunc which can be called from
++ * DISPATCH must not be called if we're running DEQUEUE which is nested
++ * inside ops.dispatch(). We don't need to check the SCX_KF_SLEEPABLE
++ * boundary thanks to the above in_interrupt() check.
++ */
++ if (unlikely(highest_bit(mask) == SCX_KF_CPU_RELEASE &&
++ (current->scx.kf_mask & higher_bits(SCX_KF_CPU_RELEASE)))) {
++ scx_ops_error("cpu_release kfunc called from a nested operation");
++ return false;
++ }
++
++ if (unlikely(highest_bit(mask) == SCX_KF_DISPATCH &&
++ (current->scx.kf_mask & higher_bits(SCX_KF_DISPATCH)))) {
++ scx_ops_error("dispatch kfunc called from a nested operation");
++ return false;
++ }
++
++ return true;
++}
++
++/* see SCX_CALL_OP_TASK() */
++static __always_inline bool scx_kf_allowed_on_arg_tasks(u32 mask,
++ struct task_struct *p)
++{
++ if (!scx_kf_allowed(mask))
++ return false;
++
++ if (unlikely((p != current->scx.kf_tasks[0] &&
++ p != current->scx.kf_tasks[1]))) {
++ scx_ops_error("called on a task not being operated on");
++ return false;
++ }
++
++ return true;
++}
++
++/**
++ * nldsq_next_task - Iterate to the next task in a non-local DSQ
++ * @dsq: user dsq being interated
++ * @cur: current position, %NULL to start iteration
++ * @rev: walk backwards
++ *
++ * Returns %NULL when iteration is finished.
++ */
++static struct task_struct *nldsq_next_task(struct scx_dispatch_q *dsq,
++ struct task_struct *cur, bool rev)
++{
++ struct list_head *list_node;
++ struct scx_dsq_list_node *dsq_lnode;
++
++ lockdep_assert_held(&dsq->lock);
++
++ if (cur)
++ list_node = &cur->scx.dsq_list.node;
++ else
++ list_node = &dsq->list;
++
++ /* find the next task, need to skip BPF iteration cursors */
++ do {
++ if (rev)
++ list_node = list_node->prev;
++ else
++ list_node = list_node->next;
++
++ if (list_node == &dsq->list)
++ return NULL;
++
++ dsq_lnode = container_of(list_node, struct scx_dsq_list_node,
++ node);
++ } while (dsq_lnode->is_bpf_iter_cursor);
++
++ return container_of(dsq_lnode, struct task_struct, scx.dsq_list);
++}
++
++#define nldsq_for_each_task(p, dsq) \
++ for ((p) = nldsq_next_task((dsq), NULL, false); (p); \
++ (p) = nldsq_next_task((dsq), (p), false))
++
++
++/*
++ * BPF DSQ iterator. Tasks in a non-local DSQ can be iterated in [reverse]
++ * dispatch order. BPF-visible iterator is opaque and larger to allow future
++ * changes without breaking backward compatibility. Can be used with
++ * bpf_for_each(). See bpf_iter_scx_dsq_*().
++ */
++enum scx_dsq_iter_flags {
++ /* iterate in the reverse dispatch order */
++ SCX_DSQ_ITER_REV = 1U << 0,
++
++ __SCX_DSQ_ITER_ALL_FLAGS = SCX_DSQ_ITER_REV,
++};
++
++struct bpf_iter_scx_dsq_kern {
++ struct scx_dsq_list_node cursor;
++ struct scx_dispatch_q *dsq;
++ u32 dsq_seq;
++ u32 flags;
++} __attribute__((aligned(8)));
++
++struct bpf_iter_scx_dsq {
++ u64 __opaque[6];
++} __attribute__((aligned(8)));
++
++
++/*
++ * SCX task iterator.
++ */
++struct scx_task_iter {
++ struct sched_ext_entity cursor;
++ struct task_struct *locked;
++ struct rq *rq;
++ struct rq_flags rf;
++};
++
++/**
++ * scx_task_iter_init - Initialize a task iterator
++ * @iter: iterator to init
++ *
++ * Initialize @iter. Must be called with scx_tasks_lock held. Once initialized,
++ * @iter must eventually be exited with scx_task_iter_exit().
++ *
++ * scx_tasks_lock may be released between this and the first next() call or
++ * between any two next() calls. If scx_tasks_lock is released between two
++ * next() calls, the caller is responsible for ensuring that the task being
++ * iterated remains accessible either through RCU read lock or obtaining a
++ * reference count.
++ *
++ * All tasks which existed when the iteration started are guaranteed to be
++ * visited as long as they still exist.
++ */
++static void scx_task_iter_init(struct scx_task_iter *iter)
++{
++ lockdep_assert_held(&scx_tasks_lock);
++
++ iter->cursor = (struct sched_ext_entity){ .flags = SCX_TASK_CURSOR };
++ list_add(&iter->cursor.tasks_node, &scx_tasks);
++ iter->locked = NULL;
++}
++
++/**
++ * scx_task_iter_rq_unlock - Unlock rq locked by a task iterator
++ * @iter: iterator to unlock rq for
++ *
++ * If @iter is in the middle of a locked iteration, it may be locking the rq of
++ * the task currently being visited. Unlock the rq if so. This function can be
++ * safely called anytime during an iteration.
++ *
++ * Returns %true if the rq @iter was locking is unlocked. %false if @iter was
++ * not locking an rq.
++ */
++static bool scx_task_iter_rq_unlock(struct scx_task_iter *iter)
++{
++ if (iter->locked) {
++ task_rq_unlock(iter->rq, iter->locked, &iter->rf);
++ iter->locked = NULL;
++ return true;
++ } else {
++ return false;
++ }
++}
++
++/**
++ * scx_task_iter_exit - Exit a task iterator
++ * @iter: iterator to exit
++ *
++ * Exit a previously initialized @iter. Must be called with scx_tasks_lock held.
++ * If the iterator holds a task's rq lock, that rq lock is released. See
++ * scx_task_iter_init() for details.
++ */
++static void scx_task_iter_exit(struct scx_task_iter *iter)
++{
++ lockdep_assert_held(&scx_tasks_lock);
++
++ scx_task_iter_rq_unlock(iter);
++ list_del_init(&iter->cursor.tasks_node);
++}
++
++/**
++ * scx_task_iter_next - Next task
++ * @iter: iterator to walk
++ *
++ * Visit the next task. See scx_task_iter_init() for details.
++ */
++static struct task_struct *scx_task_iter_next(struct scx_task_iter *iter)
++{
++ struct list_head *cursor = &iter->cursor.tasks_node;
++ struct sched_ext_entity *pos;
++
++ lockdep_assert_held(&scx_tasks_lock);
++
++ list_for_each_entry(pos, cursor, tasks_node) {
++ if (&pos->tasks_node == &scx_tasks)
++ return NULL;
++ if (!(pos->flags & SCX_TASK_CURSOR)) {
++ list_move(cursor, &pos->tasks_node);
++ return container_of(pos, struct task_struct, scx);
++ }
++ }
++
++ /* can't happen, should always terminate at scx_tasks above */
++ BUG();
++}
++
++/**
++ * scx_task_iter_next_locked - Next non-idle task with its rq locked
++ * @iter: iterator to walk
++ * @include_dead: Whether we should include dead tasks in the iteration
++ *
++ * Visit the non-idle task with its rq lock held. Allows callers to specify
++ * whether they would like to filter out dead tasks. See scx_task_iter_init()
++ * for details.
++ */
++static struct task_struct *
++scx_task_iter_next_locked(struct scx_task_iter *iter, bool include_dead)
++{
++ struct task_struct *p;
++retry:
++ scx_task_iter_rq_unlock(iter);
++
++ while ((p = scx_task_iter_next(iter))) {
++ /*
++ * is_idle_task() tests %PF_IDLE which may not be set for CPUs
++ * which haven't yet been onlined. Test sched_class directly.
++ */
++ if (p->sched_class != &idle_sched_class)
++ break;
++ }
++ if (!p)
++ return NULL;
++
++ iter->rq = task_rq_lock(p, &iter->rf);
++ iter->locked = p;
++
++ /*
++ * If we see %TASK_DEAD, @p already disabled preemption, is about to do
++ * the final __schedule(), won't ever need to be scheduled again and can
++ * thus be safely ignored. If we don't see %TASK_DEAD, @p can't enter
++ * the final __schedle() while we're locking its rq and thus will stay
++ * alive until the rq is unlocked.
++ */
++ if (!include_dead && READ_ONCE(p->__state) == TASK_DEAD)
++ goto retry;
++
++ return p;
++}
++
++static enum scx_ops_enable_state scx_ops_enable_state(void)
++{
++ return atomic_read(&scx_ops_enable_state_var);
++}
++
++static enum scx_ops_enable_state
++scx_ops_set_enable_state(enum scx_ops_enable_state to)
++{
++ return atomic_xchg(&scx_ops_enable_state_var, to);
++}
++
++static bool scx_ops_tryset_enable_state(enum scx_ops_enable_state to,
++ enum scx_ops_enable_state from)
++{
++ int from_v = from;
++
++ return atomic_try_cmpxchg(&scx_ops_enable_state_var, &from_v, to);
++}
++
++static bool scx_ops_bypassing(void)
++{
++ return unlikely(atomic_read(&scx_ops_bypass_depth));
++}
++
++/**
++ * wait_ops_state - Busy-wait the specified ops state to end
++ * @p: target task
++ * @opss: state to wait the end of
++ *
++ * Busy-wait for @p to transition out of @opss. This can only be used when the
++ * state part of @opss is %SCX_QUEUEING or %SCX_DISPATCHING. This function also
++ * has load_acquire semantics to ensure that the caller can see the updates made
++ * in the enqueueing and dispatching paths.
++ */
++static void wait_ops_state(struct task_struct *p, unsigned long opss)
++{
++ do {
++ cpu_relax();
++ } while (atomic_long_read_acquire(&p->scx.ops_state) == opss);
++}
++
++/**
++ * ops_cpu_valid - Verify a cpu number
++ * @cpu: cpu number which came from a BPF ops
++ * @where: extra information reported on error
++ *
++ * @cpu is a cpu number which came from the BPF scheduler and can be any value.
++ * Verify that it is in range and one of the possible cpus. If invalid, trigger
++ * an ops error.
++ */
++static bool ops_cpu_valid(s32 cpu, const char *where)
++{
++ if (likely(cpu >= 0 && cpu < nr_cpu_ids && cpu_possible(cpu))) {
++ return true;
++ } else {
++ scx_ops_error("invalid CPU %d%s%s", cpu,
++ where ? " " : "", where ?: "");
++ return false;
++ }
++}
++
++/**
++ * ops_sanitize_err - Sanitize a -errno value
++ * @ops_name: operation to blame on failure
++ * @err: -errno value to sanitize
++ *
++ * Verify @err is a valid -errno. If not, trigger scx_ops_error() and return
++ * -%EPROTO. This is necessary because returning a rogue -errno up the chain can
++ * cause misbehaviors. For an example, a large negative return from
++ * ops.init_task() triggers an oops when passed up the call chain because the
++ * value fails IS_ERR() test after being encoded with ERR_PTR() and then is
++ * handled as a pointer.
++ */
++static int ops_sanitize_err(const char *ops_name, s32 err)
++{
++ if (err < 0 && err >= -MAX_ERRNO)
++ return err;
++
++ scx_ops_error("ops.%s() returned an invalid errno %d", ops_name, err);
++ return -EPROTO;
++}
++
++static void run_deferred(struct rq *rq)
++{
++ process_ddsp_deferred_locals(rq);
++}
++
++#ifdef CONFIG_SMP
++static void deferred_bal_cb_workfn(struct rq *rq)
++{
++ run_deferred(rq);
++}
++#endif
++
++static void deferred_irq_workfn(struct irq_work *irq_work)
++{
++ struct rq *rq = container_of(irq_work, struct rq, scx.deferred_irq_work);
++
++ raw_spin_rq_lock(rq);
++ run_deferred(rq);
++ raw_spin_rq_unlock(rq);
++}
++
++/**
++ * schedule_deferred - Schedule execution of deferred actions on an rq
++ * @rq: target rq
++ *
++ * Schedule execution of deferred actions on @rq. Must be called with @rq
++ * locked. Deferred actions are executed with @rq locked but unpinned, and thus
++ * can unlock @rq to e.g. migrate tasks to other rqs.
++ */
++static void schedule_deferred(struct rq *rq)
++{
++ lockdep_assert_rq_held(rq);
++
++#ifdef CONFIG_SMP
++ /*
++ * If in the middle of waking up a task, task_woken_scx() will be called
++ * afterwards which will then run the deferred actions, no need to
++ * schedule anything.
++ */
++ if (rq->scx.flags & SCX_RQ_IN_WAKEUP)
++ return;
++
++ /*
++ * If in balance, the balance callbacks will be called before rq lock is
++ * released. Schedule one.
++ */
++ if (rq->scx.flags & SCX_RQ_IN_BALANCE) {
++ queue_balance_callback(rq, &rq->scx.deferred_bal_cb,
++ deferred_bal_cb_workfn);
++ return;
++ }
++#endif
++ /*
++ * No scheduler hooks available. Queue an irq work. They are executed on
++ * IRQ re-enable which may take a bit longer than the scheduler hooks.
++ * The above WAKEUP and BALANCE paths should cover most of the cases and
++ * the time to IRQ re-enable shouldn't be long.
++ */
++ irq_work_queue(&rq->scx.deferred_irq_work);
++}
++
++/**
++ * touch_core_sched - Update timestamp used for core-sched task ordering
++ * @rq: rq to read clock from, must be locked
++ * @p: task to update the timestamp for
++ *
++ * Update @p->scx.core_sched_at timestamp. This is used by scx_prio_less() to
++ * implement global or local-DSQ FIFO ordering for core-sched. Should be called
++ * when a task becomes runnable and its turn on the CPU ends (e.g. slice
++ * exhaustion).
++ */
++static void touch_core_sched(struct rq *rq, struct task_struct *p)
++{
++#ifdef CONFIG_SCHED_CORE
++ /*
++ * It's okay to update the timestamp spuriously. Use
++ * sched_core_disabled() which is cheaper than enabled().
++ */
++ if (!sched_core_disabled())
++ p->scx.core_sched_at = rq_clock_task(rq);
++#endif
++}
++
++/**
++ * touch_core_sched_dispatch - Update core-sched timestamp on dispatch
++ * @rq: rq to read clock from, must be locked
++ * @p: task being dispatched
++ *
++ * If the BPF scheduler implements custom core-sched ordering via
++ * ops.core_sched_before(), @p->scx.core_sched_at is used to implement FIFO
++ * ordering within each local DSQ. This function is called from dispatch paths
++ * and updates @p->scx.core_sched_at if custom core-sched ordering is in effect.
++ */
++static void touch_core_sched_dispatch(struct rq *rq, struct task_struct *p)
++{
++ lockdep_assert_rq_held(rq);
++ assert_clock_updated(rq);
++
++#ifdef CONFIG_SCHED_CORE
++ if (SCX_HAS_OP(core_sched_before))
++ touch_core_sched(rq, p);
++#endif
++}
++
++static void update_curr_scx(struct rq *rq)
++{
++ struct task_struct *curr = rq->curr;
++ u64 now = rq_clock_task(rq);
++ u64 delta_exec;
++
++ if (time_before_eq64(now, curr->se.exec_start))
++ return;
++
++ delta_exec = now - curr->se.exec_start;
++ curr->se.exec_start = now;
++ curr->se.sum_exec_runtime += delta_exec;
++ account_group_exec_runtime(curr, delta_exec);
++ cgroup_account_cputime(curr, delta_exec);
++
++ if (curr->scx.slice != SCX_SLICE_INF) {
++ curr->scx.slice -= min(curr->scx.slice, delta_exec);
++ if (!curr->scx.slice)
++ touch_core_sched(rq, curr);
++ }
++}
++
++static bool scx_dsq_priq_less(struct rb_node *node_a,
++ const struct rb_node *node_b)
++{
++ const struct task_struct *a =
++ container_of(node_a, struct task_struct, scx.dsq_priq);
++ const struct task_struct *b =
++ container_of(node_b, struct task_struct, scx.dsq_priq);
++
++ return time_before64(a->scx.dsq_vtime, b->scx.dsq_vtime);
++}
++
++static void dsq_mod_nr(struct scx_dispatch_q *dsq, s32 delta)
++{
++ /* scx_bpf_dsq_nr_queued() reads ->nr without locking, use WRITE_ONCE() */
++ WRITE_ONCE(dsq->nr, dsq->nr + delta);
++}
++
++static void dispatch_enqueue(struct scx_dispatch_q *dsq, struct task_struct *p,
++ u64 enq_flags)
++{
++ bool is_local = dsq->id == SCX_DSQ_LOCAL;
++
++ WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
++ WARN_ON_ONCE((p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) ||
++ !RB_EMPTY_NODE(&p->scx.dsq_priq));
++
++ if (!is_local) {
++ raw_spin_lock(&dsq->lock);
++ if (unlikely(dsq->id == SCX_DSQ_INVALID)) {
++ scx_ops_error("attempting to dispatch to a destroyed dsq");
++ /* fall back to the global dsq */
++ raw_spin_unlock(&dsq->lock);
++ dsq = &scx_dsq_global;
++ raw_spin_lock(&dsq->lock);
++ }
++ }
++
++ if (unlikely((dsq->id & SCX_DSQ_FLAG_BUILTIN) &&
++ (enq_flags & SCX_ENQ_DSQ_PRIQ))) {
++ /*
++ * SCX_DSQ_LOCAL and SCX_DSQ_GLOBAL DSQs always consume from
++ * their FIFO queues. To avoid confusion and accidentally
++ * starving vtime-dispatched tasks by FIFO-dispatched tasks, we
++ * disallow any internal DSQ from doing vtime ordering of
++ * tasks.
++ */
++ scx_ops_error("cannot use vtime ordering for built-in DSQs");
++ enq_flags &= ~SCX_ENQ_DSQ_PRIQ;
++ }
++
++ if (enq_flags & SCX_ENQ_DSQ_PRIQ) {
++ struct rb_node *rbp;
++
++ /*
++ * A PRIQ DSQ shouldn't be using FIFO enqueueing. As tasks are
++ * linked to both the rbtree and list on PRIQs, this can only be
++ * tested easily when adding the first task.
++ */
++ if (unlikely(RB_EMPTY_ROOT(&dsq->priq) &&
++ nldsq_next_task(dsq, NULL, false)))
++ scx_ops_error("DSQ ID 0x%016llx already had FIFO-enqueued tasks",
++ dsq->id);
++
++ p->scx.dsq_flags |= SCX_TASK_DSQ_ON_PRIQ;
++ rb_add(&p->scx.dsq_priq, &dsq->priq, scx_dsq_priq_less);
++
++ /*
++ * Find the previous task and insert after it on the list so
++ * that @dsq->list is vtime ordered.
++ */
++ rbp = rb_prev(&p->scx.dsq_priq);
++ if (rbp) {
++ struct task_struct *prev =
++ container_of(rbp, struct task_struct,
++ scx.dsq_priq);
++ list_add(&p->scx.dsq_list.node, &prev->scx.dsq_list.node);
++ } else {
++ list_add(&p->scx.dsq_list.node, &dsq->list);
++ }
++ } else {
++ /* a FIFO DSQ shouldn't be using PRIQ enqueuing */
++ if (unlikely(!RB_EMPTY_ROOT(&dsq->priq)))
++ scx_ops_error("DSQ ID 0x%016llx already had PRIQ-enqueued tasks",
++ dsq->id);
++
++ if (enq_flags & (SCX_ENQ_HEAD | SCX_ENQ_PREEMPT))
++ list_add(&p->scx.dsq_list.node, &dsq->list);
++ else
++ list_add_tail(&p->scx.dsq_list.node, &dsq->list);
++ }
++
++ /* seq records the order tasks are queued, used by BPF DSQ iterator */
++ dsq->seq++;
++ p->scx.dsq_seq = dsq->seq;
++
++ dsq_mod_nr(dsq, 1);
++ p->scx.dsq = dsq;
++
++ /*
++ * scx.ddsp_dsq_id and scx.ddsp_enq_flags are only relevant on the
++ * direct dispatch path, but we clear them here because the direct
++ * dispatch verdict may be overridden on the enqueue path during e.g.
++ * bypass.
++ */
++ p->scx.ddsp_dsq_id = SCX_DSQ_INVALID;
++ p->scx.ddsp_enq_flags = 0;
++
++ /*
++ * We're transitioning out of QUEUEING or DISPATCHING. store_release to
++ * match waiters' load_acquire.
++ */
++ if (enq_flags & SCX_ENQ_CLEAR_OPSS)
++ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
++
++ if (is_local) {
++ struct rq *rq = container_of(dsq, struct rq, scx.local_dsq);
++ bool preempt = false;
++
++ if ((enq_flags & SCX_ENQ_PREEMPT) && p != rq->curr &&
++ rq->curr->sched_class == &ext_sched_class) {
++ rq->curr->scx.slice = 0;
++ preempt = true;
++ }
++
++ if (preempt || sched_class_above(&ext_sched_class,
++ rq->curr->sched_class))
++ resched_curr(rq);
++ } else {
++ raw_spin_unlock(&dsq->lock);
++ }
++}
++
++static void task_unlink_from_dsq(struct task_struct *p,
++ struct scx_dispatch_q *dsq)
++{
++ if (p->scx.dsq_flags & SCX_TASK_DSQ_ON_PRIQ) {
++ rb_erase(&p->scx.dsq_priq, &dsq->priq);
++ RB_CLEAR_NODE(&p->scx.dsq_priq);
++ p->scx.dsq_flags &= ~SCX_TASK_DSQ_ON_PRIQ;
++ }
++
++ list_del_init(&p->scx.dsq_list.node);
++}
++
++static void dispatch_dequeue(struct rq *rq, struct task_struct *p)
++{
++ struct scx_dispatch_q *dsq = p->scx.dsq;
++ bool is_local = dsq == &rq->scx.local_dsq;
++
++ if (!dsq) {
++ /*
++ * If !dsq && on-list, @p is on @rq's ddsp_deferred_locals.
++ * Unlinking is all that's needed to cancel.
++ */
++ if (unlikely(!list_empty(&p->scx.dsq_list.node)))
++ list_del_init(&p->scx.dsq_list.node);
++
++ /*
++ * When dispatching directly from the BPF scheduler to a local
++ * DSQ, the task isn't associated with any DSQ but
++ * @p->scx.holding_cpu may be set under the protection of
++ * %SCX_OPSS_DISPATCHING.
++ */
++ if (p->scx.holding_cpu >= 0)
++ p->scx.holding_cpu = -1;
++
++ return;
++ }
++
++ if (!is_local)
++ raw_spin_lock(&dsq->lock);
++
++ /*
++ * Now that we hold @dsq->lock, @p->holding_cpu and @p->scx.dsq_* can't
++ * change underneath us.
++ */
++ if (p->scx.holding_cpu < 0) {
++ /* @p must still be on @dsq, dequeue */
++ WARN_ON_ONCE(list_empty(&p->scx.dsq_list.node));
++ task_unlink_from_dsq(p, dsq);
++ dsq_mod_nr(dsq, -1);
++ } else {
++ /*
++ * We're racing against dispatch_to_local_dsq() which already
++ * removed @p from @dsq and set @p->scx.holding_cpu. Clear the
++ * holding_cpu which tells dispatch_to_local_dsq() that it lost
++ * the race.
++ */
++ WARN_ON_ONCE(!list_empty(&p->scx.dsq_list.node));
++ p->scx.holding_cpu = -1;
++ }
++ p->scx.dsq = NULL;
++
++ if (!is_local)
++ raw_spin_unlock(&dsq->lock);
++}
++
++static struct scx_dispatch_q *find_user_dsq(u64 dsq_id)
++{
++ return rhashtable_lookup_fast(&dsq_hash, &dsq_id, dsq_hash_params);
++}
++
++static struct scx_dispatch_q *find_non_local_dsq(u64 dsq_id)
++{
++ lockdep_assert(rcu_read_lock_any_held());
++
++ if (dsq_id == SCX_DSQ_GLOBAL)
++ return &scx_dsq_global;
++ else
++ return find_user_dsq(dsq_id);
++}
++
++static struct scx_dispatch_q *find_dsq_for_dispatch(struct rq *rq, u64 dsq_id,
++ struct task_struct *p)
++{
++ struct scx_dispatch_q *dsq;
++
++ if (dsq_id == SCX_DSQ_LOCAL)
++ return &rq->scx.local_dsq;
++
++ dsq = find_non_local_dsq(dsq_id);
++ if (unlikely(!dsq)) {
++ scx_ops_error("non-existent DSQ 0x%llx for %s[%d]",
++ dsq_id, p->comm, p->pid);
++ return &scx_dsq_global;
++ }
++
++ return dsq;
++}
++
++static void mark_direct_dispatch(struct task_struct *ddsp_task,
++ struct task_struct *p, u64 dsq_id,
++ u64 enq_flags)
++{
++ /*
++ * Mark that dispatch already happened from ops.select_cpu() or
++ * ops.enqueue() by spoiling direct_dispatch_task with a non-NULL value
++ * which can never match a valid task pointer.
++ */
++ __this_cpu_write(direct_dispatch_task, ERR_PTR(-ESRCH));
++
++ /* @p must match the task on the enqueue path */
++ if (unlikely(p != ddsp_task)) {
++ if (IS_ERR(ddsp_task))
++ scx_ops_error("%s[%d] already direct-dispatched",
++ p->comm, p->pid);
++ else
++ scx_ops_error("scheduling for %s[%d] but trying to direct-dispatch %s[%d]",
++ ddsp_task->comm, ddsp_task->pid,
++ p->comm, p->pid);
++ return;
++ }
++
++ WARN_ON_ONCE(p->scx.ddsp_dsq_id != SCX_DSQ_INVALID);
++ WARN_ON_ONCE(p->scx.ddsp_enq_flags);
++
++ p->scx.ddsp_dsq_id = dsq_id;
++ p->scx.ddsp_enq_flags = enq_flags;
++}
++
++static void direct_dispatch(struct task_struct *p, u64 enq_flags)
++{
++ struct rq *rq = task_rq(p);
++ struct scx_dispatch_q *dsq;
++ u64 dsq_id = p->scx.ddsp_dsq_id;
++
++ touch_core_sched_dispatch(rq, p);
++
++ p->scx.ddsp_enq_flags |= enq_flags;
++
++ /*
++ * We are in the enqueue path with @rq locked and pinned, and thus can't
++ * double lock a remote rq and enqueue to its local DSQ. For
++ * DSQ_LOCAL_ON verdicts targeting the local DSQ of a remote CPU, defer
++ * the enqueue so that it's executed when @rq can be unlocked.
++ */
++ if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
++ s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK;
++ unsigned long opss;
++
++ if (cpu == cpu_of(rq)) {
++ dsq_id = SCX_DSQ_LOCAL;
++ goto dispatch;
++ }
++
++ opss = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_STATE_MASK;
++
++ switch (opss & SCX_OPSS_STATE_MASK) {
++ case SCX_OPSS_NONE:
++ break;
++ case SCX_OPSS_QUEUEING:
++ /*
++ * As @p was never passed to the BPF side, _release is
++ * not strictly necessary. Still do it for consistency.
++ */
++ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
++ break;
++ default:
++ WARN_ONCE(true, "sched_ext: %s[%d] has invalid ops state 0x%lx in direct_dispatch()",
++ p->comm, p->pid, opss);
++ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
++ break;
++ }
++
++ WARN_ON_ONCE(p->scx.dsq || !list_empty(&p->scx.dsq_list.node));
++ list_add_tail(&p->scx.dsq_list.node,
++ &rq->scx.ddsp_deferred_locals);
++ schedule_deferred(rq);
++ return;
++ }
++
++dispatch:
++ dsq = find_dsq_for_dispatch(rq, dsq_id, p);
++ dispatch_enqueue(dsq, p, p->scx.ddsp_enq_flags | SCX_ENQ_CLEAR_OPSS);
++}
++
++static bool scx_rq_online(struct rq *rq)
++{
++ return likely(rq->scx.flags & SCX_RQ_ONLINE);
++}
++
++static void do_enqueue_task(struct rq *rq, struct task_struct *p, u64 enq_flags,
++ int sticky_cpu)
++{
++ struct task_struct **ddsp_taskp;
++ unsigned long qseq;
++
++ WARN_ON_ONCE(!(p->scx.flags & SCX_TASK_QUEUED));
++
++ /* rq migration */
++ if (sticky_cpu == cpu_of(rq))
++ goto local_norefill;
++
++ /*
++ * If !scx_rq_online(), we already told the BPF scheduler that the CPU
++ * is offline and are just running the hotplug path. Don't bother the
++ * BPF scheduler.
++ */
++ if (!scx_rq_online(rq))
++ goto local;
++
++ if (scx_ops_bypassing()) {
++ if (enq_flags & SCX_ENQ_LAST)
++ goto local;
++ else
++ goto global;
++ }
++
++ if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)
++ goto direct;
++
++ /* see %SCX_OPS_ENQ_EXITING */
++ if (!static_branch_unlikely(&scx_ops_enq_exiting) &&
++ unlikely(p->flags & PF_EXITING))
++ goto local;
++
++ /* see %SCX_OPS_ENQ_LAST */
++ if (!static_branch_unlikely(&scx_ops_enq_last) &&
++ (enq_flags & SCX_ENQ_LAST))
++ goto local;
++
++ if (!SCX_HAS_OP(enqueue))
++ goto global;
++
++ /* DSQ bypass didn't trigger, enqueue on the BPF scheduler */
++ qseq = rq->scx.ops_qseq++ << SCX_OPSS_QSEQ_SHIFT;
++
++ WARN_ON_ONCE(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
++ atomic_long_set(&p->scx.ops_state, SCX_OPSS_QUEUEING | qseq);
++
++ ddsp_taskp = this_cpu_ptr(&direct_dispatch_task);
++ WARN_ON_ONCE(*ddsp_taskp);
++ *ddsp_taskp = p;
++
++ SCX_CALL_OP_TASK(SCX_KF_ENQUEUE, enqueue, p, enq_flags);
++
++ *ddsp_taskp = NULL;
++ if (p->scx.ddsp_dsq_id != SCX_DSQ_INVALID)
++ goto direct;
++
++ /*
++ * If not directly dispatched, QUEUEING isn't clear yet and dispatch or
++ * dequeue may be waiting. The store_release matches their load_acquire.
++ */
++ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_QUEUED | qseq);
++ return;
++
++direct:
++ direct_dispatch(p, enq_flags);
++ return;
++
++local:
++ /*
++ * For task-ordering, slice refill must be treated as implying the end
++ * of the current slice. Otherwise, the longer @p stays on the CPU, the
++ * higher priority it becomes from scx_prio_less()'s POV.
++ */
++ touch_core_sched(rq, p);
++ p->scx.slice = SCX_SLICE_DFL;
++local_norefill:
++ dispatch_enqueue(&rq->scx.local_dsq, p, enq_flags);
++ return;
++
++global:
++ touch_core_sched(rq, p); /* see the comment in local: */
++ p->scx.slice = SCX_SLICE_DFL;
++ dispatch_enqueue(&scx_dsq_global, p, enq_flags);
++}
++
++static bool task_runnable(const struct task_struct *p)
++{
++ return !list_empty(&p->scx.runnable_node);
++}
++
++static void set_task_runnable(struct rq *rq, struct task_struct *p)
++{
++ lockdep_assert_rq_held(rq);
++
++ if (p->scx.flags & SCX_TASK_RESET_RUNNABLE_AT) {
++ p->scx.runnable_at = jiffies;
++ p->scx.flags &= ~SCX_TASK_RESET_RUNNABLE_AT;
++ }
++
++ /*
++ * list_add_tail() must be used. scx_ops_bypass() depends on tasks being
++ * appened to the runnable_list.
++ */
++ list_add_tail(&p->scx.runnable_node, &rq->scx.runnable_list);
++}
++
++static void clr_task_runnable(struct task_struct *p, bool reset_runnable_at)
++{
++ list_del_init(&p->scx.runnable_node);
++ if (reset_runnable_at)
++ p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
++}
++
++static void enqueue_task_scx(struct rq *rq, struct task_struct *p, int enq_flags)
++{
++ int sticky_cpu = p->scx.sticky_cpu;
++
++ if (enq_flags & ENQUEUE_WAKEUP)
++ rq->scx.flags |= SCX_RQ_IN_WAKEUP;
++
++ enq_flags |= rq->scx.extra_enq_flags;
++
++ if (sticky_cpu >= 0)
++ p->scx.sticky_cpu = -1;
++
++ /*
++ * Restoring a running task will be immediately followed by
++ * set_next_task_scx() which expects the task to not be on the BPF
++ * scheduler as tasks can only start running through local DSQs. Force
++ * direct-dispatch into the local DSQ by setting the sticky_cpu.
++ */
++ if (unlikely(enq_flags & ENQUEUE_RESTORE) && task_current(rq, p))
++ sticky_cpu = cpu_of(rq);
++
++ if (p->scx.flags & SCX_TASK_QUEUED) {
++ WARN_ON_ONCE(!task_runnable(p));
++ goto out;
++ }
++
++ set_task_runnable(rq, p);
++ p->scx.flags |= SCX_TASK_QUEUED;
++ rq->scx.nr_running++;
++ add_nr_running(rq, 1);
++
++ if (SCX_HAS_OP(runnable))
++ SCX_CALL_OP_TASK(SCX_KF_REST, runnable, p, enq_flags);
++
++ if (enq_flags & SCX_ENQ_WAKEUP)
++ touch_core_sched(rq, p);
++
++ do_enqueue_task(rq, p, enq_flags, sticky_cpu);
++out:
++ rq->scx.flags &= ~SCX_RQ_IN_WAKEUP;
++}
++
++static void ops_dequeue(struct task_struct *p, u64 deq_flags)
++{
++ unsigned long opss;
++
++ /* dequeue is always temporary, don't reset runnable_at */
++ clr_task_runnable(p, false);
++
++ /* acquire ensures that we see the preceding updates on QUEUED */
++ opss = atomic_long_read_acquire(&p->scx.ops_state);
++
++ switch (opss & SCX_OPSS_STATE_MASK) {
++ case SCX_OPSS_NONE:
++ break;
++ case SCX_OPSS_QUEUEING:
++ /*
++ * QUEUEING is started and finished while holding @p's rq lock.
++ * As we're holding the rq lock now, we shouldn't see QUEUEING.
++ */
++ BUG();
++ case SCX_OPSS_QUEUED:
++ if (SCX_HAS_OP(dequeue))
++ SCX_CALL_OP_TASK(SCX_KF_REST, dequeue, p, deq_flags);
++
++ if (atomic_long_try_cmpxchg(&p->scx.ops_state, &opss,
++ SCX_OPSS_NONE))
++ break;
++ fallthrough;
++ case SCX_OPSS_DISPATCHING:
++ /*
++ * If @p is being dispatched from the BPF scheduler to a DSQ,
++ * wait for the transfer to complete so that @p doesn't get
++ * added to its DSQ after dequeueing is complete.
++ *
++ * As we're waiting on DISPATCHING with the rq locked, the
++ * dispatching side shouldn't try to lock the rq while
++ * DISPATCHING is set. See dispatch_to_local_dsq().
++ *
++ * DISPATCHING shouldn't have qseq set and control can reach
++ * here with NONE @opss from the above QUEUED case block.
++ * Explicitly wait on %SCX_OPSS_DISPATCHING instead of @opss.
++ */
++ wait_ops_state(p, SCX_OPSS_DISPATCHING);
++ BUG_ON(atomic_long_read(&p->scx.ops_state) != SCX_OPSS_NONE);
++ break;
++ }
++}
++
++static void dequeue_task_scx(struct rq *rq, struct task_struct *p, int deq_flags)
++{
++ if (!(p->scx.flags & SCX_TASK_QUEUED)) {
++ WARN_ON_ONCE(task_runnable(p));
++ return;
++ }
++
++ ops_dequeue(p, deq_flags);
++
++ /*
++ * A currently running task which is going off @rq first gets dequeued
++ * and then stops running. As we want running <-> stopping transitions
++ * to be contained within runnable <-> quiescent transitions, trigger
++ * ->stopping() early here instead of in put_prev_task_scx().
++ *
++ * @p may go through multiple stopping <-> running transitions between
++ * here and put_prev_task_scx() if task attribute changes occur while
++ * balance_scx() leaves @rq unlocked. However, they don't contain any
++ * information meaningful to the BPF scheduler and can be suppressed by
++ * skipping the callbacks if the task is !QUEUED.
++ */
++ if (SCX_HAS_OP(stopping) && task_current(rq, p)) {
++ update_curr_scx(rq);
++ SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, false);
++ }
++
++ if (SCX_HAS_OP(quiescent))
++ SCX_CALL_OP_TASK(SCX_KF_REST, quiescent, p, deq_flags);
++
++ if (deq_flags & SCX_DEQ_SLEEP)
++ p->scx.flags |= SCX_TASK_DEQD_FOR_SLEEP;
++ else
++ p->scx.flags &= ~SCX_TASK_DEQD_FOR_SLEEP;
++
++ p->scx.flags &= ~SCX_TASK_QUEUED;
++ rq->scx.nr_running--;
++ sub_nr_running(rq, 1);
++
++ dispatch_dequeue(rq, p);
++}
++
++static void yield_task_scx(struct rq *rq)
++{
++ struct task_struct *p = rq->curr;
++
++ if (SCX_HAS_OP(yield))
++ SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, p, NULL);
++ else
++ p->scx.slice = 0;
++}
++
++static bool yield_to_task_scx(struct rq *rq, struct task_struct *to)
++{
++ struct task_struct *from = rq->curr;
++
++ if (SCX_HAS_OP(yield))
++ return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, yield, from, to);
++ else
++ return false;
++}
++
++#ifdef CONFIG_SMP
++/**
++ * move_task_to_local_dsq - Move a task from a different rq to a local DSQ
++ * @rq: rq to move the task into, currently locked
++ * @p: task to move
++ * @enq_flags: %SCX_ENQ_*
++ *
++ * Move @p which is currently on a different rq to @rq's local DSQ. The caller
++ * must:
++ *
++ * 1. Start with exclusive access to @p either through its DSQ lock or
++ * %SCX_OPSS_DISPATCHING flag.
++ *
++ * 2. Set @p->scx.holding_cpu to raw_smp_processor_id().
++ *
++ * 3. Remember task_rq(@p). Release the exclusive access so that we don't
++ * deadlock with dequeue.
++ *
++ * 4. Lock @rq and the task_rq from #3.
++ *
++ * 5. Call this function.
++ *
++ * Returns %true if @p was successfully moved. %false after racing dequeue and
++ * losing.
++ */
++static bool move_task_to_local_dsq(struct rq *rq, struct task_struct *p,
++ u64 enq_flags)
++{
++ struct rq *task_rq;
++
++ lockdep_assert_rq_held(rq);
++
++ /*
++ * If dequeue got to @p while we were trying to lock both rq's, it'd
++ * have cleared @p->scx.holding_cpu to -1. While other cpus may have
++ * updated it to different values afterwards, as this operation can't be
++ * preempted or recurse, @p->scx.holding_cpu can never become
++ * raw_smp_processor_id() again before we're done. Thus, we can tell
++ * whether we lost to dequeue by testing whether @p->scx.holding_cpu is
++ * still raw_smp_processor_id().
++ *
++ * See dispatch_dequeue() for the counterpart.
++ */
++ if (unlikely(p->scx.holding_cpu != raw_smp_processor_id()))
++ return false;
++
++ /* @p->rq couldn't have changed if we're still the holding cpu */
++ task_rq = task_rq(p);
++ lockdep_assert_rq_held(task_rq);
++
++ WARN_ON_ONCE(!cpumask_test_cpu(cpu_of(rq), p->cpus_ptr));
++ deactivate_task(task_rq, p, 0);
++ set_task_cpu(p, cpu_of(rq));
++ p->scx.sticky_cpu = cpu_of(rq);
++
++ /*
++ * We want to pass scx-specific enq_flags but activate_task() will
++ * truncate the upper 32 bit. As we own @rq, we can pass them through
++ * @rq->scx.extra_enq_flags instead.
++ */
++ WARN_ON_ONCE(rq->scx.extra_enq_flags);
++ rq->scx.extra_enq_flags = enq_flags;
++ activate_task(rq, p, 0);
++ rq->scx.extra_enq_flags = 0;
++
++ return true;
++}
++
++/**
++ * dispatch_to_local_dsq_lock - Ensure source and destination rq's are locked
++ * @rq: current rq which is locked
++ * @src_rq: rq to move task from
++ * @dst_rq: rq to move task to
++ *
++ * We're holding @rq lock and trying to dispatch a task from @src_rq to
++ * @dst_rq's local DSQ and thus need to lock both @src_rq and @dst_rq. Whether
++ * @rq stays locked isn't important as long as the state is restored after
++ * dispatch_to_local_dsq_unlock().
++ */
++static void dispatch_to_local_dsq_lock(struct rq *rq, struct rq *src_rq,
++ struct rq *dst_rq)
++{
++ if (src_rq == dst_rq) {
++ raw_spin_rq_unlock(rq);
++ raw_spin_rq_lock(dst_rq);
++ } else if (rq == src_rq) {
++ double_lock_balance(rq, dst_rq);
++ } else if (rq == dst_rq) {
++ double_lock_balance(rq, src_rq);
++ } else {
++ raw_spin_rq_unlock(rq);
++ double_rq_lock(src_rq, dst_rq);
++ }
++}
++
++/**
++ * dispatch_to_local_dsq_unlock - Undo dispatch_to_local_dsq_lock()
++ * @rq: current rq which is locked
++ * @src_rq: rq to move task from
++ * @dst_rq: rq to move task to
++ *
++ * Unlock @src_rq and @dst_rq and ensure that @rq is locked on return.
++ */
++static void dispatch_to_local_dsq_unlock(struct rq *rq, struct rq *src_rq,
++ struct rq *dst_rq)
++{
++ if (src_rq == dst_rq) {
++ raw_spin_rq_unlock(dst_rq);
++ raw_spin_rq_lock(rq);
++ } else if (rq == src_rq) {
++ double_unlock_balance(rq, dst_rq);
++ } else if (rq == dst_rq) {
++ double_unlock_balance(rq, src_rq);
++ } else {
++ double_rq_unlock(src_rq, dst_rq);
++ raw_spin_rq_lock(rq);
++ }
++}
++#endif /* CONFIG_SMP */
++
++static void consume_local_task(struct rq *rq, struct scx_dispatch_q *dsq,
++ struct task_struct *p)
++{
++ lockdep_assert_held(&dsq->lock); /* released on return */
++
++ /* @dsq is locked and @p is on this rq */
++ WARN_ON_ONCE(p->scx.holding_cpu >= 0);
++ task_unlink_from_dsq(p, dsq);
++ list_add_tail(&p->scx.dsq_list.node, &rq->scx.local_dsq.list);
++ dsq_mod_nr(dsq, -1);
++ dsq_mod_nr(&rq->scx.local_dsq, 1);
++ p->scx.dsq = &rq->scx.local_dsq;
++ raw_spin_unlock(&dsq->lock);
++}
++
++#ifdef CONFIG_SMP
++/*
++ * Similar to kernel/sched/core.c::is_cpu_allowed() but we're testing whether @p
++ * can be pulled to @rq.
++ */
++static bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *rq)
++{
++ int cpu = cpu_of(rq);
++
++ if (!cpumask_test_cpu(cpu, p->cpus_ptr))
++ return false;
++ if (unlikely(is_migration_disabled(p)))
++ return false;
++ if (!(p->flags & PF_KTHREAD) && unlikely(!task_cpu_possible(cpu, p)))
++ return false;
++ if (!scx_rq_online(rq))
++ return false;
++ return true;
++}
++
++static bool consume_remote_task(struct rq *rq, struct scx_dispatch_q *dsq,
++ struct task_struct *p, struct rq *task_rq)
++{
++ bool moved = false;
++
++ lockdep_assert_held(&dsq->lock); /* released on return */
++
++ /*
++ * @dsq is locked and @p is on a remote rq. @p is currently protected by
++ * @dsq->lock. We want to pull @p to @rq but may deadlock if we grab
++ * @task_rq while holding @dsq and @rq locks. As dequeue can't drop the
++ * rq lock or fail, do a little dancing from our side. See
++ * move_task_to_local_dsq().
++ */
++ WARN_ON_ONCE(p->scx.holding_cpu >= 0);
++ task_unlink_from_dsq(p, dsq);
++ dsq_mod_nr(dsq, -1);
++ p->scx.holding_cpu = raw_smp_processor_id();
++ raw_spin_unlock(&dsq->lock);
++
++ double_lock_balance(rq, task_rq);
++
++ moved = move_task_to_local_dsq(rq, p, 0);
++
++ double_unlock_balance(rq, task_rq);
++
++ return moved;
++}
++#else /* CONFIG_SMP */
++static bool task_can_run_on_remote_rq(struct task_struct *p, struct rq *rq) { return false; }
++static bool consume_remote_task(struct rq *rq, struct scx_dispatch_q *dsq,
++ struct task_struct *p, struct rq *task_rq) { return false; }
++#endif /* CONFIG_SMP */
++
++static bool consume_dispatch_q(struct rq *rq, struct scx_dispatch_q *dsq)
++{
++ struct task_struct *p;
++retry:
++ /*
++ * The caller can't expect to successfully consume a task if the task's
++ * addition to @dsq isn't guaranteed to be visible somehow. Test
++ * @dsq->list without locking and skip if it seems empty.
++ */
++ if (list_empty(&dsq->list))
++ return false;
++
++ raw_spin_lock(&dsq->lock);
++
++ nldsq_for_each_task(p, dsq) {
++ struct rq *task_rq = task_rq(p);
++
++ if (rq == task_rq) {
++ consume_local_task(rq, dsq, p);
++ return true;
++ }
++
++ if (task_can_run_on_remote_rq(p, rq)) {
++ if (likely(consume_remote_task(rq, dsq, p, task_rq)))
++ return true;
++ goto retry;
++ }
++ }
++
++ raw_spin_unlock(&dsq->lock);
++ return false;
++}
++
++enum dispatch_to_local_dsq_ret {
++ DTL_DISPATCHED, /* successfully dispatched */
++ DTL_LOST, /* lost race to dequeue */
++ DTL_NOT_LOCAL, /* destination is not a local DSQ */
++ DTL_INVALID, /* invalid local dsq_id */
++};
++
++/**
++ * dispatch_to_local_dsq - Dispatch a task to a local dsq
++ * @rq: current rq which is locked
++ * @dsq_id: destination dsq ID
++ * @p: task to dispatch
++ * @enq_flags: %SCX_ENQ_*
++ *
++ * We're holding @rq lock and want to dispatch @p to the local DSQ identified by
++ * @dsq_id. This function performs all the synchronization dancing needed
++ * because local DSQs are protected with rq locks.
++ *
++ * The caller must have exclusive ownership of @p (e.g. through
++ * %SCX_OPSS_DISPATCHING).
++ */
++static enum dispatch_to_local_dsq_ret
++dispatch_to_local_dsq(struct rq *rq, u64 dsq_id, struct task_struct *p,
++ u64 enq_flags)
++{
++ struct rq *src_rq = task_rq(p);
++ struct rq *dst_rq;
++
++ /*
++ * We're synchronized against dequeue through DISPATCHING. As @p can't
++ * be dequeued, its task_rq and cpus_allowed are stable too.
++ */
++ if (dsq_id == SCX_DSQ_LOCAL) {
++ dst_rq = rq;
++ } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
++ s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK;
++
++ if (!ops_cpu_valid(cpu, "in SCX_DSQ_LOCAL_ON dispatch verdict"))
++ return DTL_INVALID;
++ dst_rq = cpu_rq(cpu);
++ } else {
++ return DTL_NOT_LOCAL;
++ }
++
++ /* if dispatching to @rq that @p is already on, no lock dancing needed */
++ if (rq == src_rq && rq == dst_rq) {
++ dispatch_enqueue(&dst_rq->scx.local_dsq, p,
++ enq_flags | SCX_ENQ_CLEAR_OPSS);
++ return DTL_DISPATCHED;
++ }
++
++#ifdef CONFIG_SMP
++ if (cpumask_test_cpu(cpu_of(dst_rq), p->cpus_ptr)) {
++ struct rq *locked_dst_rq = dst_rq;
++ bool dsp;
++
++ /*
++ * @p is on a possibly remote @src_rq which we need to lock to
++ * move the task. If dequeue is in progress, it'd be locking
++ * @src_rq and waiting on DISPATCHING, so we can't grab @src_rq
++ * lock while holding DISPATCHING.
++ *
++ * As DISPATCHING guarantees that @p is wholly ours, we can
++ * pretend that we're moving from a DSQ and use the same
++ * mechanism - mark the task under transfer with holding_cpu,
++ * release DISPATCHING and then follow the same protocol.
++ */
++ p->scx.holding_cpu = raw_smp_processor_id();
++
++ /* store_release ensures that dequeue sees the above */
++ atomic_long_set_release(&p->scx.ops_state, SCX_OPSS_NONE);
++
++ dispatch_to_local_dsq_lock(rq, src_rq, locked_dst_rq);
++
++ /*
++ * We don't require the BPF scheduler to avoid dispatching to
++ * offline CPUs mostly for convenience but also because CPUs can
++ * go offline between scx_bpf_dispatch() calls and here. If @p
++ * is destined to an offline CPU, queue it on its current CPU
++ * instead, which should always be safe. As this is an allowed
++ * behavior, don't trigger an ops error.
++ */
++ if (!scx_rq_online(dst_rq))
++ dst_rq = src_rq;
++
++ if (src_rq == dst_rq) {
++ /*
++ * As @p is staying on the same rq, there's no need to
++ * go through the full deactivate/activate cycle.
++ * Optimize by abbreviating the operations in
++ * move_task_to_local_dsq().
++ */
++ dsp = p->scx.holding_cpu == raw_smp_processor_id();
++ if (likely(dsp)) {
++ p->scx.holding_cpu = -1;
++ dispatch_enqueue(&dst_rq->scx.local_dsq, p,
++ enq_flags);
++ }
++ } else {
++ dsp = move_task_to_local_dsq(dst_rq, p, enq_flags);
++ }
++
++ /* if the destination CPU is idle, wake it up */
++ if (dsp && sched_class_above(p->sched_class,
++ dst_rq->curr->sched_class))
++ resched_curr(dst_rq);
++
++ dispatch_to_local_dsq_unlock(rq, src_rq, locked_dst_rq);
++
++ return dsp ? DTL_DISPATCHED : DTL_LOST;
++ }
++#endif /* CONFIG_SMP */
++
++ scx_ops_error("SCX_DSQ_LOCAL[_ON] verdict target cpu %d not allowed for %s[%d]",
++ cpu_of(dst_rq), p->comm, p->pid);
++ return DTL_INVALID;
++}
++
++/**
++ * finish_dispatch - Asynchronously finish dispatching a task
++ * @rq: current rq which is locked
++ * @p: task to finish dispatching
++ * @qseq_at_dispatch: qseq when @p started getting dispatched
++ * @dsq_id: destination DSQ ID
++ * @enq_flags: %SCX_ENQ_*
++ *
++ * Dispatching to local DSQs may need to wait for queueing to complete or
++ * require rq lock dancing. As we don't wanna do either while inside
++ * ops.dispatch() to avoid locking order inversion, we split dispatching into
++ * two parts. scx_bpf_dispatch() which is called by ops.dispatch() records the
++ * task and its qseq. Once ops.dispatch() returns, this function is called to
++ * finish up.
++ *
++ * There is no guarantee that @p is still valid for dispatching or even that it
++ * was valid in the first place. Make sure that the task is still owned by the
++ * BPF scheduler and claim the ownership before dispatching.
++ */
++static void finish_dispatch(struct rq *rq, struct task_struct *p,
++ unsigned long qseq_at_dispatch,
++ u64 dsq_id, u64 enq_flags)
++{
++ struct scx_dispatch_q *dsq;
++ unsigned long opss;
++
++ touch_core_sched_dispatch(rq, p);
++retry:
++ /*
++ * No need for _acquire here. @p is accessed only after a successful
++ * try_cmpxchg to DISPATCHING.
++ */
++ opss = atomic_long_read(&p->scx.ops_state);
++
++ switch (opss & SCX_OPSS_STATE_MASK) {
++ case SCX_OPSS_DISPATCHING:
++ case SCX_OPSS_NONE:
++ /* someone else already got to it */
++ return;
++ case SCX_OPSS_QUEUED:
++ /*
++ * If qseq doesn't match, @p has gone through at least one
++ * dispatch/dequeue and re-enqueue cycle between
++ * scx_bpf_dispatch() and here and we have no claim on it.
++ */
++ if ((opss & SCX_OPSS_QSEQ_MASK) != qseq_at_dispatch)
++ return;
++
++ /*
++ * While we know @p is accessible, we don't yet have a claim on
++ * it - the BPF scheduler is allowed to dispatch tasks
++ * spuriously and there can be a racing dequeue attempt. Let's
++ * claim @p by atomically transitioning it from QUEUED to
++ * DISPATCHING.
++ */
++ if (likely(atomic_long_try_cmpxchg(&p->scx.ops_state, &opss,
++ SCX_OPSS_DISPATCHING)))
++ break;
++ goto retry;
++ case SCX_OPSS_QUEUEING:
++ /*
++ * do_enqueue_task() is in the process of transferring the task
++ * to the BPF scheduler while holding @p's rq lock. As we aren't
++ * holding any kernel or BPF resource that the enqueue path may
++ * depend upon, it's safe to wait.
++ */
++ wait_ops_state(p, opss);
++ goto retry;
++ }
++
++ BUG_ON(!(p->scx.flags & SCX_TASK_QUEUED));
++
++ switch (dispatch_to_local_dsq(rq, dsq_id, p, enq_flags)) {
++ case DTL_DISPATCHED:
++ break;
++ case DTL_LOST:
++ break;
++ case DTL_INVALID:
++ dsq_id = SCX_DSQ_GLOBAL;
++ fallthrough;
++ case DTL_NOT_LOCAL:
++ dsq = find_dsq_for_dispatch(cpu_rq(raw_smp_processor_id()),
++ dsq_id, p);
++ dispatch_enqueue(dsq, p, enq_flags | SCX_ENQ_CLEAR_OPSS);
++ break;
++ }
++}
++
++static void flush_dispatch_buf(struct rq *rq)
++{
++ struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
++ u32 u;
++
++ for (u = 0; u < dspc->cursor; u++) {
++ struct scx_dsp_buf_ent *ent = &dspc->buf[u];
++
++ finish_dispatch(rq, ent->task, ent->qseq, ent->dsq_id,
++ ent->enq_flags);
++ }
++
++ dspc->nr_tasks += dspc->cursor;
++ dspc->cursor = 0;
++}
++
++static int balance_one(struct rq *rq, struct task_struct *prev, bool local)
++{
++ struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
++ bool prev_on_scx = prev->sched_class == &ext_sched_class;
++ int nr_loops = SCX_DSP_MAX_LOOPS;
++ bool has_tasks = false;
++
++ lockdep_assert_rq_held(rq);
++ rq->scx.flags |= SCX_RQ_IN_BALANCE;
++
++ if (static_branch_unlikely(&scx_ops_cpu_preempt) &&
++ unlikely(rq->scx.cpu_released)) {
++ /*
++ * If the previous sched_class for the current CPU was not SCX,
++ * notify the BPF scheduler that it again has control of the
++ * core. This callback complements ->cpu_release(), which is
++ * emitted in scx_next_task_picked().
++ */
++ if (SCX_HAS_OP(cpu_acquire))
++ SCX_CALL_OP(0, cpu_acquire, cpu_of(rq), NULL);
++ rq->scx.cpu_released = false;
++ }
++
++ if (prev_on_scx) {
++ WARN_ON_ONCE(local && (prev->scx.flags & SCX_TASK_BAL_KEEP));
++ update_curr_scx(rq);
++
++ /*
++ * If @prev is runnable & has slice left, it has priority and
++ * fetching more just increases latency for the fetched tasks.
++ * Tell put_prev_task_scx() to put @prev on local_dsq. If the
++ * BPF scheduler wants to handle this explicitly, it should
++ * implement ->cpu_released().
++ *
++ * See scx_ops_disable_workfn() for the explanation on the
++ * bypassing test.
++ *
++ * When balancing a remote CPU for core-sched, there won't be a
++ * following put_prev_task_scx() call and we don't own
++ * %SCX_TASK_BAL_KEEP. Instead, pick_task_scx() will test the
++ * same conditions later and pick @rq->curr accordingly.
++ */
++ if ((prev->scx.flags & SCX_TASK_QUEUED) &&
++ prev->scx.slice && !scx_ops_bypassing()) {
++ if (local)
++ prev->scx.flags |= SCX_TASK_BAL_KEEP;
++ goto has_tasks;
++ }
++ }
++
++ /* if there already are tasks to run, nothing to do */
++ if (rq->scx.local_dsq.nr)
++ goto has_tasks;
++
++ if (consume_dispatch_q(rq, &scx_dsq_global))
++ goto has_tasks;
++
++ if (!SCX_HAS_OP(dispatch) || scx_ops_bypassing() || !scx_rq_online(rq))
++ goto out;
++
++ dspc->rq = rq;
++
++ /*
++ * The dispatch loop. Because flush_dispatch_buf() may drop the rq lock,
++ * the local DSQ might still end up empty after a successful
++ * ops.dispatch(). If the local DSQ is empty even after ops.dispatch()
++ * produced some tasks, retry. The BPF scheduler may depend on this
++ * looping behavior to simplify its implementation.
++ */
++ do {
++ dspc->nr_tasks = 0;
++
++ SCX_CALL_OP(SCX_KF_DISPATCH, dispatch, cpu_of(rq),
++ prev_on_scx ? prev : NULL);
++
++ flush_dispatch_buf(rq);
++
++ if (rq->scx.local_dsq.nr)
++ goto has_tasks;
++ if (consume_dispatch_q(rq, &scx_dsq_global))
++ goto has_tasks;
++
++ /*
++ * ops.dispatch() can trap us in this loop by repeatedly
++ * dispatching ineligible tasks. Break out once in a while to
++ * allow the watchdog to run. As IRQ can't be enabled in
++ * balance(), we want to complete this scheduling cycle and then
++ * start a new one. IOW, we want to call resched_curr() on the
++ * next, most likely idle, task, not the current one. Use
++ * scx_bpf_kick_cpu() for deferred kicking.
++ */
++ if (unlikely(!--nr_loops)) {
++ scx_bpf_kick_cpu(cpu_of(rq), 0);
++ break;
++ }
++ } while (dspc->nr_tasks);
++
++ goto out;
++
++has_tasks:
++ has_tasks = true;
++out:
++ rq->scx.flags &= ~SCX_RQ_IN_BALANCE;
++ return has_tasks;
++}
++
++#ifdef CONFIG_SMP
++static int balance_scx(struct rq *rq, struct task_struct *prev,
++ struct rq_flags *rf)
++{
++ int ret;
++
++ rq_unpin_lock(rq, rf);
++
++ ret = balance_one(rq, prev, true);
++
++#ifdef CONFIG_SCHED_SMT
++ /*
++ * When core-sched is enabled, this ops.balance() call will be followed
++ * by put_prev_scx() and pick_task_scx() on this CPU and pick_task_scx()
++ * on the SMT siblings. Balance the siblings too.
++ */
++ if (sched_core_enabled(rq)) {
++ const struct cpumask *smt_mask = cpu_smt_mask(cpu_of(rq));
++ int scpu;
++
++ for_each_cpu_andnot(scpu, smt_mask, cpumask_of(cpu_of(rq))) {
++ struct rq *srq = cpu_rq(scpu);
++ struct task_struct *sprev = srq->curr;
++
++ WARN_ON_ONCE(__rq_lockp(rq) != __rq_lockp(srq));
++ update_rq_clock(srq);
++ balance_one(srq, sprev, false);
++ }
++ }
++#endif
++ rq_repin_lock(rq, rf);
++
++ return ret;
++}
++#endif
++
++static void set_next_task_scx(struct rq *rq, struct task_struct *p, bool first)
++{
++ if (p->scx.flags & SCX_TASK_QUEUED) {
++ /*
++ * Core-sched might decide to execute @p before it is
++ * dispatched. Call ops_dequeue() to notify the BPF scheduler.
++ */
++ ops_dequeue(p, SCX_DEQ_CORE_SCHED_EXEC);
++ dispatch_dequeue(rq, p);
++ }
++
++ p->se.exec_start = rq_clock_task(rq);
++
++ /* see dequeue_task_scx() on why we skip when !QUEUED */
++ if (SCX_HAS_OP(running) && (p->scx.flags & SCX_TASK_QUEUED))
++ SCX_CALL_OP_TASK(SCX_KF_REST, running, p);
++
++ clr_task_runnable(p, true);
++
++ /*
++ * @p is getting newly scheduled or got kicked after someone updated its
++ * slice. Refresh whether tick can be stopped. See scx_can_stop_tick().
++ */
++ if ((p->scx.slice == SCX_SLICE_INF) !=
++ (bool)(rq->scx.flags & SCX_RQ_CAN_STOP_TICK)) {
++ if (p->scx.slice == SCX_SLICE_INF)
++ rq->scx.flags |= SCX_RQ_CAN_STOP_TICK;
++ else
++ rq->scx.flags &= ~SCX_RQ_CAN_STOP_TICK;
++
++ sched_update_tick_dependency(rq);
++
++ /*
++ * For now, let's refresh the load_avgs just when transitioning
++ * in and out of nohz. In the future, we might want to add a
++ * mechanism which calls the following periodically on
++ * tick-stopped CPUs.
++ */
++ update_other_load_avgs(rq);
++ }
++}
++
++static void process_ddsp_deferred_locals(struct rq *rq)
++{
++ struct task_struct *p, *tmp;
++
++ lockdep_assert_rq_held(rq);
++
++ /*
++ * Now that @rq can be unlocked, execute the deferred enqueueing of
++ * tasks directly dispatched to the local DSQs of other CPUs. See
++ * direct_dispatch().
++ */
++ list_for_each_entry_safe(p, tmp, &rq->scx.ddsp_deferred_locals,
++ scx.dsq_list.node) {
++ s32 ret;
++
++ list_del_init(&p->scx.dsq_list.node);
++
++ ret = dispatch_to_local_dsq(rq, p->scx.ddsp_dsq_id, p,
++ p->scx.ddsp_enq_flags);
++ WARN_ON_ONCE(ret == DTL_NOT_LOCAL);
++ }
++}
++
++static void put_prev_task_scx(struct rq *rq, struct task_struct *p)
++{
++#ifndef CONFIG_SMP
++ /*
++ * UP workaround.
++ *
++ * Because SCX may transfer tasks across CPUs during dispatch, dispatch
++ * is performed from its balance operation which isn't called in UP.
++ * Let's work around by calling it from the operations which come right
++ * after.
++ *
++ * 1. If the prev task is on SCX, pick_next_task() calls
++ * .put_prev_task() right after. As .put_prev_task() is also called
++ * from other places, we need to distinguish the calls which can be
++ * done by looking at the previous task's state - if still queued or
++ * dequeued with %SCX_DEQ_SLEEP, the caller must be pick_next_task().
++ * This case is handled here.
++ *
++ * 2. If the prev task is not on SCX, the first following call into SCX
++ * will be .pick_next_task(), which is covered by calling
++ * balance_scx() from pick_next_task_scx().
++ *
++ * Note that we can't merge the first case into the second as
++ * balance_scx() must be called before the previous SCX task goes
++ * through put_prev_task_scx().
++ *
++ * @rq is pinned and can't be unlocked. As UP doesn't transfer tasks
++ * around, balance_one() doesn't need to.
++ */
++ if (p->scx.flags & (SCX_TASK_QUEUED | SCX_TASK_DEQD_FOR_SLEEP))
++ balance_one(rq, p, true);
++#endif
++
++ update_curr_scx(rq);
++
++ /* see dequeue_task_scx() on why we skip when !QUEUED */
++ if (SCX_HAS_OP(stopping) && (p->scx.flags & SCX_TASK_QUEUED))
++ SCX_CALL_OP_TASK(SCX_KF_REST, stopping, p, true);
++
++ /*
++ * If we're being called from put_prev_task_balance(), balance_scx() may
++ * have decided that @p should keep running.
++ */
++ if (p->scx.flags & SCX_TASK_BAL_KEEP) {
++ p->scx.flags &= ~SCX_TASK_BAL_KEEP;
++ set_task_runnable(rq, p);
++ dispatch_enqueue(&rq->scx.local_dsq, p, SCX_ENQ_HEAD);
++ return;
++ }
++
++ if (p->scx.flags & SCX_TASK_QUEUED) {
++ set_task_runnable(rq, p);
++
++ /*
++ * If @p has slice left and balance_scx() didn't tag it for
++ * keeping, @p is getting preempted by a higher priority
++ * scheduler class or core-sched forcing a different task. Leave
++ * it at the head of the local DSQ.
++ */
++ if (p->scx.slice && !scx_ops_bypassing()) {
++ dispatch_enqueue(&rq->scx.local_dsq, p, SCX_ENQ_HEAD);
++ return;
++ }
++
++ /*
++ * If we're in the pick_next_task path, balance_scx() should
++ * have already populated the local DSQ if there are any other
++ * available tasks. If empty, tell ops.enqueue() that @p is the
++ * only one available for this cpu. ops.enqueue() should put it
++ * on the local DSQ so that the subsequent pick_next_task_scx()
++ * can find the task unless it wants to trigger a separate
++ * follow-up scheduling event.
++ */
++ if (list_empty(&rq->scx.local_dsq.list))
++ do_enqueue_task(rq, p, SCX_ENQ_LAST, -1);
++ else
++ do_enqueue_task(rq, p, 0, -1);
++ }
++}
++
++static struct task_struct *first_local_task(struct rq *rq)
++{
++ return list_first_entry_or_null(&rq->scx.local_dsq.list,
++ struct task_struct, scx.dsq_list.node);
++}
++
++static struct task_struct *pick_next_task_scx(struct rq *rq)
++{
++ struct task_struct *p;
++
++#ifndef CONFIG_SMP
++ /* UP workaround - see the comment at the head of put_prev_task_scx() */
++ if (unlikely(rq->curr->sched_class != &ext_sched_class))
++ balance_one(rq, rq->curr, true);
++#endif
++
++ p = first_local_task(rq);
++ if (!p)
++ return NULL;
++
++ set_next_task_scx(rq, p, true);
++
++ if (unlikely(!p->scx.slice)) {
++ if (!scx_ops_bypassing() && !scx_warned_zero_slice) {
++ printk_deferred(KERN_WARNING "sched_ext: %s[%d] has zero slice in pick_next_task_scx()\n",
++ p->comm, p->pid);
++ scx_warned_zero_slice = true;
++ }
++ p->scx.slice = SCX_SLICE_DFL;
++ }
++
++ return p;
++}
++
++#ifdef CONFIG_SCHED_CORE
++/**
++ * scx_prio_less - Task ordering for core-sched
++ * @a: task A
++ * @b: task B
++ *
++ * Core-sched is implemented as an additional scheduling layer on top of the
++ * usual sched_class'es and needs to find out the expected task ordering. For
++ * SCX, core-sched calls this function to interrogate the task ordering.
++ *
++ * Unless overridden by ops.core_sched_before(), @p->scx.core_sched_at is used
++ * to implement the default task ordering. The older the timestamp, the higher
++ * prority the task - the global FIFO ordering matching the default scheduling
++ * behavior.
++ *
++ * When ops.core_sched_before() is enabled, @p->scx.core_sched_at is used to
++ * implement FIFO ordering within each local DSQ. See pick_task_scx().
++ */
++bool scx_prio_less(const struct task_struct *a, const struct task_struct *b,
++ bool in_fi)
++{
++ /*
++ * The const qualifiers are dropped from task_struct pointers when
++ * calling ops.core_sched_before(). Accesses are controlled by the
++ * verifier.
++ */
++ if (SCX_HAS_OP(core_sched_before) && !scx_ops_bypassing())
++ return SCX_CALL_OP_2TASKS_RET(SCX_KF_REST, core_sched_before,
++ (struct task_struct *)a,
++ (struct task_struct *)b);
++ else
++ return time_after64(a->scx.core_sched_at, b->scx.core_sched_at);
++}
++
++/**
++ * pick_task_scx - Pick a candidate task for core-sched
++ * @rq: rq to pick the candidate task from
++ *
++ * Core-sched calls this function on each SMT sibling to determine the next
++ * tasks to run on the SMT siblings. balance_one() has been called on all
++ * siblings and put_prev_task_scx() has been called only for the current CPU.
++ *
++ * As put_prev_task_scx() hasn't been called on remote CPUs, we can't just look
++ * at the first task in the local dsq. @rq->curr has to be considered explicitly
++ * to mimic %SCX_TASK_BAL_KEEP.
++ */
++static struct task_struct *pick_task_scx(struct rq *rq)
++{
++ struct task_struct *curr = rq->curr;
++ struct task_struct *first = first_local_task(rq);
++
++ if (curr->scx.flags & SCX_TASK_QUEUED) {
++ /* is curr the only runnable task? */
++ if (!first)
++ return curr;
++
++ /*
++ * Does curr trump first? We can always go by core_sched_at for
++ * this comparison as it represents global FIFO ordering when
++ * the default core-sched ordering is used and local-DSQ FIFO
++ * ordering otherwise.
++ *
++ * We can have a task with an earlier timestamp on the DSQ. For
++ * example, when a current task is preempted by a sibling
++ * picking a different cookie, the task would be requeued at the
++ * head of the local DSQ with an earlier timestamp than the
++ * core-sched picked next task. Besides, the BPF scheduler may
++ * dispatch any tasks to the local DSQ anytime.
++ */
++ if (curr->scx.slice && time_before64(curr->scx.core_sched_at,
++ first->scx.core_sched_at))
++ return curr;
++ }
++
++ return first; /* this may be %NULL */
++}
++#endif /* CONFIG_SCHED_CORE */
++
++static enum scx_cpu_preempt_reason
++preempt_reason_from_class(const struct sched_class *class)
++{
++#ifdef CONFIG_SMP
++ if (class == &stop_sched_class)
++ return SCX_CPU_PREEMPT_STOP;
++#endif
++ if (class == &dl_sched_class)
++ return SCX_CPU_PREEMPT_DL;
++ if (class == &rt_sched_class)
++ return SCX_CPU_PREEMPT_RT;
++ return SCX_CPU_PREEMPT_UNKNOWN;
++}
++
++static void switch_class_scx(struct rq *rq, struct task_struct *next)
++{
++ const struct sched_class *next_class = next->sched_class;
++
++ if (!scx_enabled())
++ return;
++#ifdef CONFIG_SMP
++ /*
++ * Pairs with the smp_load_acquire() issued by a CPU in
++ * kick_cpus_irq_workfn() who is waiting for this CPU to perform a
++ * resched.
++ */
++ smp_store_release(&rq->scx.pnt_seq, rq->scx.pnt_seq + 1);
++#endif
++ if (!static_branch_unlikely(&scx_ops_cpu_preempt))
++ return;
++
++ /*
++ * The callback is conceptually meant to convey that the CPU is no
++ * longer under the control of SCX. Therefore, don't invoke the callback
++ * if the next class is below SCX (in which case the BPF scheduler has
++ * actively decided not to schedule any tasks on the CPU).
++ */
++ if (sched_class_above(&ext_sched_class, next_class))
++ return;
++
++ /*
++ * At this point we know that SCX was preempted by a higher priority
++ * sched_class, so invoke the ->cpu_release() callback if we have not
++ * done so already. We only send the callback once between SCX being
++ * preempted, and it regaining control of the CPU.
++ *
++ * ->cpu_release() complements ->cpu_acquire(), which is emitted the
++ * next time that balance_scx() is invoked.
++ */
++ if (!rq->scx.cpu_released) {
++ if (SCX_HAS_OP(cpu_release)) {
++ struct scx_cpu_release_args args = {
++ .reason = preempt_reason_from_class(next_class),
++ .task = next,
++ };
++
++ SCX_CALL_OP(SCX_KF_CPU_RELEASE,
++ cpu_release, cpu_of(rq), &args);
++ }
++ rq->scx.cpu_released = true;
++ }
++}
++
++#ifdef CONFIG_SMP
++
++static bool test_and_clear_cpu_idle(int cpu)
++{
++#ifdef CONFIG_SCHED_SMT
++ /*
++ * SMT mask should be cleared whether we can claim @cpu or not. The SMT
++ * cluster is not wholly idle either way. This also prevents
++ * scx_pick_idle_cpu() from getting caught in an infinite loop.
++ */
++ if (sched_smt_active()) {
++ const struct cpumask *smt = cpu_smt_mask(cpu);
++
++ /*
++ * If offline, @cpu is not its own sibling and
++ * scx_pick_idle_cpu() can get caught in an infinite loop as
++ * @cpu is never cleared from idle_masks.smt. Ensure that @cpu
++ * is eventually cleared.
++ */
++ if (cpumask_intersects(smt, idle_masks.smt))
++ cpumask_andnot(idle_masks.smt, idle_masks.smt, smt);
++ else if (cpumask_test_cpu(cpu, idle_masks.smt))
++ __cpumask_clear_cpu(cpu, idle_masks.smt);
++ }
++#endif
++ return cpumask_test_and_clear_cpu(cpu, idle_masks.cpu);
++}
++
++static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, u64 flags)
++{
++ int cpu;
++
++retry:
++ if (sched_smt_active()) {
++ cpu = cpumask_any_and_distribute(idle_masks.smt, cpus_allowed);
++ if (cpu < nr_cpu_ids)
++ goto found;
++
++ if (flags & SCX_PICK_IDLE_CORE)
++ return -EBUSY;
++ }
++
++ cpu = cpumask_any_and_distribute(idle_masks.cpu, cpus_allowed);
++ if (cpu >= nr_cpu_ids)
++ return -EBUSY;
++
++found:
++ if (test_and_clear_cpu_idle(cpu))
++ return cpu;
++ else
++ goto retry;
++}
++
++static s32 scx_select_cpu_dfl(struct task_struct *p, s32 prev_cpu,
++ u64 wake_flags, bool *found)
++{
++ s32 cpu;
++
++ *found = false;
++
++ if (!static_branch_likely(&scx_builtin_idle_enabled)) {
++ scx_ops_error("built-in idle tracking is disabled");
++ return prev_cpu;
++ }
++
++ /*
++ * If WAKE_SYNC, the waker's local DSQ is empty, and the system is
++ * under utilized, wake up @p to the local DSQ of the waker. Checking
++ * only for an empty local DSQ is insufficient as it could give the
++ * wakee an unfair advantage when the system is oversaturated.
++ * Checking only for the presence of idle CPUs is also insufficient as
++ * the local DSQ of the waker could have tasks piled up on it even if
++ * there is an idle core elsewhere on the system.
++ */
++ cpu = smp_processor_id();
++ if ((wake_flags & SCX_WAKE_SYNC) && p->nr_cpus_allowed > 1 &&
++ !cpumask_empty(idle_masks.cpu) && !(current->flags & PF_EXITING) &&
++ cpu_rq(cpu)->scx.local_dsq.nr == 0) {
++ if (cpumask_test_cpu(cpu, p->cpus_ptr))
++ goto cpu_found;
++ }
++
++ if (p->nr_cpus_allowed == 1) {
++ if (test_and_clear_cpu_idle(prev_cpu)) {
++ cpu = prev_cpu;
++ goto cpu_found;
++ } else {
++ return prev_cpu;
++ }
++ }
++
++ /*
++ * If CPU has SMT, any wholly idle CPU is likely a better pick than
++ * partially idle @prev_cpu.
++ */
++ if (sched_smt_active()) {
++ if (cpumask_test_cpu(prev_cpu, idle_masks.smt) &&
++ test_and_clear_cpu_idle(prev_cpu)) {
++ cpu = prev_cpu;
++ goto cpu_found;
++ }
++
++ cpu = scx_pick_idle_cpu(p->cpus_ptr, SCX_PICK_IDLE_CORE);
++ if (cpu >= 0)
++ goto cpu_found;
++ }
++
++ if (test_and_clear_cpu_idle(prev_cpu)) {
++ cpu = prev_cpu;
++ goto cpu_found;
++ }
++
++ cpu = scx_pick_idle_cpu(p->cpus_ptr, 0);
++ if (cpu >= 0)
++ goto cpu_found;
++
++ return prev_cpu;
++
++cpu_found:
++ *found = true;
++ return cpu;
++}
++
++static int select_task_rq_scx(struct task_struct *p, int prev_cpu, int wake_flags)
++{
++ /*
++ * sched_exec() calls with %WF_EXEC when @p is about to exec(2) as it
++ * can be a good migration opportunity with low cache and memory
++ * footprint. Returning a CPU different than @prev_cpu triggers
++ * immediate rq migration. However, for SCX, as the current rq
++ * association doesn't dictate where the task is going to run, this
++ * doesn't fit well. If necessary, we can later add a dedicated method
++ * which can decide to preempt self to force it through the regular
++ * scheduling path.
++ */
++ if (unlikely(wake_flags & WF_EXEC))
++ return prev_cpu;
++
++ if (SCX_HAS_OP(select_cpu)) {
++ s32 cpu;
++ struct task_struct **ddsp_taskp;
++
++ ddsp_taskp = this_cpu_ptr(&direct_dispatch_task);
++ WARN_ON_ONCE(*ddsp_taskp);
++ *ddsp_taskp = p;
++
++ cpu = SCX_CALL_OP_TASK_RET(SCX_KF_ENQUEUE | SCX_KF_SELECT_CPU,
++ select_cpu, p, prev_cpu, wake_flags);
++ *ddsp_taskp = NULL;
++ if (ops_cpu_valid(cpu, "from ops.select_cpu()"))
++ return cpu;
++ else
++ return prev_cpu;
++ } else {
++ bool found;
++ s32 cpu;
++
++ cpu = scx_select_cpu_dfl(p, prev_cpu, wake_flags, &found);
++ if (found) {
++ p->scx.slice = SCX_SLICE_DFL;
++ p->scx.ddsp_dsq_id = SCX_DSQ_LOCAL;
++ }
++ return cpu;
++ }
++}
++
++static void task_woken_scx(struct rq *rq, struct task_struct *p)
++{
++ run_deferred(rq);
++}
++
++static void set_cpus_allowed_scx(struct task_struct *p,
++ struct affinity_context *ac)
++{
++ set_cpus_allowed_common(p, ac);
++
++ /*
++ * The effective cpumask is stored in @p->cpus_ptr which may temporarily
++ * differ from the configured one in @p->cpus_mask. Always tell the bpf
++ * scheduler the effective one.
++ *
++ * Fine-grained memory write control is enforced by BPF making the const
++ * designation pointless. Cast it away when calling the operation.
++ */
++ if (SCX_HAS_OP(set_cpumask))
++ SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p,
++ (struct cpumask *)p->cpus_ptr);
++}
++
++static void reset_idle_masks(void)
++{
++ /*
++ * Consider all online cpus idle. Should converge to the actual state
++ * quickly.
++ */
++ cpumask_copy(idle_masks.cpu, cpu_online_mask);
++ cpumask_copy(idle_masks.smt, cpu_online_mask);
++}
++
++void __scx_update_idle(struct rq *rq, bool idle)
++{
++ int cpu = cpu_of(rq);
++
++ if (SCX_HAS_OP(update_idle)) {
++ SCX_CALL_OP(SCX_KF_REST, update_idle, cpu_of(rq), idle);
++ if (!static_branch_unlikely(&scx_builtin_idle_enabled))
++ return;
++ }
++
++ if (idle)
++ cpumask_set_cpu(cpu, idle_masks.cpu);
++ else
++ cpumask_clear_cpu(cpu, idle_masks.cpu);
++
++#ifdef CONFIG_SCHED_SMT
++ if (sched_smt_active()) {
++ const struct cpumask *smt = cpu_smt_mask(cpu);
++
++ if (idle) {
++ /*
++ * idle_masks.smt handling is racy but that's fine as
++ * it's only for optimization and self-correcting.
++ */
++ for_each_cpu(cpu, smt) {
++ if (!cpumask_test_cpu(cpu, idle_masks.cpu))
++ return;
++ }
++ cpumask_or(idle_masks.smt, idle_masks.smt, smt);
++ } else {
++ cpumask_andnot(idle_masks.smt, idle_masks.smt, smt);
++ }
++ }
++#endif
++}
++
++static void handle_hotplug(struct rq *rq, bool online)
++{
++ int cpu = cpu_of(rq);
++
++ atomic_long_inc(&scx_hotplug_seq);
++
++ if (online && SCX_HAS_OP(cpu_online))
++ SCX_CALL_OP(SCX_KF_SLEEPABLE, cpu_online, cpu);
++ else if (!online && SCX_HAS_OP(cpu_offline))
++ SCX_CALL_OP(SCX_KF_SLEEPABLE, cpu_offline, cpu);
++ else
++ scx_ops_exit(SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG,
++ "cpu %d going %s, exiting scheduler", cpu,
++ online ? "online" : "offline");
++}
++
++void scx_rq_activate(struct rq *rq)
++{
++ handle_hotplug(rq, true);
++}
++
++void scx_rq_deactivate(struct rq *rq)
++{
++ handle_hotplug(rq, false);
++}
++
++static void rq_online_scx(struct rq *rq)
++{
++ rq->scx.flags |= SCX_RQ_ONLINE;
++}
++
++static void rq_offline_scx(struct rq *rq)
++{
++ rq->scx.flags &= ~SCX_RQ_ONLINE;
++}
++
++#else /* CONFIG_SMP */
++
++static bool test_and_clear_cpu_idle(int cpu) { return false; }
++static s32 scx_pick_idle_cpu(const struct cpumask *cpus_allowed, u64 flags) { return -EBUSY; }
++static void reset_idle_masks(void) {}
++
++#endif /* CONFIG_SMP */
++
++static bool check_rq_for_timeouts(struct rq *rq)
++{
++ struct task_struct *p;
++ struct rq_flags rf;
++ bool timed_out = false;
++
++ rq_lock_irqsave(rq, &rf);
++ list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node) {
++ unsigned long last_runnable = p->scx.runnable_at;
++
++ if (unlikely(time_after(jiffies,
++ last_runnable + scx_watchdog_timeout))) {
++ u32 dur_ms = jiffies_to_msecs(jiffies - last_runnable);
++
++ scx_ops_error_kind(SCX_EXIT_ERROR_STALL,
++ "%s[%d] failed to run for %u.%03us",
++ p->comm, p->pid,
++ dur_ms / 1000, dur_ms % 1000);
++ timed_out = true;
++ break;
++ }
++ }
++ rq_unlock_irqrestore(rq, &rf);
++
++ return timed_out;
++}
++
++static void scx_watchdog_workfn(struct work_struct *work)
++{
++ int cpu;
++
++ WRITE_ONCE(scx_watchdog_timestamp, jiffies);
++
++ for_each_online_cpu(cpu) {
++ if (unlikely(check_rq_for_timeouts(cpu_rq(cpu))))
++ break;
++
++ cond_resched();
++ }
++ queue_delayed_work(system_unbound_wq, to_delayed_work(work),
++ scx_watchdog_timeout / 2);
++}
++
++void scx_tick(struct rq *rq)
++{
++ unsigned long last_check;
++
++ if (!scx_enabled())
++ return;
++
++ last_check = READ_ONCE(scx_watchdog_timestamp);
++ if (unlikely(time_after(jiffies,
++ last_check + READ_ONCE(scx_watchdog_timeout)))) {
++ u32 dur_ms = jiffies_to_msecs(jiffies - last_check);
++
++ scx_ops_error_kind(SCX_EXIT_ERROR_STALL,
++ "watchdog failed to check in for %u.%03us",
++ dur_ms / 1000, dur_ms % 1000);
++ }
++
++ update_other_load_avgs(rq);
++}
++
++static void task_tick_scx(struct rq *rq, struct task_struct *curr, int queued)
++{
++ update_curr_scx(rq);
++
++ /*
++ * While disabling, always resched and refresh core-sched timestamp as
++ * we can't trust the slice management or ops.core_sched_before().
++ */
++ if (scx_ops_bypassing()) {
++ curr->scx.slice = 0;
++ touch_core_sched(rq, curr);
++ } else if (SCX_HAS_OP(tick)) {
++ SCX_CALL_OP(SCX_KF_REST, tick, curr);
++ }
++
++ if (!curr->scx.slice)
++ resched_curr(rq);
++}
++
++static enum scx_task_state scx_get_task_state(const struct task_struct *p)
++{
++ return (p->scx.flags & SCX_TASK_STATE_MASK) >> SCX_TASK_STATE_SHIFT;
++}
++
++static void scx_set_task_state(struct task_struct *p, enum scx_task_state state)
++{
++ enum scx_task_state prev_state = scx_get_task_state(p);
++ bool warn = false;
++
++ BUILD_BUG_ON(SCX_TASK_NR_STATES > (1 << SCX_TASK_STATE_BITS));
++
++ switch (state) {
++ case SCX_TASK_NONE:
++ break;
++ case SCX_TASK_INIT:
++ warn = prev_state != SCX_TASK_NONE;
++ break;
++ case SCX_TASK_READY:
++ warn = prev_state == SCX_TASK_NONE;
++ break;
++ case SCX_TASK_ENABLED:
++ warn = prev_state != SCX_TASK_READY;
++ break;
++ default:
++ warn = true;
++ return;
++ }
++
++ WARN_ONCE(warn, "sched_ext: Invalid task state transition %d -> %d for %s[%d]",
++ prev_state, state, p->comm, p->pid);
++
++ p->scx.flags &= ~SCX_TASK_STATE_MASK;
++ p->scx.flags |= state << SCX_TASK_STATE_SHIFT;
++}
++
++static int scx_ops_init_task(struct task_struct *p, struct task_group *tg, bool fork)
++{
++ int ret;
++
++ p->scx.disallow = false;
++
++ if (SCX_HAS_OP(init_task)) {
++ struct scx_init_task_args args = {
++ .fork = fork,
++ };
++
++ ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, init_task, p, &args);
++ if (unlikely(ret)) {
++ ret = ops_sanitize_err("init_task", ret);
++ return ret;
++ }
++ }
++
++ scx_set_task_state(p, SCX_TASK_INIT);
++
++ if (p->scx.disallow) {
++ struct rq *rq;
++ struct rq_flags rf;
++
++ rq = task_rq_lock(p, &rf);
++
++ /*
++ * We're either in fork or load path and @p->policy will be
++ * applied right after. Reverting @p->policy here and rejecting
++ * %SCHED_EXT transitions from scx_check_setscheduler()
++ * guarantees that if ops.init_task() sets @p->disallow, @p can
++ * never be in SCX.
++ */
++ if (p->policy == SCHED_EXT) {
++ p->policy = SCHED_NORMAL;
++ atomic_long_inc(&scx_nr_rejected);
++ }
++
++ task_rq_unlock(rq, p, &rf);
++ }
++
++ p->scx.flags |= SCX_TASK_RESET_RUNNABLE_AT;
++ return 0;
++}
++
++static void scx_ops_enable_task(struct task_struct *p)
++{
++ u32 weight;
++
++ lockdep_assert_rq_held(task_rq(p));
++
++ /*
++ * Set the weight before calling ops.enable() so that the scheduler
++ * doesn't see a stale value if they inspect the task struct.
++ */
++ if (task_has_idle_policy(p))
++ weight = WEIGHT_IDLEPRIO;
++ else
++ weight = sched_prio_to_weight[p->static_prio - MAX_RT_PRIO];
++
++ p->scx.weight = sched_weight_to_cgroup(weight);
++
++ if (SCX_HAS_OP(enable))
++ SCX_CALL_OP_TASK(SCX_KF_REST, enable, p);
++ scx_set_task_state(p, SCX_TASK_ENABLED);
++
++ if (SCX_HAS_OP(set_weight))
++ SCX_CALL_OP(SCX_KF_REST, set_weight, p, p->scx.weight);
++}
++
++static void scx_ops_disable_task(struct task_struct *p)
++{
++ lockdep_assert_rq_held(task_rq(p));
++ WARN_ON_ONCE(scx_get_task_state(p) != SCX_TASK_ENABLED);
++
++ if (SCX_HAS_OP(disable))
++ SCX_CALL_OP(SCX_KF_REST, disable, p);
++ scx_set_task_state(p, SCX_TASK_READY);
++}
++
++static void scx_ops_exit_task(struct task_struct *p)
++{
++ struct scx_exit_task_args args = {
++ .cancelled = false,
++ };
++
++ lockdep_assert_rq_held(task_rq(p));
++
++ switch (scx_get_task_state(p)) {
++ case SCX_TASK_NONE:
++ return;
++ case SCX_TASK_INIT:
++ args.cancelled = true;
++ break;
++ case SCX_TASK_READY:
++ break;
++ case SCX_TASK_ENABLED:
++ scx_ops_disable_task(p);
++ break;
++ default:
++ WARN_ON_ONCE(true);
++ return;
++ }
++
++ if (SCX_HAS_OP(exit_task))
++ SCX_CALL_OP(SCX_KF_REST, exit_task, p, &args);
++ scx_set_task_state(p, SCX_TASK_NONE);
++}
++
++void init_scx_entity(struct sched_ext_entity *scx)
++{
++ /*
++ * init_idle() calls this function again after fork sequence is
++ * complete. Don't touch ->tasks_node as it's already linked.
++ */
++ memset(scx, 0, offsetof(struct sched_ext_entity, tasks_node));
++
++ INIT_LIST_HEAD(&scx->dsq_list.node);
++ RB_CLEAR_NODE(&scx->dsq_priq);
++ scx->sticky_cpu = -1;
++ scx->holding_cpu = -1;
++ INIT_LIST_HEAD(&scx->runnable_node);
++ scx->runnable_at = jiffies;
++ scx->ddsp_dsq_id = SCX_DSQ_INVALID;
++ scx->slice = SCX_SLICE_DFL;
++}
++
++void scx_pre_fork(struct task_struct *p)
++{
++ /*
++ * BPF scheduler enable/disable paths want to be able to iterate and
++ * update all tasks which can become complex when racing forks. As
++ * enable/disable are very cold paths, let's use a percpu_rwsem to
++ * exclude forks.
++ */
++ percpu_down_read(&scx_fork_rwsem);
++}
++
++int scx_fork(struct task_struct *p)
++{
++ percpu_rwsem_assert_held(&scx_fork_rwsem);
++
++ if (scx_enabled())
++ return scx_ops_init_task(p, task_group(p), true);
++ else
++ return 0;
++}
++
++void scx_post_fork(struct task_struct *p)
++{
++ if (scx_enabled()) {
++ scx_set_task_state(p, SCX_TASK_READY);
++
++ /*
++ * Enable the task immediately if it's running on sched_ext.
++ * Otherwise, it'll be enabled in switching_to_scx() if and
++ * when it's ever configured to run with a SCHED_EXT policy.
++ */
++ if (p->sched_class == &ext_sched_class) {
++ struct rq_flags rf;
++ struct rq *rq;
++
++ rq = task_rq_lock(p, &rf);
++ scx_ops_enable_task(p);
++ task_rq_unlock(rq, p, &rf);
++ }
++ }
++
++ spin_lock_irq(&scx_tasks_lock);
++ list_add_tail(&p->scx.tasks_node, &scx_tasks);
++ spin_unlock_irq(&scx_tasks_lock);
++
++ percpu_up_read(&scx_fork_rwsem);
++}
++
++void scx_cancel_fork(struct task_struct *p)
++{
++ if (scx_enabled()) {
++ struct rq *rq;
++ struct rq_flags rf;
++
++ rq = task_rq_lock(p, &rf);
++ WARN_ON_ONCE(scx_get_task_state(p) >= SCX_TASK_READY);
++ scx_ops_exit_task(p);
++ task_rq_unlock(rq, p, &rf);
++ }
++
++ percpu_up_read(&scx_fork_rwsem);
++}
++
++void sched_ext_free(struct task_struct *p)
++{
++ unsigned long flags;
++
++ spin_lock_irqsave(&scx_tasks_lock, flags);
++ list_del_init(&p->scx.tasks_node);
++ spin_unlock_irqrestore(&scx_tasks_lock, flags);
++
++ /*
++ * @p is off scx_tasks and wholly ours. scx_ops_enable()'s READY ->
++ * ENABLED transitions can't race us. Disable ops for @p.
++ */
++ if (scx_get_task_state(p) != SCX_TASK_NONE) {
++ struct rq_flags rf;
++ struct rq *rq;
++
++ rq = task_rq_lock(p, &rf);
++ scx_ops_exit_task(p);
++ task_rq_unlock(rq, p, &rf);
++ }
++}
++
++static void reweight_task_scx(struct rq *rq, struct task_struct *p,
++ const struct load_weight *lw)
++{
++ lockdep_assert_rq_held(task_rq(p));
++
++ p->scx.weight = sched_weight_to_cgroup(scale_load_down(lw->weight));
++ if (SCX_HAS_OP(set_weight))
++ SCX_CALL_OP_TASK(SCX_KF_REST, set_weight, p, p->scx.weight);
++}
++
++static void prio_changed_scx(struct rq *rq, struct task_struct *p, int oldprio)
++{
++}
++
++static void switching_to_scx(struct rq *rq, struct task_struct *p)
++{
++ scx_ops_enable_task(p);
++
++ /*
++ * set_cpus_allowed_scx() is not called while @p is associated with a
++ * different scheduler class. Keep the BPF scheduler up-to-date.
++ */
++ if (SCX_HAS_OP(set_cpumask))
++ SCX_CALL_OP_TASK(SCX_KF_REST, set_cpumask, p,
++ (struct cpumask *)p->cpus_ptr);
++}
++
++static void switched_from_scx(struct rq *rq, struct task_struct *p)
++{
++ scx_ops_disable_task(p);
++}
++
++static void wakeup_preempt_scx(struct rq *rq, struct task_struct *p,int wake_flags) {}
++static void switched_to_scx(struct rq *rq, struct task_struct *p) {}
++
++int scx_check_setscheduler(struct task_struct *p, int policy)
++{
++ lockdep_assert_rq_held(task_rq(p));
++
++ /* if disallow, reject transitioning into SCX */
++ if (scx_enabled() && READ_ONCE(p->scx.disallow) &&
++ p->policy != policy && policy == SCHED_EXT)
++ return -EACCES;
++
++ return 0;
++}
++
++#ifdef CONFIG_NO_HZ_FULL
++bool scx_can_stop_tick(struct rq *rq)
++{
++ struct task_struct *p = rq->curr;
++
++ if (scx_ops_bypassing())
++ return false;
++
++ if (p->sched_class != &ext_sched_class)
++ return true;
++
++ /*
++ * @rq can dispatch from different DSQs, so we can't tell whether it
++ * needs the tick or not by looking at nr_running. Allow stopping ticks
++ * iff the BPF scheduler indicated so. See set_next_task_scx().
++ */
++ return rq->scx.flags & SCX_RQ_CAN_STOP_TICK;
++}
++#endif
++
++/*
++ * Omitted operations:
++ *
++ * - wakeup_preempt: NOOP as it isn't useful in the wakeup path because the task
++ * isn't tied to the CPU at that point. Preemption is implemented by resetting
++ * the victim task's slice to 0 and triggering reschedule on the target CPU.
++ *
++ * - migrate_task_rq: Unnecessary as task to cpu mapping is transient.
++ *
++ * - task_fork/dead: We need fork/dead notifications for all tasks regardless of
++ * their current sched_class. Call them directly from sched core instead.
++ */
++DEFINE_SCHED_CLASS(ext) = {
++ .enqueue_task = enqueue_task_scx,
++ .dequeue_task = dequeue_task_scx,
++ .yield_task = yield_task_scx,
++ .yield_to_task = yield_to_task_scx,
++
++ .wakeup_preempt = wakeup_preempt_scx,
++
++ .pick_next_task = pick_next_task_scx,
++
++ .put_prev_task = put_prev_task_scx,
++ .set_next_task = set_next_task_scx,
++
++ .switch_class = switch_class_scx,
++
++#ifdef CONFIG_SMP
++ .balance = balance_scx,
++ .select_task_rq = select_task_rq_scx,
++ .task_woken = task_woken_scx,
++ .set_cpus_allowed = set_cpus_allowed_scx,
++
++ .rq_online = rq_online_scx,
++ .rq_offline = rq_offline_scx,
++#endif
++
++#ifdef CONFIG_SCHED_CORE
++ .pick_task = pick_task_scx,
++#endif
++
++ .task_tick = task_tick_scx,
++
++ .switching_to = switching_to_scx,
++ .switched_from = switched_from_scx,
++ .switched_to = switched_to_scx,
++ .reweight_task = reweight_task_scx,
++ .prio_changed = prio_changed_scx,
++
++ .update_curr = update_curr_scx,
++
++#ifdef CONFIG_UCLAMP_TASK
++ .uclamp_enabled = 1,
++#endif
++};
++
++static void init_dsq(struct scx_dispatch_q *dsq, u64 dsq_id)
++{
++ memset(dsq, 0, sizeof(*dsq));
++
++ raw_spin_lock_init(&dsq->lock);
++ INIT_LIST_HEAD(&dsq->list);
++ dsq->id = dsq_id;
++}
++
++static struct scx_dispatch_q *create_dsq(u64 dsq_id, int node)
++{
++ struct scx_dispatch_q *dsq;
++ int ret;
++
++ if (dsq_id & SCX_DSQ_FLAG_BUILTIN)
++ return ERR_PTR(-EINVAL);
++
++ dsq = kmalloc_node(sizeof(*dsq), GFP_KERNEL, node);
++ if (!dsq)
++ return ERR_PTR(-ENOMEM);
++
++ init_dsq(dsq, dsq_id);
++
++ ret = rhashtable_insert_fast(&dsq_hash, &dsq->hash_node,
++ dsq_hash_params);
++ if (ret) {
++ kfree(dsq);
++ return ERR_PTR(ret);
++ }
++ return dsq;
++}
++
++static void free_dsq_irq_workfn(struct irq_work *irq_work)
++{
++ struct llist_node *to_free = llist_del_all(&dsqs_to_free);
++ struct scx_dispatch_q *dsq, *tmp_dsq;
++
++ llist_for_each_entry_safe(dsq, tmp_dsq, to_free, free_node)
++ kfree_rcu(dsq, rcu);
++}
++
++static DEFINE_IRQ_WORK(free_dsq_irq_work, free_dsq_irq_workfn);
++
++static void destroy_dsq(u64 dsq_id)
++{
++ struct scx_dispatch_q *dsq;
++ unsigned long flags;
++
++ rcu_read_lock();
++
++ dsq = find_user_dsq(dsq_id);
++ if (!dsq)
++ goto out_unlock_rcu;
++
++ raw_spin_lock_irqsave(&dsq->lock, flags);
++
++ if (dsq->nr) {
++ scx_ops_error("attempting to destroy in-use dsq 0x%016llx (nr=%u)",
++ dsq->id, dsq->nr);
++ goto out_unlock_dsq;
++ }
++
++ if (rhashtable_remove_fast(&dsq_hash, &dsq->hash_node, dsq_hash_params))
++ goto out_unlock_dsq;
++
++ /*
++ * Mark dead by invalidating ->id to prevent dispatch_enqueue() from
++ * queueing more tasks. As this function can be called from anywhere,
++ * freeing is bounced through an irq work to avoid nesting RCU
++ * operations inside scheduler locks.
++ */
++ dsq->id = SCX_DSQ_INVALID;
++ llist_add(&dsq->free_node, &dsqs_to_free);
++ irq_work_queue(&free_dsq_irq_work);
++
++out_unlock_dsq:
++ raw_spin_unlock_irqrestore(&dsq->lock, flags);
++out_unlock_rcu:
++ rcu_read_unlock();
++}
++
++
++/********************************************************************************
++ * Sysfs interface and ops enable/disable.
++ */
++
++#define SCX_ATTR(_name) \
++ static struct kobj_attribute scx_attr_##_name = { \
++ .attr = { .name = __stringify(_name), .mode = 0444 }, \
++ .show = scx_attr_##_name##_show, \
++ }
++
++static ssize_t scx_attr_state_show(struct kobject *kobj,
++ struct kobj_attribute *ka, char *buf)
++{
++ return sysfs_emit(buf, "%s\n",
++ scx_ops_enable_state_str[scx_ops_enable_state()]);
++}
++SCX_ATTR(state);
++
++static ssize_t scx_attr_switch_all_show(struct kobject *kobj,
++ struct kobj_attribute *ka, char *buf)
++{
++ return sysfs_emit(buf, "%d\n", READ_ONCE(scx_switching_all));
++}
++SCX_ATTR(switch_all);
++
++static ssize_t scx_attr_nr_rejected_show(struct kobject *kobj,
++ struct kobj_attribute *ka, char *buf)
++{
++ return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_nr_rejected));
++}
++SCX_ATTR(nr_rejected);
++
++static ssize_t scx_attr_hotplug_seq_show(struct kobject *kobj,
++ struct kobj_attribute *ka, char *buf)
++{
++ return sysfs_emit(buf, "%ld\n", atomic_long_read(&scx_hotplug_seq));
++}
++SCX_ATTR(hotplug_seq);
++
++static struct attribute *scx_global_attrs[] = {
++ &scx_attr_state.attr,
++ &scx_attr_switch_all.attr,
++ &scx_attr_nr_rejected.attr,
++ &scx_attr_hotplug_seq.attr,
++ NULL,
++};
++
++static const struct attribute_group scx_global_attr_group = {
++ .attrs = scx_global_attrs,
++};
++
++static void scx_kobj_release(struct kobject *kobj)
++{
++ kfree(kobj);
++}
++
++static ssize_t scx_attr_ops_show(struct kobject *kobj,
++ struct kobj_attribute *ka, char *buf)
++{
++ return sysfs_emit(buf, "%s\n", scx_ops.name);
++}
++SCX_ATTR(ops);
++
++static struct attribute *scx_sched_attrs[] = {
++ &scx_attr_ops.attr,
++ NULL,
++};
++ATTRIBUTE_GROUPS(scx_sched);
++
++static const struct kobj_type scx_ktype = {
++ .release = scx_kobj_release,
++ .sysfs_ops = &kobj_sysfs_ops,
++ .default_groups = scx_sched_groups,
++};
++
++static int scx_uevent(const struct kobject *kobj, struct kobj_uevent_env *env)
++{
++ return add_uevent_var(env, "SCXOPS=%s", scx_ops.name);
++}
++
++static const struct kset_uevent_ops scx_uevent_ops = {
++ .uevent = scx_uevent,
++};
++
++/*
++ * Used by sched_fork() and __setscheduler_prio() to pick the matching
++ * sched_class. dl/rt are already handled.
++ */
++bool task_should_scx(struct task_struct *p)
++{
++ if (!scx_enabled() ||
++ unlikely(scx_ops_enable_state() == SCX_OPS_DISABLING))
++ return false;
++ if (READ_ONCE(scx_switching_all))
++ return true;
++ return p->policy == SCHED_EXT;
++}
++
++/**
++ * scx_ops_bypass - [Un]bypass scx_ops and guarantee forward progress
++ *
++ * Bypassing guarantees that all runnable tasks make forward progress without
++ * trusting the BPF scheduler. We can't grab any mutexes or rwsems as they might
++ * be held by tasks that the BPF scheduler is forgetting to run, which
++ * unfortunately also excludes toggling the static branches.
++ *
++ * Let's work around by overriding a couple ops and modifying behaviors based on
++ * the DISABLING state and then cycling the queued tasks through dequeue/enqueue
++ * to force global FIFO scheduling.
++ *
++ * a. ops.enqueue() is ignored and tasks are queued in simple global FIFO order.
++ *
++ * b. ops.dispatch() is ignored.
++ *
++ * c. balance_scx() never sets %SCX_TASK_BAL_KEEP as the slice value can't be
++ * trusted. Whenever a tick triggers, the running task is rotated to the tail
++ * of the queue with core_sched_at touched.
++ *
++ * d. pick_next_task() suppresses zero slice warning.
++ *
++ * e. scx_bpf_kick_cpu() is disabled to avoid irq_work malfunction during PM
++ * operations.
++ *
++ * f. scx_prio_less() reverts to the default core_sched_at order.
++ */
++static void scx_ops_bypass(bool bypass)
++{
++ int depth, cpu;
++
++ if (bypass) {
++ depth = atomic_inc_return(&scx_ops_bypass_depth);
++ WARN_ON_ONCE(depth <= 0);
++ if (depth != 1)
++ return;
++ } else {
++ depth = atomic_dec_return(&scx_ops_bypass_depth);
++ WARN_ON_ONCE(depth < 0);
++ if (depth != 0)
++ return;
++ }
++
++ /*
++ * We need to guarantee that no tasks are on the BPF scheduler while
++ * bypassing. Either we see enabled or the enable path sees the
++ * increased bypass_depth before moving tasks to SCX.
++ */
++ if (!scx_enabled())
++ return;
++
++ /*
++ * No task property is changing. We just need to make sure all currently
++ * queued tasks are re-queued according to the new scx_ops_bypassing()
++ * state. As an optimization, walk each rq's runnable_list instead of
++ * the scx_tasks list.
++ *
++ * This function can't trust the scheduler and thus can't use
++ * cpus_read_lock(). Walk all possible CPUs instead of online.
++ */
++ for_each_possible_cpu(cpu) {
++ struct rq *rq = cpu_rq(cpu);
++ struct rq_flags rf;
++ struct task_struct *p, *n;
++
++ rq_lock_irqsave(rq, &rf);
++
++ /*
++ * The use of list_for_each_entry_safe_reverse() is required
++ * because each task is going to be removed from and added back
++ * to the runnable_list during iteration. Because they're added
++ * to the tail of the list, safe reverse iteration can still
++ * visit all nodes.
++ */
++ list_for_each_entry_safe_reverse(p, n, &rq->scx.runnable_list,
++ scx.runnable_node) {
++ struct sched_enq_and_set_ctx ctx;
++
++ /* cycling deq/enq is enough, see the function comment */
++ sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx);
++ sched_enq_and_set_task(&ctx);
++ }
++
++ rq_unlock_irqrestore(rq, &rf);
++
++ /* kick to restore ticks */
++ resched_cpu(cpu);
++ }
++}
++
++static void free_exit_info(struct scx_exit_info *ei)
++{
++ kfree(ei->dump);
++ kfree(ei->msg);
++ kfree(ei->bt);
++ kfree(ei);
++}
++
++static struct scx_exit_info *alloc_exit_info(size_t exit_dump_len)
++{
++ struct scx_exit_info *ei;
++
++ ei = kzalloc(sizeof(*ei), GFP_KERNEL);
++ if (!ei)
++ return NULL;
++
++ ei->bt = kcalloc(SCX_EXIT_BT_LEN, sizeof(ei->bt[0]), GFP_KERNEL);
++ ei->msg = kzalloc(SCX_EXIT_MSG_LEN, GFP_KERNEL);
++ ei->dump = kzalloc(exit_dump_len, GFP_KERNEL);
++
++ if (!ei->bt || !ei->msg || !ei->dump) {
++ free_exit_info(ei);
++ return NULL;
++ }
++
++ return ei;
++}
++
++static const char *scx_exit_reason(enum scx_exit_kind kind)
++{
++ switch (kind) {
++ case SCX_EXIT_UNREG:
++ return "Scheduler unregistered from user space";
++ case SCX_EXIT_UNREG_BPF:
++ return "Scheduler unregistered from BPF";
++ case SCX_EXIT_UNREG_KERN:
++ return "Scheduler unregistered from the main kernel";
++ case SCX_EXIT_SYSRQ:
++ return "disabled by sysrq-S";
++ case SCX_EXIT_ERROR:
++ return "runtime error";
++ case SCX_EXIT_ERROR_BPF:
++ return "scx_bpf_error";
++ case SCX_EXIT_ERROR_STALL:
++ return "runnable task stall";
++ default:
++ return "<UNKNOWN>";
++ }
++}
++
++static void scx_ops_disable_workfn(struct kthread_work *work)
++{
++ struct scx_exit_info *ei = scx_exit_info;
++ struct scx_task_iter sti;
++ struct task_struct *p;
++ struct rhashtable_iter rht_iter;
++ struct scx_dispatch_q *dsq;
++ int i, kind;
++
++ kind = atomic_read(&scx_exit_kind);
++ while (true) {
++ /*
++ * NONE indicates that a new scx_ops has been registered since
++ * disable was scheduled - don't kill the new ops. DONE
++ * indicates that the ops has already been disabled.
++ */
++ if (kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE)
++ return;
++ if (atomic_try_cmpxchg(&scx_exit_kind, &kind, SCX_EXIT_DONE))
++ break;
++ }
++ ei->kind = kind;
++ ei->reason = scx_exit_reason(ei->kind);
++
++ /* guarantee forward progress by bypassing scx_ops */
++ scx_ops_bypass(true);
++
++ switch (scx_ops_set_enable_state(SCX_OPS_DISABLING)) {
++ case SCX_OPS_DISABLING:
++ WARN_ONCE(true, "sched_ext: duplicate disabling instance?");
++ break;
++ case SCX_OPS_DISABLED:
++ pr_warn("sched_ext: ops error detected without ops (%s)\n",
++ scx_exit_info->msg);
++ WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) !=
++ SCX_OPS_DISABLING);
++ goto done;
++ default:
++ break;
++ }
++
++ /*
++ * Here, every runnable task is guaranteed to make forward progress and
++ * we can safely use blocking synchronization constructs. Actually
++ * disable ops.
++ */
++ mutex_lock(&scx_ops_enable_mutex);
++
++ static_branch_disable(&__scx_switched_all);
++ WRITE_ONCE(scx_switching_all, false);
++
++ /*
++ * Avoid racing against fork. See scx_ops_enable() for explanation on
++ * the locking order.
++ */
++ percpu_down_write(&scx_fork_rwsem);
++ cpus_read_lock();
++
++ spin_lock_irq(&scx_tasks_lock);
++ scx_task_iter_init(&sti);
++ /*
++ * Invoke scx_ops_exit_task() on all non-idle tasks, including
++ * TASK_DEAD tasks. Because dead tasks may have a nonzero refcount,
++ * we may not have invoked sched_ext_free() on them by the time a
++ * scheduler is disabled. We must therefore exit the task here, or we'd
++ * fail to invoke ops.exit_task(), as the scheduler will have been
++ * unloaded by the time the task is subsequently exited on the
++ * sched_ext_free() path.
++ */
++ while ((p = scx_task_iter_next_locked(&sti, true))) {
++ const struct sched_class *old_class = p->sched_class;
++ struct sched_enq_and_set_ctx ctx;
++
++ if (READ_ONCE(p->__state) != TASK_DEAD) {
++ sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE,
++ &ctx);
++
++ p->scx.slice = min_t(u64, p->scx.slice, SCX_SLICE_DFL);
++ __setscheduler_prio(p, p->prio);
++ check_class_changing(task_rq(p), p, old_class);
++
++ sched_enq_and_set_task(&ctx);
++
++ check_class_changed(task_rq(p), p, old_class, p->prio);
++ }
++ scx_ops_exit_task(p);
++ }
++ scx_task_iter_exit(&sti);
++ spin_unlock_irq(&scx_tasks_lock);
++
++ /* no task is on scx, turn off all the switches and flush in-progress calls */
++ static_branch_disable_cpuslocked(&__scx_ops_enabled);
++ for (i = SCX_OPI_BEGIN; i < SCX_OPI_END; i++)
++ static_branch_disable_cpuslocked(&scx_has_op[i]);
++ static_branch_disable_cpuslocked(&scx_ops_enq_last);
++ static_branch_disable_cpuslocked(&scx_ops_enq_exiting);
++ static_branch_disable_cpuslocked(&scx_ops_cpu_preempt);
++ static_branch_disable_cpuslocked(&scx_builtin_idle_enabled);
++ synchronize_rcu();
++
++ cpus_read_unlock();
++ percpu_up_write(&scx_fork_rwsem);
++
++ if (ei->kind >= SCX_EXIT_ERROR) {
++ printk(KERN_ERR "sched_ext: BPF scheduler \"%s\" errored, disabling\n", scx_ops.name);
++
++ if (ei->msg[0] == '\0')
++ printk(KERN_ERR "sched_ext: %s\n", ei->reason);
++ else
++ printk(KERN_ERR "sched_ext: %s (%s)\n", ei->reason, ei->msg);
++
++ stack_trace_print(ei->bt, ei->bt_len, 2);
++ }
++
++ if (scx_ops.exit)
++ SCX_CALL_OP(SCX_KF_UNLOCKED, exit, ei);
++
++ cancel_delayed_work_sync(&scx_watchdog_work);
++
++ /*
++ * Delete the kobject from the hierarchy eagerly in addition to just
++ * dropping a reference. Otherwise, if the object is deleted
++ * asynchronously, sysfs could observe an object of the same name still
++ * in the hierarchy when another scheduler is loaded.
++ */
++ kobject_del(scx_root_kobj);
++ kobject_put(scx_root_kobj);
++ scx_root_kobj = NULL;
++
++ memset(&scx_ops, 0, sizeof(scx_ops));
++
++ rhashtable_walk_enter(&dsq_hash, &rht_iter);
++ do {
++ rhashtable_walk_start(&rht_iter);
++
++ while ((dsq = rhashtable_walk_next(&rht_iter)) && !IS_ERR(dsq))
++ destroy_dsq(dsq->id);
++
++ rhashtable_walk_stop(&rht_iter);
++ } while (dsq == ERR_PTR(-EAGAIN));
++ rhashtable_walk_exit(&rht_iter);
++
++ free_percpu(scx_dsp_ctx);
++ scx_dsp_ctx = NULL;
++ scx_dsp_max_batch = 0;
++
++ free_exit_info(scx_exit_info);
++ scx_exit_info = NULL;
++
++ mutex_unlock(&scx_ops_enable_mutex);
++
++ WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_DISABLED) !=
++ SCX_OPS_DISABLING);
++done:
++ scx_ops_bypass(false);
++}
++
++static DEFINE_KTHREAD_WORK(scx_ops_disable_work, scx_ops_disable_workfn);
++
++static void schedule_scx_ops_disable_work(void)
++{
++ struct kthread_worker *helper = READ_ONCE(scx_ops_helper);
++
++ /*
++ * We may be called spuriously before the first bpf_sched_ext_reg(). If
++ * scx_ops_helper isn't set up yet, there's nothing to do.
++ */
++ if (helper)
++ kthread_queue_work(helper, &scx_ops_disable_work);
++}
++
++static void scx_ops_disable(enum scx_exit_kind kind)
++{
++ int none = SCX_EXIT_NONE;
++
++ if (WARN_ON_ONCE(kind == SCX_EXIT_NONE || kind == SCX_EXIT_DONE))
++ kind = SCX_EXIT_ERROR;
++
++ atomic_try_cmpxchg(&scx_exit_kind, &none, kind);
++
++ schedule_scx_ops_disable_work();
++}
++
++static void dump_newline(struct seq_buf *s)
++{
++ trace_sched_ext_dump("");
++
++ /* @s may be zero sized and seq_buf triggers WARN if so */
++ if (s->size)
++ seq_buf_putc(s, '\n');
++}
++
++static __printf(2, 3) void dump_line(struct seq_buf *s, const char *fmt, ...)
++{
++ va_list args;
++
++#ifdef CONFIG_TRACEPOINTS
++ if (trace_sched_ext_dump_enabled()) {
++ /* protected by scx_dump_state()::dump_lock */
++ static char line_buf[SCX_EXIT_MSG_LEN];
++
++ va_start(args, fmt);
++ vscnprintf(line_buf, sizeof(line_buf), fmt, args);
++ va_end(args);
++
++ trace_sched_ext_dump(line_buf);
++ }
++#endif
++ /* @s may be zero sized and seq_buf triggers WARN if so */
++ if (s->size) {
++ va_start(args, fmt);
++ seq_buf_vprintf(s, fmt, args);
++ va_end(args);
++
++ seq_buf_putc(s, '\n');
++ }
++}
++
++static void dump_stack_trace(struct seq_buf *s, const char *prefix,
++ const unsigned long *bt, unsigned int len)
++{
++ unsigned int i;
++
++ for (i = 0; i < len; i++)
++ dump_line(s, "%s%pS", prefix, (void *)bt[i]);
++}
++
++static void ops_dump_init(struct seq_buf *s, const char *prefix)
++{
++ struct scx_dump_data *dd = &scx_dump_data;
++
++ lockdep_assert_irqs_disabled();
++
++ dd->cpu = smp_processor_id(); /* allow scx_bpf_dump() */
++ dd->first = true;
++ dd->cursor = 0;
++ dd->s = s;
++ dd->prefix = prefix;
++}
++
++static void ops_dump_flush(void)
++{
++ struct scx_dump_data *dd = &scx_dump_data;
++ char *line = dd->buf.line;
++
++ if (!dd->cursor)
++ return;
++
++ /*
++ * There's something to flush and this is the first line. Insert a blank
++ * line to distinguish ops dump.
++ */
++ if (dd->first) {
++ dump_newline(dd->s);
++ dd->first = false;
++ }
++
++ /*
++ * There may be multiple lines in $line. Scan and emit each line
++ * separately.
++ */
++ while (true) {
++ char *end = line;
++ char c;
++
++ while (*end != '\n' && *end != '\0')
++ end++;
++
++ /*
++ * If $line overflowed, it may not have newline at the end.
++ * Always emit with a newline.
++ */
++ c = *end;
++ *end = '\0';
++ dump_line(dd->s, "%s%s", dd->prefix, line);
++ if (c == '\0')
++ break;
++
++ /* move to the next line */
++ end++;
++ if (*end == '\0')
++ break;
++ line = end;
++ }
++
++ dd->cursor = 0;
++}
++
++static void ops_dump_exit(void)
++{
++ ops_dump_flush();
++ scx_dump_data.cpu = -1;
++}
++
++static void scx_dump_task(struct seq_buf *s, struct scx_dump_ctx *dctx,
++ struct task_struct *p, char marker)
++{
++ static unsigned long bt[SCX_EXIT_BT_LEN];
++ char dsq_id_buf[19] = "(n/a)";
++ unsigned long ops_state = atomic_long_read(&p->scx.ops_state);
++ unsigned int bt_len;
++
++ if (p->scx.dsq)
++ scnprintf(dsq_id_buf, sizeof(dsq_id_buf), "0x%llx",
++ (unsigned long long)p->scx.dsq->id);
++
++ dump_newline(s);
++ dump_line(s, " %c%c %s[%d] %+ldms",
++ marker, task_state_to_char(p), p->comm, p->pid,
++ jiffies_delta_msecs(p->scx.runnable_at, dctx->at_jiffies));
++ dump_line(s, " scx_state/flags=%u/0x%x dsq_flags=0x%x ops_state/qseq=%lu/%lu",
++ scx_get_task_state(p), p->scx.flags & ~SCX_TASK_STATE_MASK,
++ p->scx.dsq_flags, ops_state & SCX_OPSS_STATE_MASK,
++ ops_state >> SCX_OPSS_QSEQ_SHIFT);
++ dump_line(s, " sticky/holding_cpu=%d/%d dsq_id=%s dsq_vtime=%llu",
++ p->scx.sticky_cpu, p->scx.holding_cpu, dsq_id_buf,
++ p->scx.dsq_vtime);
++ dump_line(s, " cpus=%*pb", cpumask_pr_args(p->cpus_ptr));
++
++ if (SCX_HAS_OP(dump_task)) {
++ ops_dump_init(s, " ");
++ SCX_CALL_OP(SCX_KF_REST, dump_task, dctx, p);
++ ops_dump_exit();
++ }
++
++ bt_len = stack_trace_save_tsk(p, bt, SCX_EXIT_BT_LEN, 1);
++ if (bt_len) {
++ dump_newline(s);
++ dump_stack_trace(s, " ", bt, bt_len);
++ }
++}
++
++static void scx_dump_state(struct scx_exit_info *ei, size_t dump_len)
++{
++ static DEFINE_SPINLOCK(dump_lock);
++ static const char trunc_marker[] = "\n\n~~~~ TRUNCATED ~~~~\n";
++ struct scx_dump_ctx dctx = {
++ .kind = ei->kind,
++ .exit_code = ei->exit_code,
++ .reason = ei->reason,
++ .at_ns = ktime_get_ns(),
++ .at_jiffies = jiffies,
++ };
++ struct seq_buf s;
++ unsigned long flags;
++ char *buf;
++ int cpu;
++
++ spin_lock_irqsave(&dump_lock, flags);
++
++ seq_buf_init(&s, ei->dump, dump_len);
++
++ if (ei->kind == SCX_EXIT_NONE) {
++ dump_line(&s, "Debug dump triggered by %s", ei->reason);
++ } else {
++ dump_line(&s, "%s[%d] triggered exit kind %d:",
++ current->comm, current->pid, ei->kind);
++ dump_line(&s, " %s (%s)", ei->reason, ei->msg);
++ dump_newline(&s);
++ dump_line(&s, "Backtrace:");
++ dump_stack_trace(&s, " ", ei->bt, ei->bt_len);
++ }
++
++ if (SCX_HAS_OP(dump)) {
++ ops_dump_init(&s, "");
++ SCX_CALL_OP(SCX_KF_UNLOCKED, dump, &dctx);
++ ops_dump_exit();
++ }
++
++ dump_newline(&s);
++ dump_line(&s, "CPU states");
++ dump_line(&s, "----------");
++
++ for_each_possible_cpu(cpu) {
++ struct rq *rq = cpu_rq(cpu);
++ struct rq_flags rf;
++ struct task_struct *p;
++ struct seq_buf ns;
++ size_t avail, used;
++ bool idle;
++
++ rq_lock(rq, &rf);
++
++ idle = list_empty(&rq->scx.runnable_list) &&
++ rq->curr->sched_class == &idle_sched_class;
++
++ if (idle && !SCX_HAS_OP(dump_cpu))
++ goto next;
++
++ /*
++ * We don't yet know whether ops.dump_cpu() will produce output
++ * and we may want to skip the default CPU dump if it doesn't.
++ * Use a nested seq_buf to generate the standard dump so that we
++ * can decide whether to commit later.
++ */
++ avail = seq_buf_get_buf(&s, &buf);
++ seq_buf_init(&ns, buf, avail);
++
++ dump_newline(&ns);
++ dump_line(&ns, "CPU %-4d: nr_run=%u flags=0x%x cpu_rel=%d ops_qseq=%lu pnt_seq=%lu",
++ cpu, rq->scx.nr_running, rq->scx.flags,
++ rq->scx.cpu_released, rq->scx.ops_qseq,
++ rq->scx.pnt_seq);
++ dump_line(&ns, " curr=%s[%d] class=%ps",
++ rq->curr->comm, rq->curr->pid,
++ rq->curr->sched_class);
++ if (!cpumask_empty(rq->scx.cpus_to_kick))
++ dump_line(&ns, " cpus_to_kick : %*pb",
++ cpumask_pr_args(rq->scx.cpus_to_kick));
++ if (!cpumask_empty(rq->scx.cpus_to_kick_if_idle))
++ dump_line(&ns, " idle_to_kick : %*pb",
++ cpumask_pr_args(rq->scx.cpus_to_kick_if_idle));
++ if (!cpumask_empty(rq->scx.cpus_to_preempt))
++ dump_line(&ns, " cpus_to_preempt: %*pb",
++ cpumask_pr_args(rq->scx.cpus_to_preempt));
++ if (!cpumask_empty(rq->scx.cpus_to_wait))
++ dump_line(&ns, " cpus_to_wait : %*pb",
++ cpumask_pr_args(rq->scx.cpus_to_wait));
++
++ used = seq_buf_used(&ns);
++ if (SCX_HAS_OP(dump_cpu)) {
++ ops_dump_init(&ns, " ");
++ SCX_CALL_OP(SCX_KF_REST, dump_cpu, &dctx, cpu, idle);
++ ops_dump_exit();
++ }
++
++ /*
++ * If idle && nothing generated by ops.dump_cpu(), there's
++ * nothing interesting. Skip.
++ */
++ if (idle && used == seq_buf_used(&ns))
++ goto next;
++
++ /*
++ * $s may already have overflowed when $ns was created. If so,
++ * calling commit on it will trigger BUG.
++ */
++ if (avail) {
++ seq_buf_commit(&s, seq_buf_used(&ns));
++ if (seq_buf_has_overflowed(&ns))
++ seq_buf_set_overflow(&s);
++ }
++
++ if (rq->curr->sched_class == &ext_sched_class)
++ scx_dump_task(&s, &dctx, rq->curr, '*');
++
++ list_for_each_entry(p, &rq->scx.runnable_list, scx.runnable_node)
++ scx_dump_task(&s, &dctx, p, ' ');
++ next:
++ rq_unlock(rq, &rf);
++ }
++
++ if (seq_buf_has_overflowed(&s) && dump_len >= sizeof(trunc_marker))
++ memcpy(ei->dump + dump_len - sizeof(trunc_marker),
++ trunc_marker, sizeof(trunc_marker));
++
++ spin_unlock_irqrestore(&dump_lock, flags);
++}
++
++static void scx_ops_error_irq_workfn(struct irq_work *irq_work)
++{
++ struct scx_exit_info *ei = scx_exit_info;
++
++ if (ei->kind >= SCX_EXIT_ERROR)
++ scx_dump_state(ei, scx_ops.exit_dump_len);
++
++ schedule_scx_ops_disable_work();
++}
++
++static DEFINE_IRQ_WORK(scx_ops_error_irq_work, scx_ops_error_irq_workfn);
++
++static __printf(3, 4) void scx_ops_exit_kind(enum scx_exit_kind kind,
++ s64 exit_code,
++ const char *fmt, ...)
++{
++ struct scx_exit_info *ei = scx_exit_info;
++ int none = SCX_EXIT_NONE;
++ va_list args;
++
++ if (!atomic_try_cmpxchg(&scx_exit_kind, &none, kind))
++ return;
++
++ ei->exit_code = exit_code;
++
++ if (kind >= SCX_EXIT_ERROR)
++ ei->bt_len = stack_trace_save(ei->bt, SCX_EXIT_BT_LEN, 1);
++
++ va_start(args, fmt);
++ vscnprintf(ei->msg, SCX_EXIT_MSG_LEN, fmt, args);
++ va_end(args);
++
++ /*
++ * Set ei->kind and ->reason for scx_dump_state(). They'll be set again
++ * in scx_ops_disable_workfn().
++ */
++ ei->kind = kind;
++ ei->reason = scx_exit_reason(ei->kind);
++
++ irq_work_queue(&scx_ops_error_irq_work);
++}
++
++static struct kthread_worker *scx_create_rt_helper(const char *name)
++{
++ struct kthread_worker *helper;
++
++ helper = kthread_create_worker(0, name);
++ if (helper)
++ sched_set_fifo(helper->task);
++ return helper;
++}
++
++static void check_hotplug_seq(const struct sched_ext_ops *ops)
++{
++ unsigned long long global_hotplug_seq;
++
++ /*
++ * If a hotplug event has occurred between when a scheduler was
++ * initialized, and when we were able to attach, exit and notify user
++ * space about it.
++ */
++ if (ops->hotplug_seq) {
++ global_hotplug_seq = atomic_long_read(&scx_hotplug_seq);
++ if (ops->hotplug_seq != global_hotplug_seq) {
++ scx_ops_exit(SCX_ECODE_ACT_RESTART | SCX_ECODE_RSN_HOTPLUG,
++ "expected hotplug seq %llu did not match actual %llu",
++ ops->hotplug_seq, global_hotplug_seq);
++ }
++ }
++}
++
++static int validate_ops(const struct sched_ext_ops *ops)
++{
++ /*
++ * It doesn't make sense to specify the SCX_OPS_ENQ_LAST flag if the
++ * ops.enqueue() callback isn't implemented.
++ */
++ if ((ops->flags & SCX_OPS_ENQ_LAST) && !ops->enqueue) {
++ scx_ops_error("SCX_OPS_ENQ_LAST requires ops.enqueue() to be implemented");
++ return -EINVAL;
++ }
++
++ return 0;
++}
++
++static int scx_ops_enable(struct sched_ext_ops *ops)
++{
++ struct scx_task_iter sti;
++ struct task_struct *p;
++ unsigned long timeout;
++ int i, cpu, ret;
++
++ if (!cpumask_equal(housekeeping_cpumask(HK_TYPE_DOMAIN),
++ cpu_possible_mask)) {
++ pr_err("sched_ext: Not compatible with \"isolcpus=\" domain isolation");
++ return -EINVAL;
++ }
++
++ mutex_lock(&scx_ops_enable_mutex);
++
++ if (!scx_ops_helper) {
++ WRITE_ONCE(scx_ops_helper,
++ scx_create_rt_helper("sched_ext_ops_helper"));
++ if (!scx_ops_helper) {
++ ret = -ENOMEM;
++ goto err_unlock;
++ }
++ }
++
++ if (scx_ops_enable_state() != SCX_OPS_DISABLED) {
++ ret = -EBUSY;
++ goto err_unlock;
++ }
++
++ scx_root_kobj = kzalloc(sizeof(*scx_root_kobj), GFP_KERNEL);
++ if (!scx_root_kobj) {
++ ret = -ENOMEM;
++ goto err_unlock;
++ }
++
++ scx_root_kobj->kset = scx_kset;
++ ret = kobject_init_and_add(scx_root_kobj, &scx_ktype, NULL, "root");
++ if (ret < 0)
++ goto err;
++
++ scx_exit_info = alloc_exit_info(ops->exit_dump_len);
++ if (!scx_exit_info) {
++ ret = -ENOMEM;
++ goto err_del;
++ }
++
++ /*
++ * Set scx_ops, transition to PREPPING and clear exit info to arm the
++ * disable path. Failure triggers full disabling from here on.
++ */
++ scx_ops = *ops;
++
++ WARN_ON_ONCE(scx_ops_set_enable_state(SCX_OPS_PREPPING) !=
++ SCX_OPS_DISABLED);
++
++ atomic_set(&scx_exit_kind, SCX_EXIT_NONE);
++ scx_warned_zero_slice = false;
++
++ atomic_long_set(&scx_nr_rejected, 0);
++
++ for_each_possible_cpu(cpu)
++ cpu_rq(cpu)->scx.cpuperf_target = SCX_CPUPERF_ONE;
++
++ /*
++ * Keep CPUs stable during enable so that the BPF scheduler can track
++ * online CPUs by watching ->on/offline_cpu() after ->init().
++ */
++ cpus_read_lock();
++
++ if (scx_ops.init) {
++ ret = SCX_CALL_OP_RET(SCX_KF_SLEEPABLE, init);
++ if (ret) {
++ ret = ops_sanitize_err("init", ret);
++ goto err_disable_unlock_cpus;
++ }
++ }
++
++ for (i = SCX_OPI_CPU_HOTPLUG_BEGIN; i < SCX_OPI_CPU_HOTPLUG_END; i++)
++ if (((void (**)(void))ops)[i])
++ static_branch_enable_cpuslocked(&scx_has_op[i]);
++
++ cpus_read_unlock();
++
++ ret = validate_ops(ops);
++ if (ret)
++ goto err_disable;
++
++ WARN_ON_ONCE(scx_dsp_ctx);
++ scx_dsp_max_batch = ops->dispatch_max_batch ?: SCX_DSP_DFL_MAX_BATCH;
++ scx_dsp_ctx = __alloc_percpu(struct_size_t(struct scx_dsp_ctx, buf,
++ scx_dsp_max_batch),
++ __alignof__(struct scx_dsp_ctx));
++ if (!scx_dsp_ctx) {
++ ret = -ENOMEM;
++ goto err_disable;
++ }
++
++ if (ops->timeout_ms)
++ timeout = msecs_to_jiffies(ops->timeout_ms);
++ else
++ timeout = SCX_WATCHDOG_MAX_TIMEOUT;
++
++ WRITE_ONCE(scx_watchdog_timeout, timeout);
++ WRITE_ONCE(scx_watchdog_timestamp, jiffies);
++ queue_delayed_work(system_unbound_wq, &scx_watchdog_work,
++ scx_watchdog_timeout / 2);
++
++ /*
++ * Lock out forks before opening the floodgate so that they don't wander
++ * into the operations prematurely.
++ *
++ * We don't need to keep the CPUs stable but grab cpus_read_lock() to
++ * ease future locking changes for cgroup suport.
++ *
++ * Note that cpu_hotplug_lock must nest inside scx_fork_rwsem due to the
++ * following dependency chain:
++ *
++ * scx_fork_rwsem --> pernet_ops_rwsem --> cpu_hotplug_lock
++ */
++ percpu_down_write(&scx_fork_rwsem);
++ cpus_read_lock();
++
++ check_hotplug_seq(ops);
++
++ for (i = SCX_OPI_NORMAL_BEGIN; i < SCX_OPI_NORMAL_END; i++)
++ if (((void (**)(void))ops)[i])
++ static_branch_enable_cpuslocked(&scx_has_op[i]);
++
++ if (ops->flags & SCX_OPS_ENQ_LAST)
++ static_branch_enable_cpuslocked(&scx_ops_enq_last);
++
++ if (ops->flags & SCX_OPS_ENQ_EXITING)
++ static_branch_enable_cpuslocked(&scx_ops_enq_exiting);
++ if (scx_ops.cpu_acquire || scx_ops.cpu_release)
++ static_branch_enable_cpuslocked(&scx_ops_cpu_preempt);
++
++ if (!ops->update_idle || (ops->flags & SCX_OPS_KEEP_BUILTIN_IDLE)) {
++ reset_idle_masks();
++ static_branch_enable_cpuslocked(&scx_builtin_idle_enabled);
++ } else {
++ static_branch_disable_cpuslocked(&scx_builtin_idle_enabled);
++ }
++
++ static_branch_enable_cpuslocked(&__scx_ops_enabled);
++
++ /*
++ * Enable ops for every task. Fork is excluded by scx_fork_rwsem
++ * preventing new tasks from being added. No need to exclude tasks
++ * leaving as sched_ext_free() can handle both prepped and enabled
++ * tasks. Prep all tasks first and then enable them with preemption
++ * disabled.
++ */
++ spin_lock_irq(&scx_tasks_lock);
++
++ scx_task_iter_init(&sti);
++ while ((p = scx_task_iter_next_locked(&sti, false))) {
++ get_task_struct(p);
++ scx_task_iter_rq_unlock(&sti);
++ spin_unlock_irq(&scx_tasks_lock);
++
++ ret = scx_ops_init_task(p, task_group(p), false);
++ if (ret) {
++ put_task_struct(p);
++ spin_lock_irq(&scx_tasks_lock);
++ scx_task_iter_exit(&sti);
++ spin_unlock_irq(&scx_tasks_lock);
++ pr_err("sched_ext: ops.init_task() failed (%d) for %s[%d] while loading\n",
++ ret, p->comm, p->pid);
++ goto err_disable_unlock_all;
++ }
++
++ put_task_struct(p);
++ spin_lock_irq(&scx_tasks_lock);
++ }
++ scx_task_iter_exit(&sti);
++
++ /*
++ * All tasks are prepped but are still ops-disabled. Ensure that
++ * %current can't be scheduled out and switch everyone.
++ * preempt_disable() is necessary because we can't guarantee that
++ * %current won't be starved if scheduled out while switching.
++ */
++ preempt_disable();
++
++ /*
++ * From here on, the disable path must assume that tasks have ops
++ * enabled and need to be recovered.
++ *
++ * Transition to ENABLING fails iff the BPF scheduler has already
++ * triggered scx_bpf_error(). Returning an error code here would lose
++ * the recorded error information. Exit indicating success so that the
++ * error is notified through ops.exit() with all the details.
++ */
++ if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLING, SCX_OPS_PREPPING)) {
++ preempt_enable();
++ spin_unlock_irq(&scx_tasks_lock);
++ WARN_ON_ONCE(atomic_read(&scx_exit_kind) == SCX_EXIT_NONE);
++ ret = 0;
++ goto err_disable_unlock_all;
++ }
++
++ /*
++ * We're fully committed and can't fail. The PREPPED -> ENABLED
++ * transitions here are synchronized against sched_ext_free() through
++ * scx_tasks_lock.
++ */
++ WRITE_ONCE(scx_switching_all, !(ops->flags & SCX_OPS_SWITCH_PARTIAL));
++
++ scx_task_iter_init(&sti);
++ while ((p = scx_task_iter_next_locked(&sti, false))) {
++ const struct sched_class *old_class = p->sched_class;
++ struct sched_enq_and_set_ctx ctx;
++
++ sched_deq_and_put_task(p, DEQUEUE_SAVE | DEQUEUE_MOVE, &ctx);
++
++ scx_set_task_state(p, SCX_TASK_READY);
++ __setscheduler_prio(p, p->prio);
++ check_class_changing(task_rq(p), p, old_class);
++
++ sched_enq_and_set_task(&ctx);
++
++ check_class_changed(task_rq(p), p, old_class, p->prio);
++ }
++ scx_task_iter_exit(&sti);
++
++ spin_unlock_irq(&scx_tasks_lock);
++ preempt_enable();
++ cpus_read_unlock();
++ percpu_up_write(&scx_fork_rwsem);
++
++ /* see above ENABLING transition for the explanation on exiting with 0 */
++ if (!scx_ops_tryset_enable_state(SCX_OPS_ENABLED, SCX_OPS_ENABLING)) {
++ WARN_ON_ONCE(atomic_read(&scx_exit_kind) == SCX_EXIT_NONE);
++ ret = 0;
++ goto err_disable;
++ }
++
++ if (!(ops->flags & SCX_OPS_SWITCH_PARTIAL))
++ static_branch_enable(&__scx_switched_all);
++
++ kobject_uevent(scx_root_kobj, KOBJ_ADD);
++ mutex_unlock(&scx_ops_enable_mutex);
++
++ return 0;
++
++err_del:
++ kobject_del(scx_root_kobj);
++err:
++ kobject_put(scx_root_kobj);
++ scx_root_kobj = NULL;
++ if (scx_exit_info) {
++ free_exit_info(scx_exit_info);
++ scx_exit_info = NULL;
++ }
++err_unlock:
++ mutex_unlock(&scx_ops_enable_mutex);
++ return ret;
++
++err_disable_unlock_all:
++ percpu_up_write(&scx_fork_rwsem);
++err_disable_unlock_cpus:
++ cpus_read_unlock();
++err_disable:
++ mutex_unlock(&scx_ops_enable_mutex);
++ /* must be fully disabled before returning */
++ scx_ops_disable(SCX_EXIT_ERROR);
++ kthread_flush_work(&scx_ops_disable_work);
++ return ret;
++}
++
++
++/********************************************************************************
++ * bpf_struct_ops plumbing.
++ */
++#include <linux/bpf_verifier.h>
++#include <linux/bpf.h>
++#include <linux/btf.h>
++
++extern struct btf *btf_vmlinux;
++static const struct btf_type *task_struct_type;
++static u32 task_struct_type_id;
++
++static bool set_arg_maybe_null(const char *op, int arg_n, int off, int size,
++ enum bpf_access_type type,
++ const struct bpf_prog *prog,
++ struct bpf_insn_access_aux *info)
++{
++ struct btf *btf = bpf_get_btf_vmlinux();
++ const struct bpf_struct_ops_desc *st_ops_desc;
++ const struct btf_member *member;
++ const struct btf_type *t;
++ u32 btf_id, member_idx;
++ const char *mname;
++
++ /* struct_ops op args are all sequential, 64-bit numbers */
++ if (off != arg_n * sizeof(__u64))
++ return false;
++
++ /* btf_id should be the type id of struct sched_ext_ops */
++ btf_id = prog->aux->attach_btf_id;
++ st_ops_desc = bpf_struct_ops_find(btf, btf_id);
++ if (!st_ops_desc)
++ return false;
++
++ /* BTF type of struct sched_ext_ops */
++ t = st_ops_desc->type;
++
++ member_idx = prog->expected_attach_type;
++ if (member_idx >= btf_type_vlen(t))
++ return false;
++
++ /*
++ * Get the member name of this struct_ops program, which corresponds to
++ * a field in struct sched_ext_ops. For example, the member name of the
++ * dispatch struct_ops program (callback) is "dispatch".
++ */
++ member = &btf_type_member(t)[member_idx];
++ mname = btf_name_by_offset(btf_vmlinux, member->name_off);
++
++ if (!strcmp(mname, op)) {
++ /*
++ * The value is a pointer to a type (struct task_struct) given
++ * by a BTF ID (PTR_TO_BTF_ID). It is trusted (PTR_TRUSTED),
++ * however, can be a NULL (PTR_MAYBE_NULL). The BPF program
++ * should check the pointer to make sure it is not NULL before
++ * using it, or the verifier will reject the program.
++ *
++ * Longer term, this is something that should be addressed by
++ * BTF, and be fully contained within the verifier.
++ */
++ info->reg_type = PTR_MAYBE_NULL | PTR_TO_BTF_ID | PTR_TRUSTED;
++ info->btf = btf_vmlinux;
++ info->btf_id = task_struct_type_id;
++
++ return true;
++ }
++
++ return false;
++}
++
++static bool bpf_scx_is_valid_access(int off, int size,
++ enum bpf_access_type type,
++ const struct bpf_prog *prog,
++ struct bpf_insn_access_aux *info)
++{
++ if (type != BPF_READ)
++ return false;
++ if (set_arg_maybe_null("dispatch", 1, off, size, type, prog, info) ||
++ set_arg_maybe_null("yield", 1, off, size, type, prog, info))
++ return true;
++ if (off < 0 || off >= sizeof(__u64) * MAX_BPF_FUNC_ARGS)
++ return false;
++ if (off % size != 0)
++ return false;
++
++ return btf_ctx_access(off, size, type, prog, info);
++}
++
++static int bpf_scx_btf_struct_access(struct bpf_verifier_log *log,
++ const struct bpf_reg_state *reg, int off,
++ int size)
++{
++ const struct btf_type *t;
++
++ t = btf_type_by_id(reg->btf, reg->btf_id);
++ if (t == task_struct_type) {
++ if (off >= offsetof(struct task_struct, scx.slice) &&
++ off + size <= offsetofend(struct task_struct, scx.slice))
++ return SCALAR_VALUE;
++ if (off >= offsetof(struct task_struct, scx.dsq_vtime) &&
++ off + size <= offsetofend(struct task_struct, scx.dsq_vtime))
++ return SCALAR_VALUE;
++ if (off >= offsetof(struct task_struct, scx.disallow) &&
++ off + size <= offsetofend(struct task_struct, scx.disallow))
++ return SCALAR_VALUE;
++ }
++
++ return -EACCES;
++}
++
++static const struct bpf_func_proto *
++bpf_scx_get_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog)
++{
++ switch (func_id) {
++ case BPF_FUNC_task_storage_get:
++ return &bpf_task_storage_get_proto;
++ case BPF_FUNC_task_storage_delete:
++ return &bpf_task_storage_delete_proto;
++ default:
++ return bpf_base_func_proto(func_id, prog);
++ }
++}
++
++static const struct bpf_verifier_ops bpf_scx_verifier_ops = {
++ .get_func_proto = bpf_scx_get_func_proto,
++ .is_valid_access = bpf_scx_is_valid_access,
++ .btf_struct_access = bpf_scx_btf_struct_access,
++};
++
++static int bpf_scx_init_member(const struct btf_type *t,
++ const struct btf_member *member,
++ void *kdata, const void *udata)
++{
++ const struct sched_ext_ops *uops = udata;
++ struct sched_ext_ops *ops = kdata;
++ u32 moff = __btf_member_bit_offset(t, member) / 8;
++ int ret;
++
++ switch (moff) {
++ case offsetof(struct sched_ext_ops, dispatch_max_batch):
++ if (*(u32 *)(udata + moff) > INT_MAX)
++ return -E2BIG;
++ ops->dispatch_max_batch = *(u32 *)(udata + moff);
++ return 1;
++ case offsetof(struct sched_ext_ops, flags):
++ if (*(u64 *)(udata + moff) & ~SCX_OPS_ALL_FLAGS)
++ return -EINVAL;
++ ops->flags = *(u64 *)(udata + moff);
++ return 1;
++ case offsetof(struct sched_ext_ops, name):
++ ret = bpf_obj_name_cpy(ops->name, uops->name,
++ sizeof(ops->name));
++ if (ret < 0)
++ return ret;
++ if (ret == 0)
++ return -EINVAL;
++ return 1;
++ case offsetof(struct sched_ext_ops, timeout_ms):
++ if (msecs_to_jiffies(*(u32 *)(udata + moff)) >
++ SCX_WATCHDOG_MAX_TIMEOUT)
++ return -E2BIG;
++ ops->timeout_ms = *(u32 *)(udata + moff);
++ return 1;
++ case offsetof(struct sched_ext_ops, exit_dump_len):
++ ops->exit_dump_len =
++ *(u32 *)(udata + moff) ?: SCX_EXIT_DUMP_DFL_LEN;
++ return 1;
++ case offsetof(struct sched_ext_ops, hotplug_seq):
++ ops->hotplug_seq = *(u64 *)(udata + moff);
++ return 1;
++ }
++
++ return 0;
++}
++
++static int bpf_scx_check_member(const struct btf_type *t,
++ const struct btf_member *member,
++ const struct bpf_prog *prog)
++{
++ u32 moff = __btf_member_bit_offset(t, member) / 8;
++
++ switch (moff) {
++ case offsetof(struct sched_ext_ops, init_task):
++ case offsetof(struct sched_ext_ops, cpu_online):
++ case offsetof(struct sched_ext_ops, cpu_offline):
++ case offsetof(struct sched_ext_ops, init):
++ case offsetof(struct sched_ext_ops, exit):
++ break;
++ default:
++ if (prog->sleepable)
++ return -EINVAL;
++ }
++
++ return 0;
++}
++
++static int bpf_scx_reg(void *kdata)
++{
++ return scx_ops_enable(kdata);
++}
++
++static void bpf_scx_unreg(void *kdata)
++{
++ scx_ops_disable(SCX_EXIT_UNREG);
++ kthread_flush_work(&scx_ops_disable_work);
++}
++
++static int bpf_scx_init(struct btf *btf)
++{
++ u32 type_id;
++
++ type_id = btf_find_by_name_kind(btf, "task_struct", BTF_KIND_STRUCT);
++ if (type_id < 0)
++ return -EINVAL;
++ task_struct_type = btf_type_by_id(btf, type_id);
++ task_struct_type_id = type_id;
++
++ return 0;
++}
++
++static int bpf_scx_update(void *kdata, void *old_kdata)
++{
++ /*
++ * sched_ext does not support updating the actively-loaded BPF
++ * scheduler, as registering a BPF scheduler can always fail if the
++ * scheduler returns an error code for e.g. ops.init(), ops.init_task(),
++ * etc. Similarly, we can always race with unregistration happening
++ * elsewhere, such as with sysrq.
++ */
++ return -EOPNOTSUPP;
++}
++
++static int bpf_scx_validate(void *kdata)
++{
++ return 0;
++}
++
++static s32 select_cpu_stub(struct task_struct *p, s32 prev_cpu, u64 wake_flags) { return -EINVAL; }
++static void enqueue_stub(struct task_struct *p, u64 enq_flags) {}
++static void dequeue_stub(struct task_struct *p, u64 enq_flags) {}
++static void dispatch_stub(s32 prev_cpu, struct task_struct *p) {}
++static void runnable_stub(struct task_struct *p, u64 enq_flags) {}
++static void running_stub(struct task_struct *p) {}
++static void stopping_stub(struct task_struct *p, bool runnable) {}
++static void quiescent_stub(struct task_struct *p, u64 deq_flags) {}
++static bool yield_stub(struct task_struct *from, struct task_struct *to) { return false; }
++static bool core_sched_before_stub(struct task_struct *a, struct task_struct *b) { return false; }
++static void set_weight_stub(struct task_struct *p, u32 weight) {}
++static void set_cpumask_stub(struct task_struct *p, const struct cpumask *mask) {}
++static void update_idle_stub(s32 cpu, bool idle) {}
++static void cpu_acquire_stub(s32 cpu, struct scx_cpu_acquire_args *args) {}
++static void cpu_release_stub(s32 cpu, struct scx_cpu_release_args *args) {}
++static s32 init_task_stub(struct task_struct *p, struct scx_init_task_args *args) { return -EINVAL; }
++static void exit_task_stub(struct task_struct *p, struct scx_exit_task_args *args) {}
++static void enable_stub(struct task_struct *p) {}
++static void disable_stub(struct task_struct *p) {}
++static void cpu_online_stub(s32 cpu) {}
++static void cpu_offline_stub(s32 cpu) {}
++static s32 init_stub(void) { return -EINVAL; }
++static void exit_stub(struct scx_exit_info *info) {}
++
++static struct sched_ext_ops __bpf_ops_sched_ext_ops = {
++ .select_cpu = select_cpu_stub,
++ .enqueue = enqueue_stub,
++ .dequeue = dequeue_stub,
++ .dispatch = dispatch_stub,
++ .runnable = runnable_stub,
++ .running = running_stub,
++ .stopping = stopping_stub,
++ .quiescent = quiescent_stub,
++ .yield = yield_stub,
++ .core_sched_before = core_sched_before_stub,
++ .set_weight = set_weight_stub,
++ .set_cpumask = set_cpumask_stub,
++ .update_idle = update_idle_stub,
++ .cpu_acquire = cpu_acquire_stub,
++ .cpu_release = cpu_release_stub,
++ .init_task = init_task_stub,
++ .exit_task = exit_task_stub,
++ .enable = enable_stub,
++ .disable = disable_stub,
++ .cpu_online = cpu_online_stub,
++ .cpu_offline = cpu_offline_stub,
++ .init = init_stub,
++ .exit = exit_stub,
++};
++
++static struct bpf_struct_ops bpf_sched_ext_ops = {
++ .verifier_ops = &bpf_scx_verifier_ops,
++ .reg = bpf_scx_reg,
++ .unreg = bpf_scx_unreg,
++ .check_member = bpf_scx_check_member,
++ .init_member = bpf_scx_init_member,
++ .init = bpf_scx_init,
++ .update = bpf_scx_update,
++ .validate = bpf_scx_validate,
++ .name = "sched_ext_ops",
++ .owner = THIS_MODULE,
++ .cfi_stubs = &__bpf_ops_sched_ext_ops
++};
++
++
++/********************************************************************************
++ * System integration and init.
++ */
++
++static void sysrq_handle_sched_ext_reset(u8 key)
++{
++ if (scx_ops_helper)
++ scx_ops_disable(SCX_EXIT_SYSRQ);
++ else
++ pr_info("sched_ext: BPF scheduler not yet used\n");
++}
++
++static const struct sysrq_key_op sysrq_sched_ext_reset_op = {
++ .handler = sysrq_handle_sched_ext_reset,
++ .help_msg = "reset-sched-ext(S)",
++ .action_msg = "Disable sched_ext and revert all tasks to CFS",
++ .enable_mask = SYSRQ_ENABLE_RTNICE,
++};
++
++static void sysrq_handle_sched_ext_dump(u8 key)
++{
++ struct scx_exit_info ei = { .kind = SCX_EXIT_NONE, .reason = "SysRq-D" };
++
++ if (scx_enabled())
++ scx_dump_state(&ei, 0);
++}
++
++static const struct sysrq_key_op sysrq_sched_ext_dump_op = {
++ .handler = sysrq_handle_sched_ext_dump,
++ .help_msg = "dump-sched-ext(D)",
++ .action_msg = "Trigger sched_ext debug dump",
++ .enable_mask = SYSRQ_ENABLE_RTNICE,
++};
++
++static bool can_skip_idle_kick(struct rq *rq)
++{
++ lockdep_assert_rq_held(rq);
++
++ /*
++ * We can skip idle kicking if @rq is going to go through at least one
++ * full SCX scheduling cycle before going idle. Just checking whether
++ * curr is not idle is insufficient because we could be racing
++ * balance_one() trying to pull the next task from a remote rq, which
++ * may fail, and @rq may become idle afterwards.
++ *
++ * The race window is small and we don't and can't guarantee that @rq is
++ * only kicked while idle anyway. Skip only when sure.
++ */
++ return !is_idle_task(rq->curr) && !(rq->scx.flags & SCX_RQ_IN_BALANCE);
++}
++
++static bool kick_one_cpu(s32 cpu, struct rq *this_rq, unsigned long *pseqs)
++{
++ struct rq *rq = cpu_rq(cpu);
++ struct scx_rq *this_scx = &this_rq->scx;
++ bool should_wait = false;
++ unsigned long flags;
++
++ raw_spin_rq_lock_irqsave(rq, flags);
++
++ /*
++ * During CPU hotplug, a CPU may depend on kicking itself to make
++ * forward progress. Allow kicking self regardless of online state.
++ */
++ if (cpu_online(cpu) || cpu == cpu_of(this_rq)) {
++ if (cpumask_test_cpu(cpu, this_scx->cpus_to_preempt)) {
++ if (rq->curr->sched_class == &ext_sched_class)
++ rq->curr->scx.slice = 0;
++ cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt);
++ }
++
++ if (cpumask_test_cpu(cpu, this_scx->cpus_to_wait)) {
++ pseqs[cpu] = rq->scx.pnt_seq;
++ should_wait = true;
++ }
++
++ resched_curr(rq);
++ } else {
++ cpumask_clear_cpu(cpu, this_scx->cpus_to_preempt);
++ cpumask_clear_cpu(cpu, this_scx->cpus_to_wait);
++ }
++
++ raw_spin_rq_unlock_irqrestore(rq, flags);
++
++ return should_wait;
++}
++
++static void kick_one_cpu_if_idle(s32 cpu, struct rq *this_rq)
++{
++ struct rq *rq = cpu_rq(cpu);
++ unsigned long flags;
++
++ raw_spin_rq_lock_irqsave(rq, flags);
++
++ if (!can_skip_idle_kick(rq) &&
++ (cpu_online(cpu) || cpu == cpu_of(this_rq)))
++ resched_curr(rq);
++
++ raw_spin_rq_unlock_irqrestore(rq, flags);
++}
++
++static void kick_cpus_irq_workfn(struct irq_work *irq_work)
++{
++ struct rq *this_rq = this_rq();
++ struct scx_rq *this_scx = &this_rq->scx;
++ unsigned long *pseqs = this_cpu_ptr(scx_kick_cpus_pnt_seqs);
++ bool should_wait = false;
++ s32 cpu;
++
++ for_each_cpu(cpu, this_scx->cpus_to_kick) {
++ should_wait |= kick_one_cpu(cpu, this_rq, pseqs);
++ cpumask_clear_cpu(cpu, this_scx->cpus_to_kick);
++ cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle);
++ }
++
++ for_each_cpu(cpu, this_scx->cpus_to_kick_if_idle) {
++ kick_one_cpu_if_idle(cpu, this_rq);
++ cpumask_clear_cpu(cpu, this_scx->cpus_to_kick_if_idle);
++ }
++
++ if (!should_wait)
++ return;
++
++ for_each_cpu(cpu, this_scx->cpus_to_wait) {
++ unsigned long *wait_pnt_seq = &cpu_rq(cpu)->scx.pnt_seq;
++
++ if (cpu != cpu_of(this_rq)) {
++ /*
++ * Pairs with smp_store_release() issued by this CPU in
++ * scx_next_task_picked() on the resched path.
++ *
++ * We busy-wait here to guarantee that no other task can
++ * be scheduled on our core before the target CPU has
++ * entered the resched path.
++ */
++ while (smp_load_acquire(wait_pnt_seq) == pseqs[cpu])
++ cpu_relax();
++ }
++
++ cpumask_clear_cpu(cpu, this_scx->cpus_to_wait);
++ }
++}
++
++/**
++ * print_scx_info - print out sched_ext scheduler state
++ * @log_lvl: the log level to use when printing
++ * @p: target task
++ *
++ * If a sched_ext scheduler is enabled, print the name and state of the
++ * scheduler. If @p is on sched_ext, print further information about the task.
++ *
++ * This function can be safely called on any task as long as the task_struct
++ * itself is accessible. While safe, this function isn't synchronized and may
++ * print out mixups or garbages of limited length.
++ */
++void print_scx_info(const char *log_lvl, struct task_struct *p)
++{
++ enum scx_ops_enable_state state = scx_ops_enable_state();
++ const char *all = READ_ONCE(scx_switching_all) ? "+all" : "";
++ char runnable_at_buf[22] = "?";
++ struct sched_class *class;
++ unsigned long runnable_at;
++
++ if (state == SCX_OPS_DISABLED)
++ return;
++
++ /*
++ * Carefully check if the task was running on sched_ext, and then
++ * carefully copy the time it's been runnable, and its state.
++ */
++ if (copy_from_kernel_nofault(&class, &p->sched_class, sizeof(class)) ||
++ class != &ext_sched_class) {
++ printk("%sSched_ext: %s (%s%s)", log_lvl, scx_ops.name,
++ scx_ops_enable_state_str[state], all);
++ return;
++ }
++
++ if (!copy_from_kernel_nofault(&runnable_at, &p->scx.runnable_at,
++ sizeof(runnable_at)))
++ scnprintf(runnable_at_buf, sizeof(runnable_at_buf), "%+ldms",
++ jiffies_delta_msecs(runnable_at, jiffies));
++
++ /* print everything onto one line to conserve console space */
++ printk("%sSched_ext: %s (%s%s), task: runnable_at=%s",
++ log_lvl, scx_ops.name, scx_ops_enable_state_str[state], all,
++ runnable_at_buf);
++}
++
++static int scx_pm_handler(struct notifier_block *nb, unsigned long event, void *ptr)
++{
++ /*
++ * SCX schedulers often have userspace components which are sometimes
++ * involved in critial scheduling paths. PM operations involve freezing
++ * userspace which can lead to scheduling misbehaviors including stalls.
++ * Let's bypass while PM operations are in progress.
++ */
++ switch (event) {
++ case PM_HIBERNATION_PREPARE:
++ case PM_SUSPEND_PREPARE:
++ case PM_RESTORE_PREPARE:
++ scx_ops_bypass(true);
++ break;
++ case PM_POST_HIBERNATION:
++ case PM_POST_SUSPEND:
++ case PM_POST_RESTORE:
++ scx_ops_bypass(false);
++ break;
++ }
++
++ return NOTIFY_OK;
++}
++
++static struct notifier_block scx_pm_notifier = {
++ .notifier_call = scx_pm_handler,
++};
++
++void __init init_sched_ext_class(void)
++{
++ s32 cpu, v;
++
++ /*
++ * The following is to prevent the compiler from optimizing out the enum
++ * definitions so that BPF scheduler implementations can use them
++ * through the generated vmlinux.h.
++ */
++ WRITE_ONCE(v, SCX_ENQ_WAKEUP | SCX_DEQ_SLEEP | SCX_KICK_PREEMPT);
++
++ BUG_ON(rhashtable_init(&dsq_hash, &dsq_hash_params));
++ init_dsq(&scx_dsq_global, SCX_DSQ_GLOBAL);
++#ifdef CONFIG_SMP
++ BUG_ON(!alloc_cpumask_var(&idle_masks.cpu, GFP_KERNEL));
++ BUG_ON(!alloc_cpumask_var(&idle_masks.smt, GFP_KERNEL));
++#endif
++ scx_kick_cpus_pnt_seqs =
++ __alloc_percpu(sizeof(scx_kick_cpus_pnt_seqs[0]) * nr_cpu_ids,
++ __alignof__(scx_kick_cpus_pnt_seqs[0]));
++ BUG_ON(!scx_kick_cpus_pnt_seqs);
++
++ for_each_possible_cpu(cpu) {
++ struct rq *rq = cpu_rq(cpu);
++
++ init_dsq(&rq->scx.local_dsq, SCX_DSQ_LOCAL);
++ INIT_LIST_HEAD(&rq->scx.runnable_list);
++ INIT_LIST_HEAD(&rq->scx.ddsp_deferred_locals);
++
++ BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_kick, GFP_KERNEL));
++ BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_kick_if_idle, GFP_KERNEL));
++ BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_preempt, GFP_KERNEL));
++ BUG_ON(!zalloc_cpumask_var(&rq->scx.cpus_to_wait, GFP_KERNEL));
++ init_irq_work(&rq->scx.deferred_irq_work, deferred_irq_workfn);
++ init_irq_work(&rq->scx.kick_cpus_irq_work, kick_cpus_irq_workfn);
++
++ if (cpu_online(cpu))
++ cpu_rq(cpu)->scx.flags |= SCX_RQ_ONLINE;
++ }
++
++ register_sysrq_key('S', &sysrq_sched_ext_reset_op);
++ register_sysrq_key('D', &sysrq_sched_ext_dump_op);
++ INIT_DELAYED_WORK(&scx_watchdog_work, scx_watchdog_workfn);
++}
++
++
++/********************************************************************************
++ * Helpers that can be called from the BPF scheduler.
++ */
++#include <linux/btf_ids.h>
++
++__bpf_kfunc_start_defs();
++
++/**
++ * scx_bpf_create_dsq - Create a custom DSQ
++ * @dsq_id: DSQ to create
++ * @node: NUMA node to allocate from
++ *
++ * Create a custom DSQ identified by @dsq_id. Can be called from ops.init() and
++ * ops.init_task().
++ */
++__bpf_kfunc s32 scx_bpf_create_dsq(u64 dsq_id, s32 node)
++{
++ if (!scx_kf_allowed(SCX_KF_SLEEPABLE))
++ return -EINVAL;
++
++ if (unlikely(node >= (int)nr_node_ids ||
++ (node < 0 && node != NUMA_NO_NODE)))
++ return -EINVAL;
++ return PTR_ERR_OR_ZERO(create_dsq(dsq_id, node));
++}
++
++__bpf_kfunc_end_defs();
++
++BTF_KFUNCS_START(scx_kfunc_ids_sleepable)
++BTF_ID_FLAGS(func, scx_bpf_create_dsq, KF_SLEEPABLE)
++BTF_KFUNCS_END(scx_kfunc_ids_sleepable)
++
++static const struct btf_kfunc_id_set scx_kfunc_set_sleepable = {
++ .owner = THIS_MODULE,
++ .set = &scx_kfunc_ids_sleepable,
++};
++
++__bpf_kfunc_start_defs();
++
++/**
++ * scx_bpf_select_cpu_dfl - The default implementation of ops.select_cpu()
++ * @p: task_struct to select a CPU for
++ * @prev_cpu: CPU @p was on previously
++ * @wake_flags: %SCX_WAKE_* flags
++ * @is_idle: out parameter indicating whether the returned CPU is idle
++ *
++ * Can only be called from ops.select_cpu() if the built-in CPU selection is
++ * enabled - ops.update_idle() is missing or %SCX_OPS_KEEP_BUILTIN_IDLE is set.
++ * @p, @prev_cpu and @wake_flags match ops.select_cpu().
++ *
++ * Returns the picked CPU with *@is_idle indicating whether the picked CPU is
++ * currently idle and thus a good candidate for direct dispatching.
++ */
++__bpf_kfunc s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu,
++ u64 wake_flags, bool *is_idle)
++{
++ if (!scx_kf_allowed(SCX_KF_SELECT_CPU)) {
++ *is_idle = false;
++ return prev_cpu;
++ }
++#ifdef CONFIG_SMP
++ return scx_select_cpu_dfl(p, prev_cpu, wake_flags, is_idle);
++#else
++ *is_idle = false;
++ return prev_cpu;
++#endif
++}
++
++__bpf_kfunc_end_defs();
++
++BTF_KFUNCS_START(scx_kfunc_ids_select_cpu)
++BTF_ID_FLAGS(func, scx_bpf_select_cpu_dfl, KF_RCU)
++BTF_KFUNCS_END(scx_kfunc_ids_select_cpu)
++
++static const struct btf_kfunc_id_set scx_kfunc_set_select_cpu = {
++ .owner = THIS_MODULE,
++ .set = &scx_kfunc_ids_select_cpu,
++};
++
++static bool scx_dispatch_preamble(struct task_struct *p, u64 enq_flags)
++{
++ if (!scx_kf_allowed(SCX_KF_ENQUEUE | SCX_KF_DISPATCH))
++ return false;
++
++ lockdep_assert_irqs_disabled();
++
++ if (unlikely(!p)) {
++ scx_ops_error("called with NULL task");
++ return false;
++ }
++
++ if (unlikely(enq_flags & __SCX_ENQ_INTERNAL_MASK)) {
++ scx_ops_error("invalid enq_flags 0x%llx", enq_flags);
++ return false;
++ }
++
++ return true;
++}
++
++static void scx_dispatch_commit(struct task_struct *p, u64 dsq_id, u64 enq_flags)
++{
++ struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
++ struct task_struct *ddsp_task;
++
++ ddsp_task = __this_cpu_read(direct_dispatch_task);
++ if (ddsp_task) {
++ mark_direct_dispatch(ddsp_task, p, dsq_id, enq_flags);
++ return;
++ }
++
++ if (unlikely(dspc->cursor >= scx_dsp_max_batch)) {
++ scx_ops_error("dispatch buffer overflow");
++ return;
++ }
++
++ dspc->buf[dspc->cursor++] = (struct scx_dsp_buf_ent){
++ .task = p,
++ .qseq = atomic_long_read(&p->scx.ops_state) & SCX_OPSS_QSEQ_MASK,
++ .dsq_id = dsq_id,
++ .enq_flags = enq_flags,
++ };
++}
++
++__bpf_kfunc_start_defs();
++
++/**
++ * scx_bpf_dispatch - Dispatch a task into the FIFO queue of a DSQ
++ * @p: task_struct to dispatch
++ * @dsq_id: DSQ to dispatch to
++ * @slice: duration @p can run for in nsecs
++ * @enq_flags: SCX_ENQ_*
++ *
++ * Dispatch @p into the FIFO queue of the DSQ identified by @dsq_id. It is safe
++ * to call this function spuriously. Can be called from ops.enqueue(),
++ * ops.select_cpu(), and ops.dispatch().
++ *
++ * When called from ops.select_cpu() or ops.enqueue(), it's for direct dispatch
++ * and @p must match the task being enqueued. Also, %SCX_DSQ_LOCAL_ON can't be
++ * used to target the local DSQ of a CPU other than the enqueueing one. Use
++ * ops.select_cpu() to be on the target CPU in the first place.
++ *
++ * When called from ops.select_cpu(), @enq_flags and @dsp_id are stored, and @p
++ * will be directly dispatched to the corresponding dispatch queue after
++ * ops.select_cpu() returns. If @p is dispatched to SCX_DSQ_LOCAL, it will be
++ * dispatched to the local DSQ of the CPU returned by ops.select_cpu().
++ * @enq_flags are OR'd with the enqueue flags on the enqueue path before the
++ * task is dispatched.
++ *
++ * When called from ops.dispatch(), there are no restrictions on @p or @dsq_id
++ * and this function can be called upto ops.dispatch_max_batch times to dispatch
++ * multiple tasks. scx_bpf_dispatch_nr_slots() returns the number of the
++ * remaining slots. scx_bpf_consume() flushes the batch and resets the counter.
++ *
++ * This function doesn't have any locking restrictions and may be called under
++ * BPF locks (in the future when BPF introduces more flexible locking).
++ *
++ * @p is allowed to run for @slice. The scheduling path is triggered on slice
++ * exhaustion. If zero, the current residual slice is maintained. If
++ * %SCX_SLICE_INF, @p never expires and the BPF scheduler must kick the CPU with
++ * scx_bpf_kick_cpu() to trigger scheduling.
++ */
++__bpf_kfunc void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice,
++ u64 enq_flags)
++{
++ if (!scx_dispatch_preamble(p, enq_flags))
++ return;
++
++ if (slice)
++ p->scx.slice = slice;
++ else
++ p->scx.slice = p->scx.slice ?: 1;
++
++ scx_dispatch_commit(p, dsq_id, enq_flags);
++}
++
++/**
++ * scx_bpf_dispatch_vtime - Dispatch a task into the vtime priority queue of a DSQ
++ * @p: task_struct to dispatch
++ * @dsq_id: DSQ to dispatch to
++ * @slice: duration @p can run for in nsecs
++ * @vtime: @p's ordering inside the vtime-sorted queue of the target DSQ
++ * @enq_flags: SCX_ENQ_*
++ *
++ * Dispatch @p into the vtime priority queue of the DSQ identified by @dsq_id.
++ * Tasks queued into the priority queue are ordered by @vtime and always
++ * consumed after the tasks in the FIFO queue. All other aspects are identical
++ * to scx_bpf_dispatch().
++ *
++ * @vtime ordering is according to time_before64() which considers wrapping. A
++ * numerically larger vtime may indicate an earlier position in the ordering and
++ * vice-versa.
++ */
++__bpf_kfunc void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id,
++ u64 slice, u64 vtime, u64 enq_flags)
++{
++ if (!scx_dispatch_preamble(p, enq_flags))
++ return;
++
++ if (slice)
++ p->scx.slice = slice;
++ else
++ p->scx.slice = p->scx.slice ?: 1;
++
++ p->scx.dsq_vtime = vtime;
++
++ scx_dispatch_commit(p, dsq_id, enq_flags | SCX_ENQ_DSQ_PRIQ);
++}
++
++__bpf_kfunc_end_defs();
++
++BTF_KFUNCS_START(scx_kfunc_ids_enqueue_dispatch)
++BTF_ID_FLAGS(func, scx_bpf_dispatch, KF_RCU)
++BTF_ID_FLAGS(func, scx_bpf_dispatch_vtime, KF_RCU)
++BTF_KFUNCS_END(scx_kfunc_ids_enqueue_dispatch)
++
++static const struct btf_kfunc_id_set scx_kfunc_set_enqueue_dispatch = {
++ .owner = THIS_MODULE,
++ .set = &scx_kfunc_ids_enqueue_dispatch,
++};
++
++__bpf_kfunc_start_defs();
++
++/**
++ * scx_bpf_dispatch_nr_slots - Return the number of remaining dispatch slots
++ *
++ * Can only be called from ops.dispatch().
++ */
++__bpf_kfunc u32 scx_bpf_dispatch_nr_slots(void)
++{
++ if (!scx_kf_allowed(SCX_KF_DISPATCH))
++ return 0;
++
++ return scx_dsp_max_batch - __this_cpu_read(scx_dsp_ctx->cursor);
++}
++
++/**
++ * scx_bpf_dispatch_cancel - Cancel the latest dispatch
++ *
++ * Cancel the latest dispatch. Can be called multiple times to cancel further
++ * dispatches. Can only be called from ops.dispatch().
++ */
++__bpf_kfunc void scx_bpf_dispatch_cancel(void)
++{
++ struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
++
++ if (!scx_kf_allowed(SCX_KF_DISPATCH))
++ return;
++
++ if (dspc->cursor > 0)
++ dspc->cursor--;
++ else
++ scx_ops_error("dispatch buffer underflow");
++}
++
++/**
++ * scx_bpf_consume - Transfer a task from a DSQ to the current CPU's local DSQ
++ * @dsq_id: DSQ to consume
++ *
++ * Consume a task from the non-local DSQ identified by @dsq_id and transfer it
++ * to the current CPU's local DSQ for execution. Can only be called from
++ * ops.dispatch().
++ *
++ * This function flushes the in-flight dispatches from scx_bpf_dispatch() before
++ * trying to consume the specified DSQ. It may also grab rq locks and thus can't
++ * be called under any BPF locks.
++ *
++ * Returns %true if a task has been consumed, %false if there isn't any task to
++ * consume.
++ */
++__bpf_kfunc bool scx_bpf_consume(u64 dsq_id)
++{
++ struct scx_dsp_ctx *dspc = this_cpu_ptr(scx_dsp_ctx);
++ struct scx_dispatch_q *dsq;
++
++ if (!scx_kf_allowed(SCX_KF_DISPATCH))
++ return false;
++
++ flush_dispatch_buf(dspc->rq);
++
++ dsq = find_non_local_dsq(dsq_id);
++ if (unlikely(!dsq)) {
++ scx_ops_error("invalid DSQ ID 0x%016llx", dsq_id);
++ return false;
++ }
++
++ if (consume_dispatch_q(dspc->rq, dsq)) {
++ /*
++ * A successfully consumed task can be dequeued before it starts
++ * running while the CPU is trying to migrate other dispatched
++ * tasks. Bump nr_tasks to tell balance_scx() to retry on empty
++ * local DSQ.
++ */
++ dspc->nr_tasks++;
++ return true;
++ } else {
++ return false;
++ }
++}
++
++__bpf_kfunc_end_defs();
++
++BTF_KFUNCS_START(scx_kfunc_ids_dispatch)
++BTF_ID_FLAGS(func, scx_bpf_dispatch_nr_slots)
++BTF_ID_FLAGS(func, scx_bpf_dispatch_cancel)
++BTF_ID_FLAGS(func, scx_bpf_consume)
++BTF_KFUNCS_END(scx_kfunc_ids_dispatch)
++
++static const struct btf_kfunc_id_set scx_kfunc_set_dispatch = {
++ .owner = THIS_MODULE,
++ .set = &scx_kfunc_ids_dispatch,
++};
++
++__bpf_kfunc_start_defs();
++
++/**
++ * scx_bpf_reenqueue_local - Re-enqueue tasks on a local DSQ
++ *
++ * Iterate over all of the tasks currently enqueued on the local DSQ of the
++ * caller's CPU, and re-enqueue them in the BPF scheduler. Returns the number of
++ * processed tasks. Can only be called from ops.cpu_release().
++ */
++__bpf_kfunc u32 scx_bpf_reenqueue_local(void)
++{
++ LIST_HEAD(tasks);
++ u32 nr_enqueued = 0;
++ struct rq *rq;
++ struct task_struct *p, *n;
++
++ if (!scx_kf_allowed(SCX_KF_CPU_RELEASE))
++ return 0;
++
++ rq = cpu_rq(smp_processor_id());
++ lockdep_assert_rq_held(rq);
++
++ /*
++ * The BPF scheduler may choose to dispatch tasks back to
++ * @rq->scx.local_dsq. Move all candidate tasks off to a private list
++ * first to avoid processing the same tasks repeatedly.
++ */
++ list_for_each_entry_safe(p, n, &rq->scx.local_dsq.list,
++ scx.dsq_list.node) {
++ /*
++ * If @p is being migrated, @p's current CPU may not agree with
++ * its allowed CPUs and the migration_cpu_stop is about to
++ * deactivate and re-activate @p anyway. Skip re-enqueueing.
++ *
++ * While racing sched property changes may also dequeue and
++ * re-enqueue a migrating task while its current CPU and allowed
++ * CPUs disagree, they use %ENQUEUE_RESTORE which is bypassed to
++ * the current local DSQ for running tasks and thus are not
++ * visible to the BPF scheduler.
++ */
++ if (p->migration_pending)
++ continue;
++
++ dispatch_dequeue(rq, p);
++ list_add_tail(&p->scx.dsq_list.node, &tasks);
++ }
++
++ list_for_each_entry_safe(p, n, &tasks, scx.dsq_list.node) {
++ list_del_init(&p->scx.dsq_list.node);
++ do_enqueue_task(rq, p, SCX_ENQ_REENQ, -1);
++ nr_enqueued++;
++ }
++
++ return nr_enqueued;
++}
++
++__bpf_kfunc_end_defs();
++
++BTF_KFUNCS_START(scx_kfunc_ids_cpu_release)
++BTF_ID_FLAGS(func, scx_bpf_reenqueue_local)
++BTF_KFUNCS_END(scx_kfunc_ids_cpu_release)
++
++static const struct btf_kfunc_id_set scx_kfunc_set_cpu_release = {
++ .owner = THIS_MODULE,
++ .set = &scx_kfunc_ids_cpu_release,
++};
++
++__bpf_kfunc_start_defs();
++
++/**
++ * scx_bpf_kick_cpu - Trigger reschedule on a CPU
++ * @cpu: cpu to kick
++ * @flags: %SCX_KICK_* flags
++ *
++ * Kick @cpu into rescheduling. This can be used to wake up an idle CPU or
++ * trigger rescheduling on a busy CPU. This can be called from any online
++ * scx_ops operation and the actual kicking is performed asynchronously through
++ * an irq work.
++ */
++__bpf_kfunc void scx_bpf_kick_cpu(s32 cpu, u64 flags)
++{
++ struct rq *this_rq;
++ unsigned long irq_flags;
++
++ if (!ops_cpu_valid(cpu, NULL))
++ return;
++
++ /*
++ * While bypassing for PM ops, IRQ handling may not be online which can
++ * lead to irq_work_queue() malfunction such as infinite busy wait for
++ * IRQ status update. Suppress kicking.
++ */
++ if (scx_ops_bypassing())
++ return;
++
++ local_irq_save(irq_flags);
++
++ this_rq = this_rq();
++
++ /*
++ * Actual kicking is bounced to kick_cpus_irq_workfn() to avoid nesting
++ * rq locks. We can probably be smarter and avoid bouncing if called
++ * from ops which don't hold a rq lock.
++ */
++ if (flags & SCX_KICK_IDLE) {
++ struct rq *target_rq = cpu_rq(cpu);
++
++ if (unlikely(flags & (SCX_KICK_PREEMPT | SCX_KICK_WAIT)))
++ scx_ops_error("PREEMPT/WAIT cannot be used with SCX_KICK_IDLE");
++
++ if (raw_spin_rq_trylock(target_rq)) {
++ if (can_skip_idle_kick(target_rq)) {
++ raw_spin_rq_unlock(target_rq);
++ goto out;
++ }
++ raw_spin_rq_unlock(target_rq);
++ }
++ cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick_if_idle);
++ } else {
++ cpumask_set_cpu(cpu, this_rq->scx.cpus_to_kick);
++
++ if (flags & SCX_KICK_PREEMPT)
++ cpumask_set_cpu(cpu, this_rq->scx.cpus_to_preempt);
++ if (flags & SCX_KICK_WAIT)
++ cpumask_set_cpu(cpu, this_rq->scx.cpus_to_wait);
++ }
++
++ irq_work_queue(&this_rq->scx.kick_cpus_irq_work);
++out:
++ local_irq_restore(irq_flags);
++}
++
++/**
++ * scx_bpf_dsq_nr_queued - Return the number of queued tasks
++ * @dsq_id: id of the DSQ
++ *
++ * Return the number of tasks in the DSQ matching @dsq_id. If not found,
++ * -%ENOENT is returned.
++ */
++__bpf_kfunc s32 scx_bpf_dsq_nr_queued(u64 dsq_id)
++{
++ struct scx_dispatch_q *dsq;
++ s32 ret;
++
++ preempt_disable();
++
++ if (dsq_id == SCX_DSQ_LOCAL) {
++ ret = READ_ONCE(this_rq()->scx.local_dsq.nr);
++ goto out;
++ } else if ((dsq_id & SCX_DSQ_LOCAL_ON) == SCX_DSQ_LOCAL_ON) {
++ s32 cpu = dsq_id & SCX_DSQ_LOCAL_CPU_MASK;
++
++ if (ops_cpu_valid(cpu, NULL)) {
++ ret = READ_ONCE(cpu_rq(cpu)->scx.local_dsq.nr);
++ goto out;
++ }
++ } else {
++ dsq = find_non_local_dsq(dsq_id);
++ if (dsq) {
++ ret = READ_ONCE(dsq->nr);
++ goto out;
++ }
++ }
++ ret = -ENOENT;
++out:
++ preempt_enable();
++ return ret;
++}
++
++/**
++ * scx_bpf_destroy_dsq - Destroy a custom DSQ
++ * @dsq_id: DSQ to destroy
++ *
++ * Destroy the custom DSQ identified by @dsq_id. Only DSQs created with
++ * scx_bpf_create_dsq() can be destroyed. The caller must ensure that the DSQ is
++ * empty and no further tasks are dispatched to it. Ignored if called on a DSQ
++ * which doesn't exist. Can be called from any online scx_ops operations.
++ */
++__bpf_kfunc void scx_bpf_destroy_dsq(u64 dsq_id)
++{
++ destroy_dsq(dsq_id);
++}
++
++/**
++ * bpf_iter_scx_dsq_new - Create a DSQ iterator
++ * @it: iterator to initialize
++ * @dsq_id: DSQ to iterate
++ * @flags: %SCX_DSQ_ITER_*
++ *
++ * Initialize BPF iterator @it which can be used with bpf_for_each() to walk
++ * tasks in the DSQ specified by @dsq_id. Iteration using @it only includes
++ * tasks which are already queued when this function is invoked.
++ */
++__bpf_kfunc int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id,
++ u64 flags)
++{
++ struct bpf_iter_scx_dsq_kern *kit = (void *)it;
++
++ BUILD_BUG_ON(sizeof(struct bpf_iter_scx_dsq_kern) >
++ sizeof(struct bpf_iter_scx_dsq));
++ BUILD_BUG_ON(__alignof__(struct bpf_iter_scx_dsq_kern) !=
++ __alignof__(struct bpf_iter_scx_dsq));
++
++ if (flags & ~__SCX_DSQ_ITER_ALL_FLAGS)
++ return -EINVAL;
++
++ kit->dsq = find_non_local_dsq(dsq_id);
++ if (!kit->dsq)
++ return -ENOENT;
++
++ INIT_LIST_HEAD(&kit->cursor.node);
++ kit->cursor.is_bpf_iter_cursor = true;
++ kit->dsq_seq = READ_ONCE(kit->dsq->seq);
++ kit->flags = flags;
++
++ return 0;
++}
++
++/**
++ * bpf_iter_scx_dsq_next - Progress a DSQ iterator
++ * @it: iterator to progress
++ *
++ * Return the next task. See bpf_iter_scx_dsq_new().
++ */
++__bpf_kfunc struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it)
++{
++ struct bpf_iter_scx_dsq_kern *kit = (void *)it;
++ bool rev = kit->flags & SCX_DSQ_ITER_REV;
++ struct task_struct *p;
++ unsigned long flags;
++
++ if (!kit->dsq)
++ return NULL;
++
++ raw_spin_lock_irqsave(&kit->dsq->lock, flags);
++
++ if (list_empty(&kit->cursor.node))
++ p = NULL;
++ else
++ p = container_of(&kit->cursor, struct task_struct, scx.dsq_list);
++
++ /*
++ * Only tasks which were queued before the iteration started are
++ * visible. This bounds BPF iterations and guarantees that vtime never
++ * jumps in the other direction while iterating.
++ */
++ do {
++ p = nldsq_next_task(kit->dsq, p, rev);
++ } while (p && unlikely(u32_before(kit->dsq_seq, p->scx.dsq_seq)));
++
++ if (p) {
++ if (rev)
++ list_move_tail(&kit->cursor.node, &p->scx.dsq_list.node);
++ else
++ list_move(&kit->cursor.node, &p->scx.dsq_list.node);
++ } else {
++ list_del_init(&kit->cursor.node);
++ }
++
++ raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);
++
++ return p;
++}
++
++/**
++ * bpf_iter_scx_dsq_destroy - Destroy a DSQ iterator
++ * @it: iterator to destroy
++ *
++ * Undo scx_iter_scx_dsq_new().
++ */
++__bpf_kfunc void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it)
++{
++ struct bpf_iter_scx_dsq_kern *kit = (void *)it;
++
++ if (!kit->dsq)
++ return;
++
++ if (!list_empty(&kit->cursor.node)) {
++ unsigned long flags;
++
++ raw_spin_lock_irqsave(&kit->dsq->lock, flags);
++ list_del_init(&kit->cursor.node);
++ raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);
++ }
++ kit->dsq = NULL;
++}
++
++__bpf_kfunc_end_defs();
++
++static s32 __bstr_format(u64 *data_buf, char *line_buf, size_t line_size,
++ char *fmt, unsigned long long *data, u32 data__sz)
++{
++ struct bpf_bprintf_data bprintf_data = { .get_bin_args = true };
++ s32 ret;
++
++ if (data__sz % 8 || data__sz > MAX_BPRINTF_VARARGS * 8 ||
++ (data__sz && !data)) {
++ scx_ops_error("invalid data=%p and data__sz=%u",
++ (void *)data, data__sz);
++ return -EINVAL;
++ }
++
++ ret = copy_from_kernel_nofault(data_buf, data, data__sz);
++ if (ret < 0) {
++ scx_ops_error("failed to read data fields (%d)", ret);
++ return ret;
++ }
++
++ ret = bpf_bprintf_prepare(fmt, UINT_MAX, data_buf, data__sz / 8,
++ &bprintf_data);
++ if (ret < 0) {
++ scx_ops_error("format preparation failed (%d)", ret);
++ return ret;
++ }
++
++ ret = bstr_printf(line_buf, line_size, fmt,
++ bprintf_data.bin_args);
++ bpf_bprintf_cleanup(&bprintf_data);
++ if (ret < 0) {
++ scx_ops_error("(\"%s\", %p, %u) failed to format",
++ fmt, data, data__sz);
++ return ret;
++ }
++
++ return ret;
++}
++
++static s32 bstr_format(struct scx_bstr_buf *buf,
++ char *fmt, unsigned long long *data, u32 data__sz)
++{
++ return __bstr_format(buf->data, buf->line, sizeof(buf->line),
++ fmt, data, data__sz);
++}
++
++__bpf_kfunc_start_defs();
++
++/**
++ * scx_bpf_exit_bstr - Gracefully exit the BPF scheduler.
++ * @exit_code: Exit value to pass to user space via struct scx_exit_info.
++ * @fmt: error message format string
++ * @data: format string parameters packaged using ___bpf_fill() macro
++ * @data__sz: @data len, must end in '__sz' for the verifier
++ *
++ * Indicate that the BPF scheduler wants to exit gracefully, and initiate ops
++ * disabling.
++ */
++__bpf_kfunc void scx_bpf_exit_bstr(s64 exit_code, char *fmt,
++ unsigned long long *data, u32 data__sz)
++{
++ unsigned long flags;
++
++ raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags);
++ if (bstr_format(&scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
++ scx_ops_exit_kind(SCX_EXIT_UNREG_BPF, exit_code, "%s",
++ scx_exit_bstr_buf.line);
++ raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags);
++}
++
++/**
++ * scx_bpf_error_bstr - Indicate fatal error
++ * @fmt: error message format string
++ * @data: format string parameters packaged using ___bpf_fill() macro
++ * @data__sz: @data len, must end in '__sz' for the verifier
++ *
++ * Indicate that the BPF scheduler encountered a fatal error and initiate ops
++ * disabling.
++ */
++__bpf_kfunc void scx_bpf_error_bstr(char *fmt, unsigned long long *data,
++ u32 data__sz)
++{
++ unsigned long flags;
++
++ raw_spin_lock_irqsave(&scx_exit_bstr_buf_lock, flags);
++ if (bstr_format(&scx_exit_bstr_buf, fmt, data, data__sz) >= 0)
++ scx_ops_exit_kind(SCX_EXIT_ERROR_BPF, 0, "%s",
++ scx_exit_bstr_buf.line);
++ raw_spin_unlock_irqrestore(&scx_exit_bstr_buf_lock, flags);
++}
++
++/**
++ * scx_bpf_dump - Generate extra debug dump specific to the BPF scheduler
++ * @fmt: format string
++ * @data: format string parameters packaged using ___bpf_fill() macro
++ * @data__sz: @data len, must end in '__sz' for the verifier
++ *
++ * To be called through scx_bpf_dump() helper from ops.dump(), dump_cpu() and
++ * dump_task() to generate extra debug dump specific to the BPF scheduler.
++ *
++ * The extra dump may be multiple lines. A single line may be split over
++ * multiple calls. The last line is automatically terminated.
++ */
++__bpf_kfunc void scx_bpf_dump_bstr(char *fmt, unsigned long long *data,
++ u32 data__sz)
++{
++ struct scx_dump_data *dd = &scx_dump_data;
++ struct scx_bstr_buf *buf = &dd->buf;
++ s32 ret;
++
++ if (raw_smp_processor_id() != dd->cpu) {
++ scx_ops_error("scx_bpf_dump() must only be called from ops.dump() and friends");
++ return;
++ }
++
++ /* append the formatted string to the line buf */
++ ret = __bstr_format(buf->data, buf->line + dd->cursor,
++ sizeof(buf->line) - dd->cursor, fmt, data, data__sz);
++ if (ret < 0) {
++ dump_line(dd->s, "%s[!] (\"%s\", %p, %u) failed to format (%d)",
++ dd->prefix, fmt, data, data__sz, ret);
++ return;
++ }
++
++ dd->cursor += ret;
++ dd->cursor = min_t(s32, dd->cursor, sizeof(buf->line));
++
++ if (!dd->cursor)
++ return;
++
++ /*
++ * If the line buf overflowed or ends in a newline, flush it into the
++ * dump. This is to allow the caller to generate a single line over
++ * multiple calls. As ops_dump_flush() can also handle multiple lines in
++ * the line buf, the only case which can lead to an unexpected
++ * truncation is when the caller keeps generating newlines in the middle
++ * instead of the end consecutively. Don't do that.
++ */
++ if (dd->cursor >= sizeof(buf->line) || buf->line[dd->cursor - 1] == '\n')
++ ops_dump_flush();
++}
++
++/**
++ * scx_bpf_cpuperf_cap - Query the maximum relative capacity of a CPU
++ * @cpu: CPU of interest
++ *
++ * Return the maximum relative capacity of @cpu in relation to the most
++ * performant CPU in the system. The return value is in the range [1,
++ * %SCX_CPUPERF_ONE]. See scx_bpf_cpuperf_cur().
++ */
++__bpf_kfunc u32 scx_bpf_cpuperf_cap(s32 cpu)
++{
++ if (ops_cpu_valid(cpu, NULL))
++ return arch_scale_cpu_capacity(cpu);
++ else
++ return SCX_CPUPERF_ONE;
++}
++
++/**
++ * scx_bpf_cpuperf_cur - Query the current relative performance of a CPU
++ * @cpu: CPU of interest
++ *
++ * Return the current relative performance of @cpu in relation to its maximum.
++ * The return value is in the range [1, %SCX_CPUPERF_ONE].
++ *
++ * The current performance level of a CPU in relation to the maximum performance
++ * available in the system can be calculated as follows:
++ *
++ * scx_bpf_cpuperf_cap() * scx_bpf_cpuperf_cur() / %SCX_CPUPERF_ONE
++ *
++ * The result is in the range [1, %SCX_CPUPERF_ONE].
++ */
++__bpf_kfunc u32 scx_bpf_cpuperf_cur(s32 cpu)
++{
++ if (ops_cpu_valid(cpu, NULL))
++ return arch_scale_freq_capacity(cpu);
++ else
++ return SCX_CPUPERF_ONE;
++}
++
++/**
++ * scx_bpf_cpuperf_set - Set the relative performance target of a CPU
++ * @cpu: CPU of interest
++ * @perf: target performance level [0, %SCX_CPUPERF_ONE]
++ * @flags: %SCX_CPUPERF_* flags
++ *
++ * Set the target performance level of @cpu to @perf. @perf is in linear
++ * relative scale between 0 and %SCX_CPUPERF_ONE. This determines how the
++ * schedutil cpufreq governor chooses the target frequency.
++ *
++ * The actual performance level chosen, CPU grouping, and the overhead and
++ * latency of the operations are dependent on the hardware and cpufreq driver in
++ * use. Consult hardware and cpufreq documentation for more information. The
++ * current performance level can be monitored using scx_bpf_cpuperf_cur().
++ */
++__bpf_kfunc void scx_bpf_cpuperf_set(s32 cpu, u32 perf)
++{
++ if (unlikely(perf > SCX_CPUPERF_ONE)) {
++ scx_ops_error("Invalid cpuperf target %u for CPU %d", perf, cpu);
++ return;
++ }
++
++ if (ops_cpu_valid(cpu, NULL)) {
++ struct rq *rq = cpu_rq(cpu);
++
++ rq->scx.cpuperf_target = perf;
++
++ rcu_read_lock_sched_notrace();
++ cpufreq_update_util(cpu_rq(cpu), 0);
++ rcu_read_unlock_sched_notrace();
++ }
++}
++
++/**
++ * scx_bpf_nr_cpu_ids - Return the number of possible CPU IDs
++ *
++ * All valid CPU IDs in the system are smaller than the returned value.
++ */
++__bpf_kfunc u32 scx_bpf_nr_cpu_ids(void)
++{
++ return nr_cpu_ids;
++}
++
++/**
++ * scx_bpf_get_possible_cpumask - Get a referenced kptr to cpu_possible_mask
++ */
++__bpf_kfunc const struct cpumask *scx_bpf_get_possible_cpumask(void)
++{
++ return cpu_possible_mask;
++}
++
++/**
++ * scx_bpf_get_online_cpumask - Get a referenced kptr to cpu_online_mask
++ */
++__bpf_kfunc const struct cpumask *scx_bpf_get_online_cpumask(void)
++{
++ return cpu_online_mask;
++}
++
++/**
++ * scx_bpf_put_cpumask - Release a possible/online cpumask
++ * @cpumask: cpumask to release
++ */
++__bpf_kfunc void scx_bpf_put_cpumask(const struct cpumask *cpumask)
++{
++ /*
++ * Empty function body because we aren't actually acquiring or releasing
++ * a reference to a global cpumask, which is read-only in the caller and
++ * is never released. The acquire / release semantics here are just used
++ * to make the cpumask is a trusted pointer in the caller.
++ */
++}
++
++/**
++ * scx_bpf_get_idle_cpumask - Get a referenced kptr to the idle-tracking
++ * per-CPU cpumask.
++ *
++ * Returns NULL if idle tracking is not enabled, or running on a UP kernel.
++ */
++__bpf_kfunc const struct cpumask *scx_bpf_get_idle_cpumask(void)
++{
++ if (!static_branch_likely(&scx_builtin_idle_enabled)) {
++ scx_ops_error("built-in idle tracking is disabled");
++ return cpu_none_mask;
++ }
++
++#ifdef CONFIG_SMP
++ return idle_masks.cpu;
++#else
++ return cpu_none_mask;
++#endif
++}
++
++/**
++ * scx_bpf_get_idle_smtmask - Get a referenced kptr to the idle-tracking,
++ * per-physical-core cpumask. Can be used to determine if an entire physical
++ * core is free.
++ *
++ * Returns NULL if idle tracking is not enabled, or running on a UP kernel.
++ */
++__bpf_kfunc const struct cpumask *scx_bpf_get_idle_smtmask(void)
++{
++ if (!static_branch_likely(&scx_builtin_idle_enabled)) {
++ scx_ops_error("built-in idle tracking is disabled");
++ return cpu_none_mask;
++ }
++
++#ifdef CONFIG_SMP
++ if (sched_smt_active())
++ return idle_masks.smt;
++ else
++ return idle_masks.cpu;
++#else
++ return cpu_none_mask;
++#endif
++}
++
++/**
++ * scx_bpf_put_idle_cpumask - Release a previously acquired referenced kptr to
++ * either the percpu, or SMT idle-tracking cpumask.
++ */
++__bpf_kfunc void scx_bpf_put_idle_cpumask(const struct cpumask *idle_mask)
++{
++ /*
++ * Empty function body because we aren't actually acquiring or releasing
++ * a reference to a global idle cpumask, which is read-only in the
++ * caller and is never released. The acquire / release semantics here
++ * are just used to make the cpumask a trusted pointer in the caller.
++ */
++}
++
++/**
++ * scx_bpf_test_and_clear_cpu_idle - Test and clear @cpu's idle state
++ * @cpu: cpu to test and clear idle for
++ *
++ * Returns %true if @cpu was idle and its idle state was successfully cleared.
++ * %false otherwise.
++ *
++ * Unavailable if ops.update_idle() is implemented and
++ * %SCX_OPS_KEEP_BUILTIN_IDLE is not set.
++ */
++__bpf_kfunc bool scx_bpf_test_and_clear_cpu_idle(s32 cpu)
++{
++ if (!static_branch_likely(&scx_builtin_idle_enabled)) {
++ scx_ops_error("built-in idle tracking is disabled");
++ return false;
++ }
++
++ if (ops_cpu_valid(cpu, NULL))
++ return test_and_clear_cpu_idle(cpu);
++ else
++ return false;
++}
++
++/**
++ * scx_bpf_pick_idle_cpu - Pick and claim an idle cpu
++ * @cpus_allowed: Allowed cpumask
++ * @flags: %SCX_PICK_IDLE_CPU_* flags
++ *
++ * Pick and claim an idle cpu in @cpus_allowed. Returns the picked idle cpu
++ * number on success. -%EBUSY if no matching cpu was found.
++ *
++ * Idle CPU tracking may race against CPU scheduling state transitions. For
++ * example, this function may return -%EBUSY as CPUs are transitioning into the
++ * idle state. If the caller then assumes that there will be dispatch events on
++ * the CPUs as they were all busy, the scheduler may end up stalling with CPUs
++ * idling while there are pending tasks. Use scx_bpf_pick_any_cpu() and
++ * scx_bpf_kick_cpu() to guarantee that there will be at least one dispatch
++ * event in the near future.
++ *
++ * Unavailable if ops.update_idle() is implemented and
++ * %SCX_OPS_KEEP_BUILTIN_IDLE is not set.
++ */
++__bpf_kfunc s32 scx_bpf_pick_idle_cpu(const struct cpumask *cpus_allowed,
++ u64 flags)
++{
++ if (!static_branch_likely(&scx_builtin_idle_enabled)) {
++ scx_ops_error("built-in idle tracking is disabled");
++ return -EBUSY;
++ }
++
++ return scx_pick_idle_cpu(cpus_allowed, flags);
++}
++
++/**
++ * scx_bpf_pick_any_cpu - Pick and claim an idle cpu if available or pick any CPU
++ * @cpus_allowed: Allowed cpumask
++ * @flags: %SCX_PICK_IDLE_CPU_* flags
++ *
++ * Pick and claim an idle cpu in @cpus_allowed. If none is available, pick any
++ * CPU in @cpus_allowed. Guaranteed to succeed and returns the picked idle cpu
++ * number if @cpus_allowed is not empty. -%EBUSY is returned if @cpus_allowed is
++ * empty.
++ *
++ * If ops.update_idle() is implemented and %SCX_OPS_KEEP_BUILTIN_IDLE is not
++ * set, this function can't tell which CPUs are idle and will always pick any
++ * CPU.
++ */
++__bpf_kfunc s32 scx_bpf_pick_any_cpu(const struct cpumask *cpus_allowed,
++ u64 flags)
++{
++ s32 cpu;
++
++ if (static_branch_likely(&scx_builtin_idle_enabled)) {
++ cpu = scx_pick_idle_cpu(cpus_allowed, flags);
++ if (cpu >= 0)
++ return cpu;
++ }
++
++ cpu = cpumask_any_distribute(cpus_allowed);
++ if (cpu < nr_cpu_ids)
++ return cpu;
++ else
++ return -EBUSY;
++}
++
++/**
++ * scx_bpf_task_running - Is task currently running?
++ * @p: task of interest
++ */
++__bpf_kfunc bool scx_bpf_task_running(const struct task_struct *p)
++{
++ return task_rq(p)->curr == p;
++}
++
++/**
++ * scx_bpf_task_cpu - CPU a task is currently associated with
++ * @p: task of interest
++ */
++__bpf_kfunc s32 scx_bpf_task_cpu(const struct task_struct *p)
++{
++ return task_cpu(p);
++}
++
++/**
++ * scx_bpf_cpu_rq - Fetch the rq of a CPU
++ * @cpu: CPU of the rq
++ */
++__bpf_kfunc struct rq *scx_bpf_cpu_rq(s32 cpu)
++{
++ if (!ops_cpu_valid(cpu, NULL))
++ return NULL;
++
++ return cpu_rq(cpu);
++}
++
++__bpf_kfunc_end_defs();
++
++BTF_KFUNCS_START(scx_kfunc_ids_any)
++BTF_ID_FLAGS(func, scx_bpf_kick_cpu)
++BTF_ID_FLAGS(func, scx_bpf_dsq_nr_queued)
++BTF_ID_FLAGS(func, scx_bpf_destroy_dsq)
++BTF_ID_FLAGS(func, bpf_iter_scx_dsq_new, KF_ITER_NEW | KF_RCU_PROTECTED)
++BTF_ID_FLAGS(func, bpf_iter_scx_dsq_next, KF_ITER_NEXT | KF_RET_NULL)
++BTF_ID_FLAGS(func, bpf_iter_scx_dsq_destroy, KF_ITER_DESTROY)
++BTF_ID_FLAGS(func, scx_bpf_exit_bstr, KF_TRUSTED_ARGS)
++BTF_ID_FLAGS(func, scx_bpf_error_bstr, KF_TRUSTED_ARGS)
++BTF_ID_FLAGS(func, scx_bpf_dump_bstr, KF_TRUSTED_ARGS)
++BTF_ID_FLAGS(func, scx_bpf_cpuperf_cap)
++BTF_ID_FLAGS(func, scx_bpf_cpuperf_cur)
++BTF_ID_FLAGS(func, scx_bpf_cpuperf_set)
++BTF_ID_FLAGS(func, scx_bpf_nr_cpu_ids)
++BTF_ID_FLAGS(func, scx_bpf_get_possible_cpumask, KF_ACQUIRE)
++BTF_ID_FLAGS(func, scx_bpf_get_online_cpumask, KF_ACQUIRE)
++BTF_ID_FLAGS(func, scx_bpf_put_cpumask, KF_RELEASE)
++BTF_ID_FLAGS(func, scx_bpf_get_idle_cpumask, KF_ACQUIRE)
++BTF_ID_FLAGS(func, scx_bpf_get_idle_smtmask, KF_ACQUIRE)
++BTF_ID_FLAGS(func, scx_bpf_put_idle_cpumask, KF_RELEASE)
++BTF_ID_FLAGS(func, scx_bpf_test_and_clear_cpu_idle)
++BTF_ID_FLAGS(func, scx_bpf_pick_idle_cpu, KF_RCU)
++BTF_ID_FLAGS(func, scx_bpf_pick_any_cpu, KF_RCU)
++BTF_ID_FLAGS(func, scx_bpf_task_running, KF_RCU)
++BTF_ID_FLAGS(func, scx_bpf_task_cpu, KF_RCU)
++BTF_ID_FLAGS(func, scx_bpf_cpu_rq)
++BTF_KFUNCS_END(scx_kfunc_ids_any)
++
++static const struct btf_kfunc_id_set scx_kfunc_set_any = {
++ .owner = THIS_MODULE,
++ .set = &scx_kfunc_ids_any,
++};
++
++static int __init scx_init(void)
++{
++ int ret;
++
++ /*
++ * kfunc registration can't be done from init_sched_ext_class() as
++ * register_btf_kfunc_id_set() needs most of the system to be up.
++ *
++ * Some kfuncs are context-sensitive and can only be called from
++ * specific SCX ops. They are grouped into BTF sets accordingly.
++ * Unfortunately, BPF currently doesn't have a way of enforcing such
++ * restrictions. Eventually, the verifier should be able to enforce
++ * them. For now, register them the same and make each kfunc explicitly
++ * check using scx_kf_allowed().
++ */
++ if ((ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
++ &scx_kfunc_set_sleepable)) ||
++ (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
++ &scx_kfunc_set_select_cpu)) ||
++ (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
++ &scx_kfunc_set_enqueue_dispatch)) ||
++ (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
++ &scx_kfunc_set_dispatch)) ||
++ (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
++ &scx_kfunc_set_cpu_release)) ||
++ (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS,
++ &scx_kfunc_set_any)) ||
++ (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_TRACING,
++ &scx_kfunc_set_any)) ||
++ (ret = register_btf_kfunc_id_set(BPF_PROG_TYPE_SYSCALL,
++ &scx_kfunc_set_any))) {
++ pr_err("sched_ext: Failed to register kfunc sets (%d)\n", ret);
++ return ret;
++ }
++
++ ret = register_bpf_struct_ops(&bpf_sched_ext_ops, sched_ext_ops);
++ if (ret) {
++ pr_err("sched_ext: Failed to register struct_ops (%d)\n", ret);
++ return ret;
++ }
++
++ ret = register_pm_notifier(&scx_pm_notifier);
++ if (ret) {
++ pr_err("sched_ext: Failed to register PM notifier (%d)\n", ret);
++ return ret;
++ }
++
++ scx_kset = kset_create_and_add("sched_ext", &scx_uevent_ops, kernel_kobj);
++ if (!scx_kset) {
++ pr_err("sched_ext: Failed to create /sys/kernel/sched_ext\n");
++ return -ENOMEM;
++ }
++
++ ret = sysfs_create_group(&scx_kset->kobj, &scx_global_attr_group);
++ if (ret < 0) {
++ pr_err("sched_ext: Failed to add global attributes\n");
++ return ret;
++ }
++
++ return 0;
++}
++__initcall(scx_init);
+diff --git a/kernel/sched/ext.h b/kernel/sched/ext.h
+new file mode 100644
+index 000000000000..32d3a51f591a
+--- /dev/null
++++ b/kernel/sched/ext.h
+@@ -0,0 +1,69 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * BPF extensible scheduler class: Documentation/scheduler/sched-ext.rst
++ *
++ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
++ */
++#ifdef CONFIG_SCHED_CLASS_EXT
++
++void scx_tick(struct rq *rq);
++void init_scx_entity(struct sched_ext_entity *scx);
++void scx_pre_fork(struct task_struct *p);
++int scx_fork(struct task_struct *p);
++void scx_post_fork(struct task_struct *p);
++void scx_cancel_fork(struct task_struct *p);
++bool scx_can_stop_tick(struct rq *rq);
++void scx_rq_activate(struct rq *rq);
++void scx_rq_deactivate(struct rq *rq);
++int scx_check_setscheduler(struct task_struct *p, int policy);
++bool task_should_scx(struct task_struct *p);
++void init_sched_ext_class(void);
++
++static inline u32 scx_cpuperf_target(s32 cpu)
++{
++ if (scx_enabled())
++ return cpu_rq(cpu)->scx.cpuperf_target;
++ else
++ return 0;
++}
++
++static inline bool task_on_scx(const struct task_struct *p)
++{
++ return scx_enabled() && p->sched_class == &ext_sched_class;
++}
++
++#ifdef CONFIG_SCHED_CORE
++bool scx_prio_less(const struct task_struct *a, const struct task_struct *b,
++ bool in_fi);
++#endif
++
++#else /* CONFIG_SCHED_CLASS_EXT */
++
++static inline void scx_tick(struct rq *rq) {}
++static inline void scx_pre_fork(struct task_struct *p) {}
++static inline int scx_fork(struct task_struct *p) { return 0; }
++static inline void scx_post_fork(struct task_struct *p) {}
++static inline void scx_cancel_fork(struct task_struct *p) {}
++static inline u32 scx_cpuperf_target(s32 cpu) { return 0; }
++static inline bool scx_can_stop_tick(struct rq *rq) { return true; }
++static inline void scx_rq_activate(struct rq *rq) {}
++static inline void scx_rq_deactivate(struct rq *rq) {}
++static inline int scx_check_setscheduler(struct task_struct *p, int policy) { return 0; }
++static inline bool task_on_scx(const struct task_struct *p) { return false; }
++static inline void init_sched_ext_class(void) {}
++
++#endif /* CONFIG_SCHED_CLASS_EXT */
++
++#if defined(CONFIG_SCHED_CLASS_EXT) && defined(CONFIG_SMP)
++void __scx_update_idle(struct rq *rq, bool idle);
++
++static inline void scx_update_idle(struct rq *rq, bool idle)
++{
++ if (scx_enabled())
++ __scx_update_idle(rq, idle);
++}
++#else
++static inline void scx_update_idle(struct rq *rq, bool idle) {}
++#endif
+diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
+index 213c94d027a4..ee4fe81ba456 100644
+--- a/kernel/sched/fair.c
++++ b/kernel/sched/fair.c
+@@ -4074,6 +4074,17 @@
+ load->inv_weight = sched_prio_to_wmult[prio];
+ }
+
++static void reweight_task_fair(struct rq *rq, struct task_struct *p,
++ const struct load_weight *lw)
++{
++ struct sched_entity *se = &p->se;
++ struct cfs_rq *cfs_rq = cfs_rq_of(se);
++ struct load_weight *load = &se->load;
++
++ reweight_entity(cfs_rq, se, lw->weight);
++ load->inv_weight = lw->inv_weight;
++}
++
+ static inline int throttled_hierarchy(struct cfs_rq *cfs_rq);
+
+ #ifdef CONFIG_FAIR_GROUP_SCHED
+@@ -8348,7 +8348,7 @@ static void check_preempt_wakeup_fair(struct rq *rq, struct task_struct *p, int
+ * Batch and idle tasks do not preempt non-idle tasks (their preemption
+ * is driven by the tick):
+ */
+- if (unlikely(p->policy != SCHED_NORMAL) || !sched_feat(WAKEUP_PREEMPTION))
++ if (unlikely(!normal_policy(p->policy)) || !sched_feat(WAKEUP_PREEMPTION))
+ return;
+
+ find_matching_se(&se, &pse);
+@@ -9309,28 +9309,18 @@ static inline void update_blocked_load_status(struct rq *rq, bool has_blocked) {
+
+ static bool __update_blocked_others(struct rq *rq, bool *done)
+ {
+- const struct sched_class *curr_class;
+- u64 now = rq_clock_pelt(rq);
+- unsigned long thermal_pressure;
+- bool decayed;
++ bool updated;
+
+ /*
+ * update_load_avg() can call cpufreq_update_util(). Make sure that RT,
+ * DL and IRQ signals have been updated before updating CFS.
+ */
+- curr_class = rq->curr->sched_class;
+-
+- thermal_pressure = arch_scale_thermal_pressure(cpu_of(rq));
+-
+- decayed = update_rt_rq_load_avg(now, rq, curr_class == &rt_sched_class) |
+- update_dl_rq_load_avg(now, rq, curr_class == &dl_sched_class) |
+- update_thermal_load_avg(rq_clock_thermal(rq), rq, thermal_pressure) |
+- update_irq_load_avg(rq, 0);
++ updated = update_other_load_avgs(rq);
+
+ if (others_have_blocked(rq))
+ *done = false;
+
+- return decayed;
++ return updated;
+ }
+
+ #ifdef CONFIG_FAIR_GROUP_SCHED
+@@ -13156,6 +13146,7 @@ DEFINE_SCHED_CLASS(fair) = {
+ .task_tick = task_tick_fair,
+ .task_fork = task_fork_fair,
+
++ .reweight_task = reweight_task_fair,
+ .prio_changed = prio_changed_fair,
+ .switched_from = switched_from_fair,
+ .switched_to = switched_to_fair,
+diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c
+index 6135fbe83d68..3b6540cc436a 100644
+--- a/kernel/sched/idle.c
++++ b/kernel/sched/idle.c
+@@ -458,11 +458,13 @@ static void wakeup_preempt_idle(struct rq *rq, struct task_struct *p, int flags)
+
+ static void put_prev_task_idle(struct rq *rq, struct task_struct *prev)
+ {
++ scx_update_idle(rq, false);
+ }
+
+ static void set_next_task_idle(struct rq *rq, struct task_struct *next, bool first)
+ {
+ update_idle_core(rq);
++ scx_update_idle(rq, true);
+ schedstat_inc(rq->sched_goidle);
+ }
+
+diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
+index ae50f212775e..9ce5074e8a8d 100644
+--- a/kernel/sched/sched.h
++++ b/kernel/sched/sched.h
+@@ -173,9 +173,19 @@ static inline int idle_policy(int policy)
+ {
+ return policy == SCHED_IDLE;
+ }
++
++static inline int normal_policy(int policy)
++{
++#ifdef CONFIG_SCHED_CLASS_EXT
++ if (policy == SCHED_EXT)
++ return true;
++#endif
++ return policy == SCHED_NORMAL;
++}
++
+ static inline int fair_policy(int policy)
+ {
+- return policy == SCHED_NORMAL || policy == SCHED_BATCH;
++ return normal_policy(policy) || policy == SCHED_BATCH;
+ }
+
+ static inline int rt_policy(int policy)
+@@ -223,6 +233,24 @@ static inline void update_avg(u64 *avg, u64 sample)
+ #define shr_bound(val, shift) \
+ (val >> min_t(typeof(shift), shift, BITS_PER_TYPE(typeof(val)) - 1))
+
++/*
++ * cgroup weight knobs should use the common MIN, DFL and MAX values which are
++ * 1, 100 and 10000 respectively. While it loses a bit of range on both ends, it
++ * maps pretty well onto the shares value used by scheduler and the round-trip
++ * conversions preserve the original value over the entire range.
++ */
++static inline unsigned long sched_weight_from_cgroup(unsigned long cgrp_weight)
++{
++ return DIV_ROUND_CLOSEST_ULL(cgrp_weight * 1024, CGROUP_WEIGHT_DFL);
++}
++
++static inline unsigned long sched_weight_to_cgroup(unsigned long weight)
++{
++ return clamp_t(unsigned long,
++ DIV_ROUND_CLOSEST_ULL(weight * CGROUP_WEIGHT_DFL, 1024),
++ CGROUP_WEIGHT_MIN, CGROUP_WEIGHT_MAX);
++}
++
+ /*
+ * !! For sched_setattr_nocheck() (kernel) only !!
+ *
+@@ -461,6 +489,11 @@ static inline int walk_tg_tree(tg_visitor down, tg_visitor up, void *data)
+ return walk_tg_tree_from(&root_task_group, down, up, data);
+ }
+
++static inline struct task_group *css_tg(struct cgroup_subsys_state *css)
++{
++ return css ? container_of(css, struct task_group, css) : NULL;
++}
++
+ extern int tg_nop(struct task_group *tg, void *data);
+
+ #ifdef CONFIG_FAIR_GROUP_SCHED
+@@ -569,6 +602,12 @@ do { \
+ # define u64_u32_load(var) u64_u32_load_copy(var, var##_copy)
+ # define u64_u32_store(var, val) u64_u32_store_copy(var, var##_copy, val)
+
++struct rq;
++struct balance_callback {
++ struct balance_callback *next;
++ void (*func)(struct rq *rq);
++};
++
+ /* CFS-related fields in a runqueue */
+ struct cfs_rq {
+ struct load_weight load;
+@@ -677,6 +716,42 @@ struct cfs_rq {
+ #endif /* CONFIG_FAIR_GROUP_SCHED */
+ };
+
++#ifdef CONFIG_SCHED_CLASS_EXT
++/* scx_rq->flags, protected by the rq lock */
++enum scx_rq_flags {
++ /*
++ * A hotplugged CPU starts scheduling before rq_online_scx(). Track
++ * ops.cpu_on/offline() state so that ops.enqueue/dispatch() are called
++ * only while the BPF scheduler considers the CPU to be online.
++ */
++ SCX_RQ_ONLINE = 1 << 0,
++ SCX_RQ_CAN_STOP_TICK = 1 << 1,
++
++ SCX_RQ_IN_WAKEUP = 1 << 16,
++ SCX_RQ_IN_BALANCE = 1 << 17,
++};
++
++struct scx_rq {
++ struct scx_dispatch_q local_dsq;
++ struct list_head runnable_list; /* runnable tasks on this rq */
++ struct list_head ddsp_deferred_locals; /* deferred ddsps from enq */
++ unsigned long ops_qseq;
++ u64 extra_enq_flags; /* see move_task_to_local_dsq() */
++ u32 nr_running;
++ u32 flags;
++ u32 cpuperf_target; /* [0, SCHED_CAPACITY_SCALE] */
++ bool cpu_released;
++ cpumask_var_t cpus_to_kick;
++ cpumask_var_t cpus_to_kick_if_idle;
++ cpumask_var_t cpus_to_preempt;
++ cpumask_var_t cpus_to_wait;
++ unsigned long pnt_seq;
++ struct balance_callback deferred_bal_cb;
++ struct irq_work deferred_irq_work;
++ struct irq_work kick_cpus_irq_work;
++};
++#endif /* CONFIG_SCHED_CLASS_EXT */
++
+ static inline int rt_bandwidth_enabled(void)
+ {
+ return sysctl_sched_rt_runtime >= 0;
+@@ -969,12 +1044,6 @@ struct uclamp_rq {
+ DECLARE_STATIC_KEY_FALSE(sched_uclamp_used);
+ #endif /* CONFIG_UCLAMP_TASK */
+
+-struct rq;
+-struct balance_callback {
+- struct balance_callback *next;
+- void (*func)(struct rq *rq);
+-};
+-
+ /*
+ * This is the main, per-CPU runqueue data structure.
+ *
+@@ -1017,6 +1086,9 @@ struct rq {
+ struct cfs_rq cfs;
+ struct rt_rq rt;
+ struct dl_rq dl;
++#ifdef CONFIG_SCHED_CLASS_EXT
++ struct scx_rq scx;
++#endif
+
+ #ifdef CONFIG_FAIR_GROUP_SCHED
+ /* list of leaf cfs_rq on this CPU: */
+@@ -2276,6 +2348,8 @@ struct sched_class {
+ void (*put_prev_task)(struct rq *rq, struct task_struct *p);
+ void (*set_next_task)(struct rq *rq, struct task_struct *p, bool first);
+
++ void (*switch_class)(struct rq *rq, struct task_struct *next);
++
+ #ifdef CONFIG_SMP
+ int (*balance)(struct rq *rq, struct task_struct *prev, struct rq_flags *rf);
+ int (*select_task_rq)(struct task_struct *p, int task_cpu, int flags);
+@@ -2303,8 +2377,11 @@ struct sched_class {
+ * cannot assume the switched_from/switched_to pair is serialized by
+ * rq->lock. They are however serialized by p->pi_lock.
+ */
++ void (*switching_to) (struct rq *this_rq, struct task_struct *task);
+ void (*switched_from)(struct rq *this_rq, struct task_struct *task);
+ void (*switched_to) (struct rq *this_rq, struct task_struct *task);
++ void (*reweight_task)(struct rq *this_rq, struct task_struct *task,
++ const struct load_weight *lw);
+ void (*prio_changed) (struct rq *this_rq, struct task_struct *task,
+ int oldprio);
+
+@@ -2353,19 +2430,54 @@ const struct sched_class name##_sched_class \
+ extern struct sched_class __sched_class_highest[];
+ extern struct sched_class __sched_class_lowest[];
+
++extern const struct sched_class stop_sched_class;
++extern const struct sched_class dl_sched_class;
++extern const struct sched_class rt_sched_class;
++extern const struct sched_class fair_sched_class;
++extern const struct sched_class idle_sched_class;
++
++#ifdef CONFIG_SCHED_CLASS_EXT
++extern const struct sched_class ext_sched_class;
++
++DECLARE_STATIC_KEY_FALSE(__scx_ops_enabled); /* SCX BPF scheduler loaded */
++DECLARE_STATIC_KEY_FALSE(__scx_switched_all); /* all fair class tasks on SCX */
++
++#define scx_enabled() static_branch_unlikely(&__scx_ops_enabled)
++#define scx_switched_all() static_branch_unlikely(&__scx_switched_all)
++#else /* !CONFIG_SCHED_CLASS_EXT */
++#define scx_enabled() false
++#define scx_switched_all() false
++#endif /* !CONFIG_SCHED_CLASS_EXT */
++
++/*
++ * Iterate only active classes. SCX can take over all fair tasks or be
++ * completely disabled. If the former, skip fair. If the latter, skip SCX.
++ */
++static inline const struct sched_class *next_active_class(const struct sched_class *class)
++{
++ class++;
++#ifdef CONFIG_SCHED_CLASS_EXT
++ if (scx_switched_all() && class == &fair_sched_class)
++ class++;
++ if (!scx_enabled() && class == &ext_sched_class)
++ class++;
++#endif
++ return class;
++}
++
+ #define for_class_range(class, _from, _to) \
+ for (class = (_from); class < (_to); class++)
+
+ #define for_each_class(class) \
+ for_class_range(class, __sched_class_highest, __sched_class_lowest)
+
+-#define sched_class_above(_a, _b) ((_a) < (_b))
++#define for_active_class_range(class, _from, _to) \
++ for (class = (_from); class != (_to); class = next_active_class(class))
+
+-extern const struct sched_class stop_sched_class;
+-extern const struct sched_class dl_sched_class;
+-extern const struct sched_class rt_sched_class;
+-extern const struct sched_class fair_sched_class;
+-extern const struct sched_class idle_sched_class;
++#define for_each_active_class(class) \
++ for_active_class_range(class, __sched_class_highest, __sched_class_lowest)
++
++#define sched_class_above(_a, _b) ((_a) < (_b))
+
+ static inline bool sched_stop_runnable(struct rq *rq)
+ {
+@@ -2462,7 +2574,7 @@ extern void init_sched_dl_class(void);
+ extern void init_sched_rt_class(void);
+ extern void init_sched_fair_class(void);
+
+-extern void reweight_task(struct task_struct *p, int prio);
++extern void __setscheduler_prio(struct task_struct *p, int prio);
+
+ extern void resched_curr(struct rq *rq);
+ extern void resched_cpu(int cpu);
+@@ -2542,6 +2654,12 @@ static inline void sub_nr_running(struct rq *rq, unsigned count)
+ extern void activate_task(struct rq *rq, struct task_struct *p, int flags);
+ extern void deactivate_task(struct rq *rq, struct task_struct *p, int flags);
+
++extern void check_class_changing(struct rq *rq, struct task_struct *p,
++ const struct sched_class *prev_class);
++extern void check_class_changed(struct rq *rq, struct task_struct *p,
++ const struct sched_class *prev_class,
++ int oldprio);
++
+ extern void wakeup_preempt(struct rq *rq, struct task_struct *p, int flags);
+
+ #ifdef CONFIG_PREEMPT_RT
+@@ -3007,6 +3125,9 @@ static inline void cpufreq_update_util(struct rq *rq, unsigned int flags) {}
+ #endif
+
+ #ifdef CONFIG_SMP
++
++bool update_other_load_avgs(struct rq *rq);
++
+ unsigned long effective_cpu_util(int cpu, unsigned long util_cfs,
+ unsigned long *min,
+ unsigned long *max);
+@@ -3049,6 +3170,8 @@ static inline unsigned long cpu_util_rt(struct rq *rq)
+ {
+ return READ_ONCE(rq->avg_rt.util_avg);
+ }
++#else /* !CONFIG_SMP */
++static inline bool update_other_load_avgs(struct rq *rq) { return false; }
+ #endif
+
+ #ifdef CONFIG_UCLAMP_TASK
+@@ -3481,4 +3604,24 @@ static inline void init_sched_mm_cid(struct task_struct *t) { }
+ extern u64 avg_vruntime(struct cfs_rq *cfs_rq);
+ extern int entity_eligible(struct cfs_rq *cfs_rq, struct sched_entity *se);
+
++#ifdef CONFIG_SCHED_CLASS_EXT
++/*
++ * Used by SCX in the enable/disable paths to move tasks between sched_classes
++ * and establish invariants.
++ */
++struct sched_enq_and_set_ctx {
++ struct task_struct *p;
++ int queue_flags;
++ bool queued;
++ bool running;
++};
++
++void sched_deq_and_put_task(struct task_struct *p, int queue_flags,
++ struct sched_enq_and_set_ctx *ctx);
++void sched_enq_and_set_task(struct sched_enq_and_set_ctx *ctx);
++
++#endif /* CONFIG_SCHED_CLASS_EXT */
++
++#include "ext.h"
++
+ #endif /* _KERNEL_SCHED_SCHED_H */
+diff --git a/lib/dump_stack.c b/lib/dump_stack.c
+index 222c6d6c8281..9581ef4efec5 100644
+--- a/lib/dump_stack.c
++++ b/lib/dump_stack.c
+@@ -68,6 +68,7 @@ void dump_stack_print_info(const char *log_lvl)
+
+ print_worker_info(log_lvl, current);
+ print_stop_info(log_lvl, current);
++ print_scx_info(log_lvl, current);
+ }
+
+ /**
+diff --git a/tools/Makefile b/tools/Makefile
+index 276f5d0d53a4..278d24723b74 100644
+--- a/tools/Makefile
++++ b/tools/Makefile
+@@ -28,6 +28,7 @@ help:
+ @echo ' pci - PCI tools'
+ @echo ' perf - Linux performance measurement and analysis tool'
+ @echo ' selftests - various kernel selftests'
++ @echo ' sched_ext - sched_ext example schedulers'
+ @echo ' bootconfig - boot config tool'
+ @echo ' spi - spi tools'
+ @echo ' tmon - thermal monitoring and tuning tool'
+@@ -91,6 +92,9 @@ perf: FORCE
+ $(Q)mkdir -p $(PERF_O) .
+ $(Q)$(MAKE) --no-print-directory -C perf O=$(PERF_O) subdir=
+
++sched_ext: FORCE
++ $(call descend,sched_ext)
++
+ selftests: FORCE
+ $(call descend,testing/$@)
+
+@@ -184,6 +188,9 @@ perf_clean:
+ $(Q)mkdir -p $(PERF_O) .
+ $(Q)$(MAKE) --no-print-directory -C perf O=$(PERF_O) subdir= clean
+
++sched_ext_clean:
++ $(call descend,sched_ext,clean)
++
+ selftests_clean:
+ $(call descend,testing/$(@:_clean=),clean)
+
+@@ -213,6 +220,7 @@ clean: acpi_clean counter_clean cpupower_clean hv_clean firewire_clean \
+ mm_clean bpf_clean iio_clean x86_energy_perf_policy_clean tmon_clean \
+ freefall_clean build_clean libbpf_clean libsubcmd_clean \
+ gpio_clean objtool_clean leds_clean wmi_clean pci_clean firmware_clean debugging_clean \
+- intel-speed-select_clean tracing_clean thermal_clean thermometer_clean thermal-engine_clean
++ intel-speed-select_clean tracing_clean thermal_clean thermometer_clean thermal-engine_clean \
++ sched_ext_clean
+
+ .PHONY: FORCE
+diff --git a/tools/sched_ext/.gitignore b/tools/sched_ext/.gitignore
+new file mode 100644
+index 000000000000..d6264fe1c8cd
+--- /dev/null
++++ b/tools/sched_ext/.gitignore
+@@ -0,0 +1,2 @@
++tools/
++build/
+diff --git a/tools/sched_ext/Makefile b/tools/sched_ext/Makefile
+new file mode 100644
+index 000000000000..bf7e108f5ae1
+--- /dev/null
++++ b/tools/sched_ext/Makefile
+@@ -0,0 +1,246 @@
++# SPDX-License-Identifier: GPL-2.0
++# Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++include ../build/Build.include
++include ../scripts/Makefile.arch
++include ../scripts/Makefile.include
++
++all: all_targets
++
++ifneq ($(LLVM),)
++ifneq ($(filter %/,$(LLVM)),)
++LLVM_PREFIX := $(LLVM)
++else ifneq ($(filter -%,$(LLVM)),)
++LLVM_SUFFIX := $(LLVM)
++endif
++
++CLANG_TARGET_FLAGS_arm := arm-linux-gnueabi
++CLANG_TARGET_FLAGS_arm64 := aarch64-linux-gnu
++CLANG_TARGET_FLAGS_hexagon := hexagon-linux-musl
++CLANG_TARGET_FLAGS_m68k := m68k-linux-gnu
++CLANG_TARGET_FLAGS_mips := mipsel-linux-gnu
++CLANG_TARGET_FLAGS_powerpc := powerpc64le-linux-gnu
++CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu
++CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu
++CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu
++CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(ARCH))
++
++ifeq ($(CROSS_COMPILE),)
++ifeq ($(CLANG_TARGET_FLAGS),)
++$(error Specify CROSS_COMPILE or add '--target=' option to lib.mk)
++else
++CLANG_FLAGS += --target=$(CLANG_TARGET_FLAGS)
++endif # CLANG_TARGET_FLAGS
++else
++CLANG_FLAGS += --target=$(notdir $(CROSS_COMPILE:%-=%))
++endif # CROSS_COMPILE
++
++CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as
++else
++CC := $(CROSS_COMPILE)gcc
++endif # LLVM
++
++CURDIR := $(abspath .)
++TOOLSDIR := $(abspath ..)
++LIBDIR := $(TOOLSDIR)/lib
++BPFDIR := $(LIBDIR)/bpf
++TOOLSINCDIR := $(TOOLSDIR)/include
++BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool
++APIDIR := $(TOOLSINCDIR)/uapi
++GENDIR := $(abspath ../../include/generated)
++GENHDR := $(GENDIR)/autoconf.h
++
++ifeq ($(O),)
++OUTPUT_DIR := $(CURDIR)/build
++else
++OUTPUT_DIR := $(O)/build
++endif # O
++OBJ_DIR := $(OUTPUT_DIR)/obj
++INCLUDE_DIR := $(OUTPUT_DIR)/include
++BPFOBJ_DIR := $(OBJ_DIR)/libbpf
++SCXOBJ_DIR := $(OBJ_DIR)/sched_ext
++BINDIR := $(OUTPUT_DIR)/bin
++BPFOBJ := $(BPFOBJ_DIR)/libbpf.a
++ifneq ($(CROSS_COMPILE),)
++HOST_BUILD_DIR := $(OBJ_DIR)/host
++HOST_OUTPUT_DIR := host-tools
++HOST_INCLUDE_DIR := $(HOST_OUTPUT_DIR)/include
++else
++HOST_BUILD_DIR := $(OBJ_DIR)
++HOST_OUTPUT_DIR := $(OUTPUT_DIR)
++HOST_INCLUDE_DIR := $(INCLUDE_DIR)
++endif
++HOST_BPFOBJ := $(HOST_BUILD_DIR)/libbpf/libbpf.a
++RESOLVE_BTFIDS := $(HOST_BUILD_DIR)/resolve_btfids/resolve_btfids
++DEFAULT_BPFTOOL := $(HOST_OUTPUT_DIR)/sbin/bpftool
++
++VMLINUX_BTF_PATHS ?= $(if $(O),$(O)/vmlinux) \
++ $(if $(KBUILD_OUTPUT),$(KBUILD_OUTPUT)/vmlinux) \
++ ../../vmlinux \
++ /sys/kernel/btf/vmlinux \
++ /boot/vmlinux-$(shell uname -r)
++VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS))))
++ifeq ($(VMLINUX_BTF),)
++$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)")
++endif
++
++BPFTOOL ?= $(DEFAULT_BPFTOOL)
++
++ifneq ($(wildcard $(GENHDR)),)
++ GENFLAGS := -DHAVE_GENHDR
++endif
++
++CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \
++ -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \
++ -I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include
++
++# Silence some warnings when compiled with clang
++ifneq ($(LLVM),)
++CFLAGS += -Wno-unused-command-line-argument
++endif
++
++LDFLAGS = -lelf -lz -lpthread
++
++IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \
++ grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')
++
++# Get Clang's default includes on this system, as opposed to those seen by
++# '-target bpf'. This fixes "missing" files on some architectures/distros,
++# such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc.
++#
++# Use '-idirafter': Don't interfere with include mechanics except where the
++# build would have failed anyways.
++define get_sys_includes
++$(shell $(1) -v -E - </dev/null 2>&1 \
++ | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \
++$(shell $(1) -dM -E - </dev/null | grep '__riscv_xlen ' | awk '{printf("-D__riscv_xlen=%d -D__BITS_PER_LONG=%d", $$3, $$3)}')
++endef
++
++BPF_CFLAGS = -g -D__TARGET_ARCH_$(SRCARCH) \
++ $(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian) \
++ -I$(CURDIR)/include -I$(CURDIR)/include/bpf-compat \
++ -I$(INCLUDE_DIR) -I$(APIDIR) \
++ -I../../include \
++ $(call get_sys_includes,$(CLANG)) \
++ -Wall -Wno-compare-distinct-pointer-types \
++ -O2 -mcpu=v3
++
++# sort removes libbpf duplicates when not cross-building
++MAKE_DIRS := $(sort $(OBJ_DIR)/libbpf $(HOST_BUILD_DIR)/libbpf \
++ $(HOST_BUILD_DIR)/bpftool $(HOST_BUILD_DIR)/resolve_btfids \
++ $(INCLUDE_DIR) $(SCXOBJ_DIR) $(BINDIR))
++
++$(MAKE_DIRS):
++ $(call msg,MKDIR,,$@)
++ $(Q)mkdir -p $@
++
++$(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \
++ $(APIDIR)/linux/bpf.h \
++ | $(OBJ_DIR)/libbpf
++ $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(OBJ_DIR)/libbpf/ \
++ EXTRA_CFLAGS='-g -O0 -fPIC' \
++ DESTDIR=$(OUTPUT_DIR) prefix= all install_headers
++
++$(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \
++ $(HOST_BPFOBJ) | $(HOST_BUILD_DIR)/bpftool
++ $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \
++ ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) \
++ EXTRA_CFLAGS='-g -O0' \
++ OUTPUT=$(HOST_BUILD_DIR)/bpftool/ \
++ LIBBPF_OUTPUT=$(HOST_BUILD_DIR)/libbpf/ \
++ LIBBPF_DESTDIR=$(HOST_OUTPUT_DIR)/ \
++ prefix= DESTDIR=$(HOST_OUTPUT_DIR)/ install-bin
++
++$(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR)
++ifeq ($(VMLINUX_H),)
++ $(call msg,GEN,,$@)
++ $(Q)$(BPFTOOL) btf dump file $(VMLINUX_BTF) format c > $@
++else
++ $(call msg,CP,,$@)
++ $(Q)cp "$(VMLINUX_H)" $@
++endif
++
++$(SCXOBJ_DIR)/%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h include/scx/*.h \
++ | $(BPFOBJ) $(SCXOBJ_DIR)
++ $(call msg,CLNG-BPF,,$(notdir $@))
++ $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@
++
++$(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BPFTOOL)
++ $(eval sched=$(notdir $@))
++ $(call msg,GEN-SKEL,,$(sched))
++ $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $<
++ $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o)
++ $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o)
++ $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o)
++ $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(subst .bpf.skel.h,,$(sched)) > $@
++ $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(subst .bpf.skel.h,,$(sched)) > $(@:.skel.h=.subskel.h)
++
++SCX_COMMON_DEPS := include/scx/common.h include/scx/user_exit_info.h | $(BINDIR)
++
++c-sched-targets = scx_simple scx_qmap scx_central
++
++$(addprefix $(BINDIR)/,$(c-sched-targets)): \
++ $(BINDIR)/%: \
++ $(filter-out %.bpf.c,%.c) \
++ $(INCLUDE_DIR)/%.bpf.skel.h \
++ $(SCX_COMMON_DEPS)
++ $(eval sched=$(notdir $@))
++ $(CC) $(CFLAGS) -c $(sched).c -o $(SCXOBJ_DIR)/$(sched).o
++ $(CC) -o $@ $(SCXOBJ_DIR)/$(sched).o $(HOST_BPFOBJ) $(LDFLAGS)
++
++$(c-sched-targets): %: $(BINDIR)/%
++
++install: all
++ $(Q)mkdir -p $(DESTDIR)/usr/local/bin/
++ $(Q)cp $(BINDIR)/* $(DESTDIR)/usr/local/bin/
++
++clean:
++ rm -rf $(OUTPUT_DIR) $(HOST_OUTPUT_DIR)
++ rm -f *.o *.bpf.o *.bpf.skel.h *.bpf.subskel.h
++ rm -f $(c-sched-targets)
++
++help:
++ @echo 'Building targets'
++ @echo '================'
++ @echo ''
++ @echo ' all - Compile all schedulers'
++ @echo ''
++ @echo 'Alternatively, you may compile individual schedulers:'
++ @echo ''
++ @printf ' %s\n' $(c-sched-targets)
++ @echo ''
++ @echo 'For any scheduler build target, you may specify an alternative'
++ @echo 'build output path with the O= environment variable. For example:'
++ @echo ''
++ @echo ' O=/tmp/sched_ext make all'
++ @echo ''
++ @echo 'will compile all schedulers, and emit the build artifacts to'
++ @echo '/tmp/sched_ext/build.'
++ @echo ''
++ @echo ''
++ @echo 'Installing targets'
++ @echo '=================='
++ @echo ''
++ @echo ' install - Compile and install all schedulers to /usr/bin.'
++ @echo ' You may specify the DESTDIR= environment variable'
++ @echo ' to indicate a prefix for /usr/bin. For example:'
++ @echo ''
++ @echo ' DESTDIR=/tmp/sched_ext make install'
++ @echo ''
++ @echo ' will build the schedulers in CWD/build, and'
++ @echo ' install the schedulers to /tmp/sched_ext/usr/bin.'
++ @echo ''
++ @echo ''
++ @echo 'Cleaning targets'
++ @echo '================'
++ @echo ''
++ @echo ' clean - Remove all generated files'
++
++all_targets: $(c-sched-targets)
++
++.PHONY: all all_targets $(c-sched-targets) clean help
++
++# delete failed targets
++.DELETE_ON_ERROR:
++
++# keep intermediate (.bpf.skel.h, .bpf.o, etc) targets
++.SECONDARY:
+diff --git a/tools/sched_ext/README.md b/tools/sched_ext/README.md
+new file mode 100644
+index 000000000000..8efe70cc4363
+--- /dev/null
++++ b/tools/sched_ext/README.md
+@@ -0,0 +1,258 @@
++SCHED_EXT EXAMPLE SCHEDULERS
++============================
++
++# Introduction
++
++This directory contains a number of example sched_ext schedulers. These
++schedulers are meant to provide examples of different types of schedulers
++that can be built using sched_ext, and illustrate how various features of
++sched_ext can be used.
++
++Some of the examples are performant, production-ready schedulers. That is, for
++the correct workload and with the correct tuning, they may be deployed in a
++production environment with acceptable or possibly even improved performance.
++Others are just examples that in practice, would not provide acceptable
++performance (though they could be improved to get there).
++
++This README will describe these example schedulers, including describing the
++types of workloads or scenarios they're designed to accommodate, and whether or
++not they're production ready. For more details on any of these schedulers,
++please see the header comment in their .bpf.c file.
++
++
++# Compiling the examples
++
++There are a few toolchain dependencies for compiling the example schedulers.
++
++## Toolchain dependencies
++
++1. clang >= 16.0.0
++
++The schedulers are BPF programs, and therefore must be compiled with clang. gcc
++is actively working on adding a BPF backend compiler as well, but are still
++missing some features such as BTF type tags which are necessary for using
++kptrs.
++
++2. pahole >= 1.25
++
++You may need pahole in order to generate BTF from DWARF.
++
++3. rust >= 1.70.0
++
++Rust schedulers uses features present in the rust toolchain >= 1.70.0. You
++should be able to use the stable build from rustup, but if that doesn't
++work, try using the rustup nightly build.
++
++There are other requirements as well, such as make, but these are the main /
++non-trivial ones.
++
++## Compiling the kernel
++
++In order to run a sched_ext scheduler, you'll have to run a kernel compiled
++with the patches in this repository, and with a minimum set of necessary
++Kconfig options:
++
++```
++CONFIG_BPF=y
++CONFIG_SCHED_CLASS_EXT=y
++CONFIG_BPF_SYSCALL=y
++CONFIG_BPF_JIT=y
++CONFIG_DEBUG_INFO_BTF=y
++```
++
++It's also recommended that you also include the following Kconfig options:
++
++```
++CONFIG_BPF_JIT_ALWAYS_ON=y
++CONFIG_BPF_JIT_DEFAULT_ON=y
++CONFIG_PAHOLE_HAS_SPLIT_BTF=y
++CONFIG_PAHOLE_HAS_BTF_TAG=y
++```
++
++There is a `Kconfig` file in this directory whose contents you can append to
++your local `.config` file, as long as there are no conflicts with any existing
++options in the file.
++
++## Getting a vmlinux.h file
++
++You may notice that most of the example schedulers include a "vmlinux.h" file.
++This is a large, auto-generated header file that contains all of the types
++defined in some vmlinux binary that was compiled with
++[BTF](https://docs.kernel.org/bpf/btf.html) (i.e. with the BTF-related Kconfig
++options specified above).
++
++The header file is created using `bpftool`, by passing it a vmlinux binary
++compiled with BTF as follows:
++
++```bash
++$ bpftool btf dump file /path/to/vmlinux format c > vmlinux.h
++```
++
++`bpftool` analyzes all of the BTF encodings in the binary, and produces a
++header file that can be included by BPF programs to access those types. For
++example, using vmlinux.h allows a scheduler to access fields defined directly
++in vmlinux as follows:
++
++```c
++#include "vmlinux.h"
++// vmlinux.h is also implicitly included by scx_common.bpf.h.
++#include "scx_common.bpf.h"
++
++/*
++ * vmlinux.h provides definitions for struct task_struct and
++ * struct scx_enable_args.
++ */
++void BPF_STRUCT_OPS(example_enable, struct task_struct *p,
++ struct scx_enable_args *args)
++{
++ bpf_printk("Task %s enabled in example scheduler", p->comm);
++}
++
++// vmlinux.h provides the definition for struct sched_ext_ops.
++SEC(".struct_ops.link")
++struct sched_ext_ops example_ops {
++ .enable = (void *)example_enable,
++ .name = "example",
++}
++```
++
++The scheduler build system will generate this vmlinux.h file as part of the
++scheduler build pipeline. It looks for a vmlinux file in the following
++dependency order:
++
++1. If the O= environment variable is defined, at `$O/vmlinux`
++2. If the KBUILD_OUTPUT= environment variable is defined, at
++ `$KBUILD_OUTPUT/vmlinux`
++3. At `../../vmlinux` (i.e. at the root of the kernel tree where you're
++ compiling the schedulers)
++3. `/sys/kernel/btf/vmlinux`
++4. `/boot/vmlinux-$(uname -r)`
++
++In other words, if you have compiled a kernel in your local repo, its vmlinux
++file will be used to generate vmlinux.h. Otherwise, it will be the vmlinux of
++the kernel you're currently running on. This means that if you're running on a
++kernel with sched_ext support, you may not need to compile a local kernel at
++all.
++
++### Aside on CO-RE
++
++One of the cooler features of BPF is that it supports
++[CO-RE](https://nakryiko.com/posts/bpf-core-reference-guide/) (Compile Once Run
++Everywhere). This feature allows you to reference fields inside of structs with
++types defined internal to the kernel, and not have to recompile if you load the
++BPF program on a different kernel with the field at a different offset. In our
++example above, we print out a task name with `p->comm`. CO-RE would perform
++relocations for that access when the program is loaded to ensure that it's
++referencing the correct offset for the currently running kernel.
++
++## Compiling the schedulers
++
++Once you have your toolchain setup, and a vmlinux that can be used to generate
++a full vmlinux.h file, you can compile the schedulers using `make`:
++
++```bash
++$ make -j($nproc)
++```
++
++# Example schedulers
++
++This directory contains the following example schedulers. These schedulers are
++for testing and demonstrating different aspects of sched_ext. While some may be
++useful in limited scenarios, they are not intended to be practical.
++
++For more scheduler implementations, tools and documentation, visit
++https://github.com/sched-ext/scx.
++
++## scx_simple
++
++A simple scheduler that provides an example of a minimal sched_ext scheduler.
++scx_simple can be run in either global weighted vtime mode, or FIFO mode.
++
++Though very simple, in limited scenarios, this scheduler can perform reasonably
++well on single-socket systems with a unified L3 cache.
++
++## scx_qmap
++
++Another simple, yet slightly more complex scheduler that provides an example of
++a basic weighted FIFO queuing policy. It also provides examples of some common
++useful BPF features, such as sleepable per-task storage allocation in the
++`ops.prep_enable()` callback, and using the `BPF_MAP_TYPE_QUEUE` map type to
++enqueue tasks. It also illustrates how core-sched support could be implemented.
++
++## scx_central
++
++A "central" scheduler where scheduling decisions are made from a single CPU.
++This scheduler illustrates how scheduling decisions can be dispatched from a
++single CPU, allowing other cores to run with infinite slices, without timer
++ticks, and without having to incur the overhead of making scheduling decisions.
++
++The approach demonstrated by this scheduler may be useful for any workload that
++benefits from minimizing scheduling overhead and timer ticks. An example of
++where this could be particularly useful is running VMs, where running with
++infinite slices and no timer ticks allows the VM to avoid unnecessary expensive
++vmexits.
++
++
++# Troubleshooting
++
++There are a number of common issues that you may run into when building the
++schedulers. We'll go over some of the common ones here.
++
++## Build Failures
++
++### Old version of clang
++
++```
++error: static assertion failed due to requirement 'SCX_DSQ_FLAG_BUILTIN': bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole
++ _Static_assert(SCX_DSQ_FLAG_BUILTIN,
++ ^~~~~~~~~~~~~~~~~~~~
++1 error generated.
++```
++
++This means you built the kernel or the schedulers with an older version of
++clang than what's supported (i.e. older than 16.0.0). To remediate this:
++
++1. `which clang` to make sure you're using a sufficiently new version of clang.
++
++2. `make fullclean` in the root path of the repository, and rebuild the kernel
++ and schedulers.
++
++3. Rebuild the kernel, and then your example schedulers.
++
++The schedulers are also cleaned if you invoke `make mrproper` in the root
++directory of the tree.
++
++### Stale kernel build / incomplete vmlinux.h file
++
++As described above, you'll need a `vmlinux.h` file that was generated from a
++vmlinux built with BTF, and with sched_ext support enabled. If you don't,
++you'll see errors such as the following which indicate that a type being
++referenced in a scheduler is unknown:
++
++```
++/path/to/sched_ext/tools/sched_ext/user_exit_info.h:25:23: note: forward declaration of 'struct scx_exit_info'
++
++const struct scx_exit_info *ei)
++
++^
++```
++
++In order to resolve this, please follow the steps above in
++[Getting a vmlinux.h file](#getting-a-vmlinuxh-file) in order to ensure your
++schedulers are using a vmlinux.h file that includes the requisite types.
++
++## Misc
++
++### llvm: [OFF]
++
++You may see the following output when building the schedulers:
++
++```
++Auto-detecting system features:
++... clang-bpf-co-re: [ on ]
++... llvm: [ OFF ]
++... libcap: [ on ]
++... libbfd: [ on ]
++```
++
++Seeing `llvm: [ OFF ]` here is not an issue. You can safely ignore.
+diff --git a/tools/sched_ext/include/bpf-compat/gnu/stubs.h b/tools/sched_ext/include/bpf-compat/gnu/stubs.h
+new file mode 100644
+index 000000000000..ad7d139ce907
+--- /dev/null
++++ b/tools/sched_ext/include/bpf-compat/gnu/stubs.h
+@@ -0,0 +1,11 @@
++/*
++ * Dummy gnu/stubs.h. clang can end up including /usr/include/gnu/stubs.h when
++ * compiling BPF files although its content doesn't play any role. The file in
++ * turn includes stubs-64.h or stubs-32.h depending on whether __x86_64__ is
++ * defined. When compiling a BPF source, __x86_64__ isn't set and thus
++ * stubs-32.h is selected. However, the file is not there if the system doesn't
++ * have 32bit glibc devel package installed leading to a build failure.
++ *
++ * The problem is worked around by making this file available in the include
++ * search paths before the system one when building BPF.
++ */
+diff --git a/tools/sched_ext/include/scx/common.bpf.h b/tools/sched_ext/include/scx/common.bpf.h
+new file mode 100644
+index 000000000000..20280df62857
+--- /dev/null
++++ b/tools/sched_ext/include/scx/common.bpf.h
+@@ -0,0 +1,401 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
++ */
++#ifndef __SCX_COMMON_BPF_H
++#define __SCX_COMMON_BPF_H
++
++#include "vmlinux.h"
++#include <bpf/bpf_helpers.h>
++#include <bpf/bpf_tracing.h>
++#include <asm-generic/errno.h>
++#include "user_exit_info.h"
++
++#define PF_WQ_WORKER 0x00000020 /* I'm a workqueue worker */
++#define PF_KTHREAD 0x00200000 /* I am a kernel thread */
++#define PF_EXITING 0x00000004
++#define CLOCK_MONOTONIC 1
++
++/*
++ * Earlier versions of clang/pahole lost upper 32bits in 64bit enums which can
++ * lead to really confusing misbehaviors. Let's trigger a build failure.
++ */
++static inline void ___vmlinux_h_sanity_check___(void)
++{
++ _Static_assert(SCX_DSQ_FLAG_BUILTIN,
++ "bpftool generated vmlinux.h is missing high bits for 64bit enums, upgrade clang and pahole");
++}
++
++s32 scx_bpf_create_dsq(u64 dsq_id, s32 node) __ksym;
++s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags, bool *is_idle) __ksym;
++void scx_bpf_dispatch(struct task_struct *p, u64 dsq_id, u64 slice, u64 enq_flags) __ksym;
++void scx_bpf_dispatch_vtime(struct task_struct *p, u64 dsq_id, u64 slice, u64 vtime, u64 enq_flags) __ksym;
++u32 scx_bpf_dispatch_nr_slots(void) __ksym;
++void scx_bpf_dispatch_cancel(void) __ksym;
++bool scx_bpf_consume(u64 dsq_id) __ksym;
++u32 scx_bpf_reenqueue_local(void) __ksym;
++void scx_bpf_kick_cpu(s32 cpu, u64 flags) __ksym;
++s32 scx_bpf_dsq_nr_queued(u64 dsq_id) __ksym;
++void scx_bpf_destroy_dsq(u64 dsq_id) __ksym;
++int bpf_iter_scx_dsq_new(struct bpf_iter_scx_dsq *it, u64 dsq_id, u64 flags) __ksym __weak;
++struct task_struct *bpf_iter_scx_dsq_next(struct bpf_iter_scx_dsq *it) __ksym __weak;
++void bpf_iter_scx_dsq_destroy(struct bpf_iter_scx_dsq *it) __ksym __weak;
++void scx_bpf_exit_bstr(s64 exit_code, char *fmt, unsigned long long *data, u32 data__sz) __ksym __weak;
++void scx_bpf_error_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym;
++void scx_bpf_dump_bstr(char *fmt, unsigned long long *data, u32 data_len) __ksym __weak;
++u32 scx_bpf_cpuperf_cap(s32 cpu) __ksym __weak;
++u32 scx_bpf_cpuperf_cur(s32 cpu) __ksym __weak;
++void scx_bpf_cpuperf_set(s32 cpu, u32 perf) __ksym __weak;
++u32 scx_bpf_nr_cpu_ids(void) __ksym __weak;
++const struct cpumask *scx_bpf_get_possible_cpumask(void) __ksym __weak;
++const struct cpumask *scx_bpf_get_online_cpumask(void) __ksym __weak;
++void scx_bpf_put_cpumask(const struct cpumask *cpumask) __ksym __weak;
++const struct cpumask *scx_bpf_get_idle_cpumask(void) __ksym;
++const struct cpumask *scx_bpf_get_idle_smtmask(void) __ksym;
++void scx_bpf_put_idle_cpumask(const struct cpumask *cpumask) __ksym;
++bool scx_bpf_test_and_clear_cpu_idle(s32 cpu) __ksym;
++s32 scx_bpf_pick_idle_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym;
++s32 scx_bpf_pick_any_cpu(const cpumask_t *cpus_allowed, u64 flags) __ksym;
++bool scx_bpf_task_running(const struct task_struct *p) __ksym;
++s32 scx_bpf_task_cpu(const struct task_struct *p) __ksym;
++struct rq *scx_bpf_cpu_rq(s32 cpu) __ksym;
++
++static inline __attribute__((format(printf, 1, 2)))
++void ___scx_bpf_bstr_format_checker(const char *fmt, ...) {}
++
++/*
++ * Helper macro for initializing the fmt and variadic argument inputs to both
++ * bstr exit kfuncs. Callers to this function should use ___fmt and ___param to
++ * refer to the initialized list of inputs to the bstr kfunc.
++ */
++#define scx_bpf_bstr_preamble(fmt, args...) \
++ static char ___fmt[] = fmt; \
++ /* \
++ * Note that __param[] must have at least one \
++ * element to keep the verifier happy. \
++ */ \
++ unsigned long long ___param[___bpf_narg(args) ?: 1] = {}; \
++ \
++ _Pragma("GCC diagnostic push") \
++ _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \
++ ___bpf_fill(___param, args); \
++ _Pragma("GCC diagnostic pop") \
++
++/*
++ * scx_bpf_exit() wraps the scx_bpf_exit_bstr() kfunc with variadic arguments
++ * instead of an array of u64. Using this macro will cause the scheduler to
++ * exit cleanly with the specified exit code being passed to user space.
++ */
++#define scx_bpf_exit(code, fmt, args...) \
++({ \
++ scx_bpf_bstr_preamble(fmt, args) \
++ scx_bpf_exit_bstr(code, ___fmt, ___param, sizeof(___param)); \
++ ___scx_bpf_bstr_format_checker(fmt, ##args); \
++})
++
++/*
++ * scx_bpf_error() wraps the scx_bpf_error_bstr() kfunc with variadic arguments
++ * instead of an array of u64. Invoking this macro will cause the scheduler to
++ * exit in an erroneous state, with diagnostic information being passed to the
++ * user.
++ */
++#define scx_bpf_error(fmt, args...) \
++({ \
++ scx_bpf_bstr_preamble(fmt, args) \
++ scx_bpf_error_bstr(___fmt, ___param, sizeof(___param)); \
++ ___scx_bpf_bstr_format_checker(fmt, ##args); \
++})
++
++/*
++ * scx_bpf_dump() wraps the scx_bpf_dump_bstr() kfunc with variadic arguments
++ * instead of an array of u64. To be used from ops.dump() and friends.
++ */
++#define scx_bpf_dump(fmt, args...) \
++({ \
++ scx_bpf_bstr_preamble(fmt, args) \
++ scx_bpf_dump_bstr(___fmt, ___param, sizeof(___param)); \
++ ___scx_bpf_bstr_format_checker(fmt, ##args); \
++})
++
++#define BPF_STRUCT_OPS(name, args...) \
++SEC("struct_ops/"#name) \
++BPF_PROG(name, ##args)
++
++#define BPF_STRUCT_OPS_SLEEPABLE(name, args...) \
++SEC("struct_ops.s/"#name) \
++BPF_PROG(name, ##args)
++
++/**
++ * RESIZABLE_ARRAY - Generates annotations for an array that may be resized
++ * @elfsec: the data section of the BPF program in which to place the array
++ * @arr: the name of the array
++ *
++ * libbpf has an API for setting map value sizes. Since data sections (i.e.
++ * bss, data, rodata) themselves are maps, a data section can be resized. If
++ * a data section has an array as its last element, the BTF info for that
++ * array will be adjusted so that length of the array is extended to meet the
++ * new length of the data section. This macro annotates an array to have an
++ * element count of one with the assumption that this array can be resized
++ * within the userspace program. It also annotates the section specifier so
++ * this array exists in a custom sub data section which can be resized
++ * independently.
++ *
++ * See RESIZE_ARRAY() for the userspace convenience macro for resizing an
++ * array declared with RESIZABLE_ARRAY().
++ */
++#define RESIZABLE_ARRAY(elfsec, arr) arr[1] SEC("."#elfsec"."#arr)
++
++/**
++ * MEMBER_VPTR - Obtain the verified pointer to a struct or array member
++ * @base: struct or array to index
++ * @member: dereferenced member (e.g. .field, [idx0][idx1], .field[idx0] ...)
++ *
++ * The verifier often gets confused by the instruction sequence the compiler
++ * generates for indexing struct fields or arrays. This macro forces the
++ * compiler to generate a code sequence which first calculates the byte offset,
++ * checks it against the struct or array size and add that byte offset to
++ * generate the pointer to the member to help the verifier.
++ *
++ * Ideally, we want to abort if the calculated offset is out-of-bounds. However,
++ * BPF currently doesn't support abort, so evaluate to %NULL instead. The caller
++ * must check for %NULL and take appropriate action to appease the verifier. To
++ * avoid confusing the verifier, it's best to check for %NULL and dereference
++ * immediately.
++ *
++ * vptr = MEMBER_VPTR(my_array, [i][j]);
++ * if (!vptr)
++ * return error;
++ * *vptr = new_value;
++ *
++ * sizeof(@base) should encompass the memory area to be accessed and thus can't
++ * be a pointer to the area. Use `MEMBER_VPTR(*ptr, .member)` instead of
++ * `MEMBER_VPTR(ptr, ->member)`.
++ */
++#define MEMBER_VPTR(base, member) (typeof((base) member) *) \
++({ \
++ u64 __base = (u64)&(base); \
++ u64 __addr = (u64)&((base) member) - __base; \
++ _Static_assert(sizeof(base) >= sizeof((base) member), \
++ "@base is smaller than @member, is @base a pointer?"); \
++ asm volatile ( \
++ "if %0 <= %[max] goto +2\n" \
++ "%0 = 0\n" \
++ "goto +1\n" \
++ "%0 += %1\n" \
++ : "+r"(__addr) \
++ : "r"(__base), \
++ [max]"i"(sizeof(base) - sizeof((base) member))); \
++ __addr; \
++})
++
++/**
++ * ARRAY_ELEM_PTR - Obtain the verified pointer to an array element
++ * @arr: array to index into
++ * @i: array index
++ * @n: number of elements in array
++ *
++ * Similar to MEMBER_VPTR() but is intended for use with arrays where the
++ * element count needs to be explicit.
++ * It can be used in cases where a global array is defined with an initial
++ * size but is intended to be be resized before loading the BPF program.
++ * Without this version of the macro, MEMBER_VPTR() will use the compile time
++ * size of the array to compute the max, which will result in rejection by
++ * the verifier.
++ */
++#define ARRAY_ELEM_PTR(arr, i, n) (typeof(arr[i]) *) \
++({ \
++ u64 __base = (u64)arr; \
++ u64 __addr = (u64)&(arr[i]) - __base; \
++ asm volatile ( \
++ "if %0 <= %[max] goto +2\n" \
++ "%0 = 0\n" \
++ "goto +1\n" \
++ "%0 += %1\n" \
++ : "+r"(__addr) \
++ : "r"(__base), \
++ [max]"r"(sizeof(arr[0]) * ((n) - 1))); \
++ __addr; \
++})
++
++
++/*
++ * BPF declarations and helpers
++ */
++
++/* list and rbtree */
++#define __contains(name, node) __attribute__((btf_decl_tag("contains:" #name ":" #node)))
++#define private(name) SEC(".data." #name) __hidden __attribute__((aligned(8)))
++
++void *bpf_obj_new_impl(__u64 local_type_id, void *meta) __ksym;
++void bpf_obj_drop_impl(void *kptr, void *meta) __ksym;
++
++#define bpf_obj_new(type) ((type *)bpf_obj_new_impl(bpf_core_type_id_local(type), NULL))
++#define bpf_obj_drop(kptr) bpf_obj_drop_impl(kptr, NULL)
++
++void bpf_list_push_front(struct bpf_list_head *head, struct bpf_list_node *node) __ksym;
++void bpf_list_push_back(struct bpf_list_head *head, struct bpf_list_node *node) __ksym;
++struct bpf_list_node *bpf_list_pop_front(struct bpf_list_head *head) __ksym;
++struct bpf_list_node *bpf_list_pop_back(struct bpf_list_head *head) __ksym;
++struct bpf_rb_node *bpf_rbtree_remove(struct bpf_rb_root *root,
++ struct bpf_rb_node *node) __ksym;
++int bpf_rbtree_add_impl(struct bpf_rb_root *root, struct bpf_rb_node *node,
++ bool (less)(struct bpf_rb_node *a, const struct bpf_rb_node *b),
++ void *meta, __u64 off) __ksym;
++#define bpf_rbtree_add(head, node, less) bpf_rbtree_add_impl(head, node, less, NULL, 0)
++
++struct bpf_rb_node *bpf_rbtree_first(struct bpf_rb_root *root) __ksym;
++
++void *bpf_refcount_acquire_impl(void *kptr, void *meta) __ksym;
++#define bpf_refcount_acquire(kptr) bpf_refcount_acquire_impl(kptr, NULL)
++
++/* task */
++struct task_struct *bpf_task_from_pid(s32 pid) __ksym;
++struct task_struct *bpf_task_acquire(struct task_struct *p) __ksym;
++void bpf_task_release(struct task_struct *p) __ksym;
++
++/* cgroup */
++struct cgroup *bpf_cgroup_ancestor(struct cgroup *cgrp, int level) __ksym;
++void bpf_cgroup_release(struct cgroup *cgrp) __ksym;
++struct cgroup *bpf_cgroup_from_id(u64 cgid) __ksym;
++
++/* css iteration */
++struct bpf_iter_css;
++struct cgroup_subsys_state;
++extern int bpf_iter_css_new(struct bpf_iter_css *it,
++ struct cgroup_subsys_state *start,
++ unsigned int flags) __weak __ksym;
++extern struct cgroup_subsys_state *
++bpf_iter_css_next(struct bpf_iter_css *it) __weak __ksym;
++extern void bpf_iter_css_destroy(struct bpf_iter_css *it) __weak __ksym;
++
++/* cpumask */
++struct bpf_cpumask *bpf_cpumask_create(void) __ksym;
++struct bpf_cpumask *bpf_cpumask_acquire(struct bpf_cpumask *cpumask) __ksym;
++void bpf_cpumask_release(struct bpf_cpumask *cpumask) __ksym;
++u32 bpf_cpumask_first(const struct cpumask *cpumask) __ksym;
++u32 bpf_cpumask_first_zero(const struct cpumask *cpumask) __ksym;
++void bpf_cpumask_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
++void bpf_cpumask_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
++bool bpf_cpumask_test_cpu(u32 cpu, const struct cpumask *cpumask) __ksym;
++bool bpf_cpumask_test_and_set_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
++bool bpf_cpumask_test_and_clear_cpu(u32 cpu, struct bpf_cpumask *cpumask) __ksym;
++void bpf_cpumask_setall(struct bpf_cpumask *cpumask) __ksym;
++void bpf_cpumask_clear(struct bpf_cpumask *cpumask) __ksym;
++bool bpf_cpumask_and(struct bpf_cpumask *dst, const struct cpumask *src1,
++ const struct cpumask *src2) __ksym;
++void bpf_cpumask_or(struct bpf_cpumask *dst, const struct cpumask *src1,
++ const struct cpumask *src2) __ksym;
++void bpf_cpumask_xor(struct bpf_cpumask *dst, const struct cpumask *src1,
++ const struct cpumask *src2) __ksym;
++bool bpf_cpumask_equal(const struct cpumask *src1, const struct cpumask *src2) __ksym;
++bool bpf_cpumask_intersects(const struct cpumask *src1, const struct cpumask *src2) __ksym;
++bool bpf_cpumask_subset(const struct cpumask *src1, const struct cpumask *src2) __ksym;
++bool bpf_cpumask_empty(const struct cpumask *cpumask) __ksym;
++bool bpf_cpumask_full(const struct cpumask *cpumask) __ksym;
++void bpf_cpumask_copy(struct bpf_cpumask *dst, const struct cpumask *src) __ksym;
++u32 bpf_cpumask_any_distribute(const struct cpumask *cpumask) __ksym;
++u32 bpf_cpumask_any_and_distribute(const struct cpumask *src1,
++ const struct cpumask *src2) __ksym;
++
++/* rcu */
++void bpf_rcu_read_lock(void) __ksym;
++void bpf_rcu_read_unlock(void) __ksym;
++
++
++/*
++ * Other helpers
++ */
++
++/* useful compiler attributes */
++#define likely(x) __builtin_expect(!!(x), 1)
++#define unlikely(x) __builtin_expect(!!(x), 0)
++#define __maybe_unused __attribute__((__unused__))
++
++/*
++ * READ/WRITE_ONCE() are from kernel (include/asm-generic/rwonce.h). They
++ * prevent compiler from caching, redoing or reordering reads or writes.
++ */
++typedef __u8 __attribute__((__may_alias__)) __u8_alias_t;
++typedef __u16 __attribute__((__may_alias__)) __u16_alias_t;
++typedef __u32 __attribute__((__may_alias__)) __u32_alias_t;
++typedef __u64 __attribute__((__may_alias__)) __u64_alias_t;
++
++static __always_inline void __read_once_size(const volatile void *p, void *res, int size)
++{
++ switch (size) {
++ case 1: *(__u8_alias_t *) res = *(volatile __u8_alias_t *) p; break;
++ case 2: *(__u16_alias_t *) res = *(volatile __u16_alias_t *) p; break;
++ case 4: *(__u32_alias_t *) res = *(volatile __u32_alias_t *) p; break;
++ case 8: *(__u64_alias_t *) res = *(volatile __u64_alias_t *) p; break;
++ default:
++ barrier();
++ __builtin_memcpy((void *)res, (const void *)p, size);
++ barrier();
++ }
++}
++
++static __always_inline void __write_once_size(volatile void *p, void *res, int size)
++{
++ switch (size) {
++ case 1: *(volatile __u8_alias_t *) p = *(__u8_alias_t *) res; break;
++ case 2: *(volatile __u16_alias_t *) p = *(__u16_alias_t *) res; break;
++ case 4: *(volatile __u32_alias_t *) p = *(__u32_alias_t *) res; break;
++ case 8: *(volatile __u64_alias_t *) p = *(__u64_alias_t *) res; break;
++ default:
++ barrier();
++ __builtin_memcpy((void *)p, (const void *)res, size);
++ barrier();
++ }
++}
++
++#define READ_ONCE(x) \
++({ \
++ union { typeof(x) __val; char __c[1]; } __u = \
++ { .__c = { 0 } }; \
++ __read_once_size(&(x), __u.__c, sizeof(x)); \
++ __u.__val; \
++})
++
++#define WRITE_ONCE(x, val) \
++({ \
++ union { typeof(x) __val; char __c[1]; } __u = \
++ { .__val = (val) }; \
++ __write_once_size(&(x), __u.__c, sizeof(x)); \
++ __u.__val; \
++})
++
++/*
++ * log2_u32 - Compute the base 2 logarithm of a 32-bit exponential value.
++ * @v: The value for which we're computing the base 2 logarithm.
++ */
++static inline u32 log2_u32(u32 v)
++{
++ u32 r;
++ u32 shift;
++
++ r = (v > 0xFFFF) << 4; v >>= r;
++ shift = (v > 0xFF) << 3; v >>= shift; r |= shift;
++ shift = (v > 0xF) << 2; v >>= shift; r |= shift;
++ shift = (v > 0x3) << 1; v >>= shift; r |= shift;
++ r |= (v >> 1);
++ return r;
++}
++
++/*
++ * log2_u64 - Compute the base 2 logarithm of a 64-bit exponential value.
++ * @v: The value for which we're computing the base 2 logarithm.
++ */
++static inline u32 log2_u64(u64 v)
++{
++ u32 hi = v >> 32;
++ if (hi)
++ return log2_u32(hi) + 32 + 1;
++ else
++ return log2_u32(v) + 1;
++}
++
++#include "compat.bpf.h"
++
++#endif /* __SCX_COMMON_BPF_H */
+diff --git a/tools/sched_ext/include/scx/common.h b/tools/sched_ext/include/scx/common.h
+new file mode 100644
+index 000000000000..5b0f90152152
+--- /dev/null
++++ b/tools/sched_ext/include/scx/common.h
+@@ -0,0 +1,75 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ */
++#ifndef __SCHED_EXT_COMMON_H
++#define __SCHED_EXT_COMMON_H
++
++#ifdef __KERNEL__
++#error "Should not be included by BPF programs"
++#endif
++
++#include <stdarg.h>
++#include <stdio.h>
++#include <stdlib.h>
++#include <stdint.h>
++#include <errno.h>
++
++typedef uint8_t u8;
++typedef uint16_t u16;
++typedef uint32_t u32;
++typedef uint64_t u64;
++typedef int8_t s8;
++typedef int16_t s16;
++typedef int32_t s32;
++typedef int64_t s64;
++
++#define SCX_BUG(__fmt, ...) \
++ do { \
++ fprintf(stderr, "[SCX_BUG] %s:%d", __FILE__, __LINE__); \
++ if (errno) \
++ fprintf(stderr, " (%s)\n", strerror(errno)); \
++ else \
++ fprintf(stderr, "\n"); \
++ fprintf(stderr, __fmt __VA_OPT__(,) __VA_ARGS__); \
++ fprintf(stderr, "\n"); \
++ \
++ exit(EXIT_FAILURE); \
++ } while (0)
++
++#define SCX_BUG_ON(__cond, __fmt, ...) \
++ do { \
++ if (__cond) \
++ SCX_BUG((__fmt) __VA_OPT__(,) __VA_ARGS__); \
++ } while (0)
++
++/**
++ * RESIZE_ARRAY - Convenience macro for resizing a BPF array
++ * @__skel: the skeleton containing the array
++ * @elfsec: the data section of the BPF program in which the array exists
++ * @arr: the name of the array
++ * @n: the desired array element count
++ *
++ * For BPF arrays declared with RESIZABLE_ARRAY(), this macro performs two
++ * operations. It resizes the map which corresponds to the custom data
++ * section that contains the target array. As a side effect, the BTF info for
++ * the array is adjusted so that the array length is sized to cover the new
++ * data section size. The second operation is reassigning the skeleton pointer
++ * for that custom data section so that it points to the newly memory mapped
++ * region.
++ */
++#define RESIZE_ARRAY(__skel, elfsec, arr, n) \
++ do { \
++ size_t __sz; \
++ bpf_map__set_value_size((__skel)->maps.elfsec##_##arr, \
++ sizeof((__skel)->elfsec##_##arr->arr[0]) * (n)); \
++ (__skel)->elfsec##_##arr = \
++ bpf_map__initial_value((__skel)->maps.elfsec##_##arr, &__sz); \
++ } while (0)
++
++#include "user_exit_info.h"
++#include "compat.h"
++
++#endif /* __SCHED_EXT_COMMON_H */
+diff --git a/tools/sched_ext/include/scx/compat.bpf.h b/tools/sched_ext/include/scx/compat.bpf.h
+new file mode 100644
+index 000000000000..3d2fe1208900
+--- /dev/null
++++ b/tools/sched_ext/include/scx/compat.bpf.h
+@@ -0,0 +1,28 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#ifndef __SCX_COMPAT_BPF_H
++#define __SCX_COMPAT_BPF_H
++
++#define __COMPAT_ENUM_OR_ZERO(__type, __ent) \
++({ \
++ __type __ret = 0; \
++ if (bpf_core_enum_value_exists(__type, __ent)) \
++ __ret = __ent; \
++ __ret; \
++})
++
++/*
++ * Define sched_ext_ops. This may be expanded to define multiple variants for
++ * backward compatibility. See compat.h::SCX_OPS_LOAD/ATTACH().
++ */
++#define SCX_OPS_DEFINE(__name, ...) \
++ SEC(".struct_ops.link") \
++ struct sched_ext_ops __name = { \
++ __VA_ARGS__, \
++ };
++
++#endif /* __SCX_COMPAT_BPF_H */
+diff --git a/tools/sched_ext/include/scx/compat.h b/tools/sched_ext/include/scx/compat.h
+new file mode 100644
+index 000000000000..1bf8eddf20c2
+--- /dev/null
++++ b/tools/sched_ext/include/scx/compat.h
+@@ -0,0 +1,187 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#ifndef __SCX_COMPAT_H
++#define __SCX_COMPAT_H
++
++#include <bpf/btf.h>
++#include <fcntl.h>
++#include <stdlib.h>
++#include <unistd.h>
++
++struct btf *__COMPAT_vmlinux_btf __attribute__((weak));
++
++static inline void __COMPAT_load_vmlinux_btf(void)
++{
++ if (!__COMPAT_vmlinux_btf) {
++ __COMPAT_vmlinux_btf = btf__load_vmlinux_btf();
++ SCX_BUG_ON(!__COMPAT_vmlinux_btf, "btf__load_vmlinux_btf()");
++ }
++}
++
++static inline bool __COMPAT_read_enum(const char *type, const char *name, u64 *v)
++{
++ const struct btf_type *t;
++ const char *n;
++ s32 tid;
++ int i;
++
++ __COMPAT_load_vmlinux_btf();
++
++ tid = btf__find_by_name(__COMPAT_vmlinux_btf, type);
++ if (tid < 0)
++ return false;
++
++ t = btf__type_by_id(__COMPAT_vmlinux_btf, tid);
++ SCX_BUG_ON(!t, "btf__type_by_id(%d)", tid);
++
++ if (btf_is_enum(t)) {
++ struct btf_enum *e = btf_enum(t);
++
++ for (i = 0; i < BTF_INFO_VLEN(t->info); i++) {
++ n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off);
++ SCX_BUG_ON(!n, "btf__name_by_offset()");
++ if (!strcmp(n, name)) {
++ *v = e[i].val;
++ return true;
++ }
++ }
++ } else if (btf_is_enum64(t)) {
++ struct btf_enum64 *e = btf_enum64(t);
++
++ for (i = 0; i < BTF_INFO_VLEN(t->info); i++) {
++ n = btf__name_by_offset(__COMPAT_vmlinux_btf, e[i].name_off);
++ SCX_BUG_ON(!n, "btf__name_by_offset()");
++ if (!strcmp(n, name)) {
++ *v = btf_enum64_value(&e[i]);
++ return true;
++ }
++ }
++ }
++
++ return false;
++}
++
++#define __COMPAT_ENUM_OR_ZERO(__type, __ent) \
++({ \
++ u64 __val = 0; \
++ __COMPAT_read_enum(__type, __ent, &__val); \
++ __val; \
++})
++
++static inline bool __COMPAT_has_ksym(const char *ksym)
++{
++ __COMPAT_load_vmlinux_btf();
++ return btf__find_by_name(__COMPAT_vmlinux_btf, ksym) >= 0;
++}
++
++static inline bool __COMPAT_struct_has_field(const char *type, const char *field)
++{
++ const struct btf_type *t;
++ const struct btf_member *m;
++ const char *n;
++ s32 tid;
++ int i;
++
++ __COMPAT_load_vmlinux_btf();
++ tid = btf__find_by_name_kind(__COMPAT_vmlinux_btf, type, BTF_KIND_STRUCT);
++ if (tid < 0)
++ return false;
++
++ t = btf__type_by_id(__COMPAT_vmlinux_btf, tid);
++ SCX_BUG_ON(!t, "btf__type_by_id(%d)", tid);
++
++ m = btf_members(t);
++
++ for (i = 0; i < BTF_INFO_VLEN(t->info); i++) {
++ n = btf__name_by_offset(__COMPAT_vmlinux_btf, m[i].name_off);
++ SCX_BUG_ON(!n, "btf__name_by_offset()");
++ if (!strcmp(n, field))
++ return true;
++ }
++
++ return false;
++}
++
++#define SCX_OPS_SWITCH_PARTIAL \
++ __COMPAT_ENUM_OR_ZERO("scx_ops_flags", "SCX_OPS_SWITCH_PARTIAL")
++
++static inline long scx_hotplug_seq(void)
++{
++ int fd;
++ char buf[32];
++ ssize_t len;
++ long val;
++
++ fd = open("/sys/kernel/sched_ext/hotplug_seq", O_RDONLY);
++ if (fd < 0)
++ return -ENOENT;
++
++ len = read(fd, buf, sizeof(buf) - 1);
++ SCX_BUG_ON(len <= 0, "read failed (%ld)", len);
++ buf[len] = 0;
++ close(fd);
++
++ val = strtoul(buf, NULL, 10);
++ SCX_BUG_ON(val < 0, "invalid num hotplug events: %lu", val);
++
++ return val;
++}
++
++/*
++ * struct sched_ext_ops can change over time. If compat.bpf.h::SCX_OPS_DEFINE()
++ * is used to define ops and compat.h::SCX_OPS_LOAD/ATTACH() are used to load
++ * and attach it, backward compatibility is automatically maintained where
++ * reasonable.
++ *
++ * ec7e3b0463e1 ("implement-ops") in https://github.com/sched-ext/sched_ext is
++ * the current minimum required kernel version.
++ */
++#define SCX_OPS_OPEN(__ops_name, __scx_name) ({ \
++ struct __scx_name *__skel; \
++ \
++ SCX_BUG_ON(!__COMPAT_struct_has_field("sched_ext_ops", "dump"), \
++ "sched_ext_ops.dump() missing, kernel too old?"); \
++ \
++ __skel = __scx_name##__open(); \
++ SCX_BUG_ON(!__skel, "Could not open " #__scx_name); \
++ __skel->struct_ops.__ops_name->hotplug_seq = scx_hotplug_seq(); \
++ __skel; \
++})
++
++#define SCX_OPS_LOAD(__skel, __ops_name, __scx_name, __uei_name) ({ \
++ UEI_SET_SIZE(__skel, __ops_name, __uei_name); \
++ SCX_BUG_ON(__scx_name##__load((__skel)), "Failed to load skel"); \
++})
++
++/*
++ * New versions of bpftool now emit additional link placeholders for BPF maps,
++ * and set up BPF skeleton in such a way that libbpf will auto-attach BPF maps
++ * automatically, assumming libbpf is recent enough (v1.5+). Old libbpf will do
++ * nothing with those links and won't attempt to auto-attach maps.
++ *
++ * To maintain compatibility with older libbpf while avoiding trying to attach
++ * twice, disable the autoattach feature on newer libbpf.
++ */
++/* BACKPORT - bpf_mpa__set_autoattach() not available yet, commented out */
++/*#if LIBBPF_MAJOR_VERSION > 1 || \
++ (LIBBPF_MAJOR_VERSION == 1 && LIBBPF_MINOR_VERSION >= 5)
++#define __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) \
++ bpf_map__set_autoattach((__skel)->maps.__ops_name, false)
++#else*/
++#define __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name) do {} while (0)
++/*#endif*/
++
++#define SCX_OPS_ATTACH(__skel, __ops_name, __scx_name) ({ \
++ struct bpf_link *__link; \
++ __SCX_OPS_DISABLE_AUTOATTACH(__skel, __ops_name); \
++ SCX_BUG_ON(__scx_name##__attach((__skel)), "Failed to attach skel"); \
++ __link = bpf_map__attach_struct_ops((__skel)->maps.__ops_name); \
++ SCX_BUG_ON(!__link, "Failed to attach struct_ops"); \
++ __link; \
++})
++
++#endif /* __SCX_COMPAT_H */
+diff --git a/tools/sched_ext/include/scx/user_exit_info.h b/tools/sched_ext/include/scx/user_exit_info.h
+new file mode 100644
+index 000000000000..891693ee604e
+--- /dev/null
++++ b/tools/sched_ext/include/scx/user_exit_info.h
+@@ -0,0 +1,111 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Define struct user_exit_info which is shared between BPF and userspace parts
++ * to communicate exit status and other information.
++ *
++ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
++ */
++#ifndef __USER_EXIT_INFO_H
++#define __USER_EXIT_INFO_H
++
++enum uei_sizes {
++ UEI_REASON_LEN = 128,
++ UEI_MSG_LEN = 1024,
++ UEI_DUMP_DFL_LEN = 32768,
++};
++
++struct user_exit_info {
++ int kind;
++ s64 exit_code;
++ char reason[UEI_REASON_LEN];
++ char msg[UEI_MSG_LEN];
++};
++
++#ifdef __bpf__
++
++#include "vmlinux.h"
++#include <bpf/bpf_core_read.h>
++
++#define UEI_DEFINE(__name) \
++ char RESIZABLE_ARRAY(data, __name##_dump); \
++ const volatile u32 __name##_dump_len; \
++ struct user_exit_info __name SEC(".data")
++
++#define UEI_RECORD(__uei_name, __ei) ({ \
++ bpf_probe_read_kernel_str(__uei_name.reason, \
++ sizeof(__uei_name.reason), (__ei)->reason); \
++ bpf_probe_read_kernel_str(__uei_name.msg, \
++ sizeof(__uei_name.msg), (__ei)->msg); \
++ bpf_probe_read_kernel_str(__uei_name##_dump, \
++ __uei_name##_dump_len, (__ei)->dump); \
++ if (bpf_core_field_exists((__ei)->exit_code)) \
++ __uei_name.exit_code = (__ei)->exit_code; \
++ /* use __sync to force memory barrier */ \
++ __sync_val_compare_and_swap(&__uei_name.kind, __uei_name.kind, \
++ (__ei)->kind); \
++})
++
++#else /* !__bpf__ */
++
++#include <stdio.h>
++#include <stdbool.h>
++
++/* no need to call the following explicitly if SCX_OPS_LOAD() is used */
++#define UEI_SET_SIZE(__skel, __ops_name, __uei_name) ({ \
++ u32 __len = (__skel)->struct_ops.__ops_name->exit_dump_len ?: UEI_DUMP_DFL_LEN; \
++ (__skel)->rodata->__uei_name##_dump_len = __len; \
++ RESIZE_ARRAY((__skel), data, __uei_name##_dump, __len); \
++})
++
++#define UEI_EXITED(__skel, __uei_name) ({ \
++ /* use __sync to force memory barrier */ \
++ __sync_val_compare_and_swap(&(__skel)->data->__uei_name.kind, -1, -1); \
++})
++
++#define UEI_REPORT(__skel, __uei_name) ({ \
++ struct user_exit_info *__uei = &(__skel)->data->__uei_name; \
++ char *__uei_dump = (__skel)->data_##__uei_name##_dump->__uei_name##_dump; \
++ if (__uei_dump[0] != '\0') { \
++ fputs("\nDEBUG DUMP\n", stderr); \
++ fputs("================================================================================\n\n", stderr); \
++ fputs(__uei_dump, stderr); \
++ fputs("\n================================================================================\n\n", stderr); \
++ } \
++ fprintf(stderr, "EXIT: %s", __uei->reason); \
++ if (__uei->msg[0] != '\0') \
++ fprintf(stderr, " (%s)", __uei->msg); \
++ fputs("\n", stderr); \
++ __uei->exit_code; \
++})
++
++/*
++ * We can't import vmlinux.h while compiling user C code. Let's duplicate
++ * scx_exit_code definition.
++ */
++enum scx_exit_code {
++ /* Reasons */
++ SCX_ECODE_RSN_HOTPLUG = 1LLU << 32,
++
++ /* Actions */
++ SCX_ECODE_ACT_RESTART = 1LLU << 48,
++};
++
++enum uei_ecode_mask {
++ UEI_ECODE_USER_MASK = ((1LLU << 32) - 1),
++ UEI_ECODE_SYS_RSN_MASK = ((1LLU << 16) - 1) << 32,
++ UEI_ECODE_SYS_ACT_MASK = ((1LLU << 16) - 1) << 48,
++};
++
++/*
++ * These macro interpret the ecode returned from UEI_REPORT().
++ */
++#define UEI_ECODE_USER(__ecode) ((__ecode) & UEI_ECODE_USER_MASK)
++#define UEI_ECODE_SYS_RSN(__ecode) ((__ecode) & UEI_ECODE_SYS_RSN_MASK)
++#define UEI_ECODE_SYS_ACT(__ecode) ((__ecode) & UEI_ECODE_SYS_ACT_MASK)
++
++#define UEI_ECODE_RESTART(__ecode) (UEI_ECODE_SYS_ACT((__ecode)) == SCX_ECODE_ACT_RESTART)
++
++#endif /* __bpf__ */
++#endif /* __USER_EXIT_INFO_H */
+diff --git a/tools/sched_ext/scx_central.bpf.c b/tools/sched_ext/scx_central.bpf.c
+new file mode 100644
+index 000000000000..1d8fd570eaa7
+--- /dev/null
++++ b/tools/sched_ext/scx_central.bpf.c
+@@ -0,0 +1,361 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A central FIFO sched_ext scheduler which demonstrates the followings:
++ *
++ * a. Making all scheduling decisions from one CPU:
++ *
++ * The central CPU is the only one making scheduling decisions. All other
++ * CPUs kick the central CPU when they run out of tasks to run.
++ *
++ * There is one global BPF queue and the central CPU schedules all CPUs by
++ * dispatching from the global queue to each CPU's local dsq from dispatch().
++ * This isn't the most straightforward. e.g. It'd be easier to bounce
++ * through per-CPU BPF queues. The current design is chosen to maximally
++ * utilize and verify various SCX mechanisms such as LOCAL_ON dispatching.
++ *
++ * b. Tickless operation
++ *
++ * All tasks are dispatched with the infinite slice which allows stopping the
++ * ticks on CONFIG_NO_HZ_FULL kernels running with the proper nohz_full
++ * parameter. The tickless operation can be observed through
++ * /proc/interrupts.
++ *
++ * Periodic switching is enforced by a periodic timer checking all CPUs and
++ * preempting them as necessary. Unfortunately, BPF timer currently doesn't
++ * have a way to pin to a specific CPU, so the periodic timer isn't pinned to
++ * the central CPU.
++ *
++ * c. Preemption
++ *
++ * Kthreads are unconditionally queued to the head of a matching local dsq
++ * and dispatched with SCX_DSQ_PREEMPT. This ensures that a kthread is always
++ * prioritized over user threads, which is required for ensuring forward
++ * progress as e.g. the periodic timer may run on a ksoftirqd and if the
++ * ksoftirqd gets starved by a user thread, there may not be anything else to
++ * vacate that user thread.
++ *
++ * SCX_KICK_PREEMPT is used to trigger scheduling and CPUs to move to the
++ * next tasks.
++ *
++ * This scheduler is designed to maximize usage of various SCX mechanisms. A
++ * more practical implementation would likely put the scheduling loop outside
++ * the central CPU's dispatch() path and add some form of priority mechanism.
++ *
++ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
++ */
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++enum {
++ FALLBACK_DSQ_ID = 0,
++ MS_TO_NS = 1000LLU * 1000,
++ TIMER_INTERVAL_NS = 1 * MS_TO_NS,
++};
++
++const volatile s32 central_cpu;
++const volatile u32 nr_cpu_ids = 1; /* !0 for veristat, set during init */
++const volatile u64 slice_ns = SCX_SLICE_DFL;
++
++bool timer_pinned = true;
++u64 nr_total, nr_locals, nr_queued, nr_lost_pids;
++u64 nr_timers, nr_dispatches, nr_mismatches, nr_retries;
++u64 nr_overflows;
++
++UEI_DEFINE(uei);
++
++struct {
++ __uint(type, BPF_MAP_TYPE_QUEUE);
++ __uint(max_entries, 4096);
++ __type(value, s32);
++} central_q SEC(".maps");
++
++/* can't use percpu map due to bad lookups */
++bool RESIZABLE_ARRAY(data, cpu_gimme_task);
++u64 RESIZABLE_ARRAY(data, cpu_started_at);
++
++struct central_timer {
++ struct bpf_timer timer;
++};
++
++struct {
++ __uint(type, BPF_MAP_TYPE_ARRAY);
++ __uint(max_entries, 1);
++ __type(key, u32);
++ __type(value, struct central_timer);
++} central_timer SEC(".maps");
++
++static bool vtime_before(u64 a, u64 b)
++{
++ return (s64)(a - b) < 0;
++}
++
++s32 BPF_STRUCT_OPS(central_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ /*
++ * Steer wakeups to the central CPU as much as possible to avoid
++ * disturbing other CPUs. It's safe to blindly return the central cpu as
++ * select_cpu() is a hint and if @p can't be on it, the kernel will
++ * automatically pick a fallback CPU.
++ */
++ return central_cpu;
++}
++
++void BPF_STRUCT_OPS(central_enqueue, struct task_struct *p, u64 enq_flags)
++{
++ s32 pid = p->pid;
++
++ __sync_fetch_and_add(&nr_total, 1);
++
++ /*
++ * Push per-cpu kthreads at the head of local dsq's and preempt the
++ * corresponding CPU. This ensures that e.g. ksoftirqd isn't blocked
++ * behind other threads which is necessary for forward progress
++ * guarantee as we depend on the BPF timer which may run from ksoftirqd.
++ */
++ if ((p->flags & PF_KTHREAD) && p->nr_cpus_allowed == 1) {
++ __sync_fetch_and_add(&nr_locals, 1);
++ scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_INF,
++ enq_flags | SCX_ENQ_PREEMPT);
++ return;
++ }
++
++ if (bpf_map_push_elem(&central_q, &pid, 0)) {
++ __sync_fetch_and_add(&nr_overflows, 1);
++ scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, enq_flags);
++ return;
++ }
++
++ __sync_fetch_and_add(&nr_queued, 1);
++
++ if (!scx_bpf_task_running(p))
++ scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT);
++}
++
++static bool dispatch_to_cpu(s32 cpu)
++{
++ struct task_struct *p;
++ s32 pid;
++
++ bpf_repeat(BPF_MAX_LOOPS) {
++ if (bpf_map_pop_elem(&central_q, &pid))
++ break;
++
++ __sync_fetch_and_sub(&nr_queued, 1);
++
++ p = bpf_task_from_pid(pid);
++ if (!p) {
++ __sync_fetch_and_add(&nr_lost_pids, 1);
++ continue;
++ }
++
++ /*
++ * If we can't run the task at the top, do the dumb thing and
++ * bounce it to the fallback dsq.
++ */
++ if (!bpf_cpumask_test_cpu(cpu, p->cpus_ptr)) {
++ __sync_fetch_and_add(&nr_mismatches, 1);
++ scx_bpf_dispatch(p, FALLBACK_DSQ_ID, SCX_SLICE_INF, 0);
++ bpf_task_release(p);
++ /*
++ * We might run out of dispatch buffer slots if we continue dispatching
++ * to the fallback DSQ, without dispatching to the local DSQ of the
++ * target CPU. In such a case, break the loop now as will fail the
++ * next dispatch operation.
++ */
++ if (!scx_bpf_dispatch_nr_slots())
++ break;
++ continue;
++ }
++
++ /* dispatch to local and mark that @cpu doesn't need more */
++ scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, SCX_SLICE_INF, 0);
++
++ if (cpu != central_cpu)
++ scx_bpf_kick_cpu(cpu, SCX_KICK_IDLE);
++
++ bpf_task_release(p);
++ return true;
++ }
++
++ return false;
++}
++
++void BPF_STRUCT_OPS(central_dispatch, s32 cpu, struct task_struct *prev)
++{
++ if (cpu == central_cpu) {
++ /* dispatch for all other CPUs first */
++ __sync_fetch_and_add(&nr_dispatches, 1);
++
++ bpf_for(cpu, 0, nr_cpu_ids) {
++ bool *gimme;
++
++ if (!scx_bpf_dispatch_nr_slots())
++ break;
++
++ /* central's gimme is never set */
++ gimme = ARRAY_ELEM_PTR(cpu_gimme_task, cpu, nr_cpu_ids);
++ if (gimme && !*gimme)
++ continue;
++
++ if (dispatch_to_cpu(cpu))
++ *gimme = false;
++ }
++
++ /*
++ * Retry if we ran out of dispatch buffer slots as we might have
++ * skipped some CPUs and also need to dispatch for self. The ext
++ * core automatically retries if the local dsq is empty but we
++ * can't rely on that as we're dispatching for other CPUs too.
++ * Kick self explicitly to retry.
++ */
++ if (!scx_bpf_dispatch_nr_slots()) {
++ __sync_fetch_and_add(&nr_retries, 1);
++ scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT);
++ return;
++ }
++
++ /* look for a task to run on the central CPU */
++ if (scx_bpf_consume(FALLBACK_DSQ_ID))
++ return;
++ dispatch_to_cpu(central_cpu);
++ } else {
++ bool *gimme;
++
++ if (scx_bpf_consume(FALLBACK_DSQ_ID))
++ return;
++
++ gimme = ARRAY_ELEM_PTR(cpu_gimme_task, cpu, nr_cpu_ids);
++ if (gimme)
++ *gimme = true;
++
++ /*
++ * Force dispatch on the scheduling CPU so that it finds a task
++ * to run for us.
++ */
++ scx_bpf_kick_cpu(central_cpu, SCX_KICK_PREEMPT);
++ }
++}
++
++void BPF_STRUCT_OPS(central_running, struct task_struct *p)
++{
++ s32 cpu = scx_bpf_task_cpu(p);
++ u64 *started_at = ARRAY_ELEM_PTR(cpu_started_at, cpu, nr_cpu_ids);
++ if (started_at)
++ *started_at = bpf_ktime_get_ns() ?: 1; /* 0 indicates idle */
++}
++
++void BPF_STRUCT_OPS(central_stopping, struct task_struct *p, bool runnable)
++{
++ s32 cpu = scx_bpf_task_cpu(p);
++ u64 *started_at = ARRAY_ELEM_PTR(cpu_started_at, cpu, nr_cpu_ids);
++ if (started_at)
++ *started_at = 0;
++}
++
++static int central_timerfn(void *map, int *key, struct bpf_timer *timer)
++{
++ u64 now = bpf_ktime_get_ns();
++ u64 nr_to_kick = nr_queued;
++ s32 i, curr_cpu;
++
++ curr_cpu = bpf_get_smp_processor_id();
++ if (timer_pinned && (curr_cpu != central_cpu)) {
++ scx_bpf_error("Central timer ran on CPU %d, not central CPU %d",
++ curr_cpu, central_cpu);
++ return 0;
++ }
++
++ bpf_for(i, 0, nr_cpu_ids) {
++ s32 cpu = (nr_timers + i) % nr_cpu_ids;
++ u64 *started_at;
++
++ if (cpu == central_cpu)
++ continue;
++
++ /* kick iff the current one exhausted its slice */
++ started_at = ARRAY_ELEM_PTR(cpu_started_at, cpu, nr_cpu_ids);
++ if (started_at && *started_at &&
++ vtime_before(now, *started_at + slice_ns))
++ continue;
++
++ /* and there's something pending */
++ if (scx_bpf_dsq_nr_queued(FALLBACK_DSQ_ID) ||
++ scx_bpf_dsq_nr_queued(SCX_DSQ_LOCAL_ON | cpu))
++ ;
++ else if (nr_to_kick)
++ nr_to_kick--;
++ else
++ continue;
++
++ scx_bpf_kick_cpu(cpu, SCX_KICK_PREEMPT);
++ }
++
++ bpf_timer_start(timer, TIMER_INTERVAL_NS, BPF_F_TIMER_CPU_PIN);
++ __sync_fetch_and_add(&nr_timers, 1);
++ return 0;
++}
++
++int BPF_STRUCT_OPS_SLEEPABLE(central_init)
++{
++ u32 key = 0;
++ struct bpf_timer *timer;
++ int ret;
++
++ ret = scx_bpf_create_dsq(FALLBACK_DSQ_ID, -1);
++ if (ret)
++ return ret;
++
++ timer = bpf_map_lookup_elem(&central_timer, &key);
++ if (!timer)
++ return -ESRCH;
++
++ if (bpf_get_smp_processor_id() != central_cpu) {
++ scx_bpf_error("init from non-central CPU");
++ return -EINVAL;
++ }
++
++ bpf_timer_init(timer, &central_timer, CLOCK_MONOTONIC);
++ bpf_timer_set_callback(timer, central_timerfn);
++
++ ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, BPF_F_TIMER_CPU_PIN);
++ /*
++ * BPF_F_TIMER_CPU_PIN is pretty new (>=6.7). If we're running in a
++ * kernel which doesn't have it, bpf_timer_start() will return -EINVAL.
++ * Retry without the PIN. This would be the perfect use case for
++ * bpf_core_enum_value_exists() but the enum type doesn't have a name
++ * and can't be used with bpf_core_enum_value_exists(). Oh well...
++ */
++ if (ret == -EINVAL) {
++ timer_pinned = false;
++ ret = bpf_timer_start(timer, TIMER_INTERVAL_NS, 0);
++ }
++ if (ret)
++ scx_bpf_error("bpf_timer_start failed (%d)", ret);
++ return ret;
++}
++
++void BPF_STRUCT_OPS(central_exit, struct scx_exit_info *ei)
++{
++ UEI_RECORD(uei, ei);
++}
++
++SCX_OPS_DEFINE(central_ops,
++ /*
++ * We are offloading all scheduling decisions to the central CPU
++ * and thus being the last task on a given CPU doesn't mean
++ * anything special. Enqueue the last tasks like any other tasks.
++ */
++ .flags = SCX_OPS_ENQ_LAST,
++
++ .select_cpu = (void *)central_select_cpu,
++ .enqueue = (void *)central_enqueue,
++ .dispatch = (void *)central_dispatch,
++ .running = (void *)central_running,
++ .stopping = (void *)central_stopping,
++ .init = (void *)central_init,
++ .exit = (void *)central_exit,
++ .name = "central");
+diff --git a/tools/sched_ext/scx_central.c b/tools/sched_ext/scx_central.c
+new file mode 100644
+index 000000000000..21deea320bd7
+--- /dev/null
++++ b/tools/sched_ext/scx_central.c
+@@ -0,0 +1,135 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
++ */
++#define _GNU_SOURCE
++#include <sched.h>
++#include <stdio.h>
++#include <unistd.h>
++#include <inttypes.h>
++#include <signal.h>
++#include <libgen.h>
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include "scx_central.bpf.skel.h"
++
++const char help_fmt[] =
++"A central FIFO sched_ext scheduler.\n"
++"\n"
++"See the top-level comment in .bpf.c for more details.\n"
++"\n"
++"Usage: %s [-s SLICE_US] [-c CPU]\n"
++"\n"
++" -s SLICE_US Override slice duration\n"
++" -c CPU Override the central CPU (default: 0)\n"
++" -v Print libbpf debug messages\n"
++" -h Display this help and exit\n";
++
++static bool verbose;
++static volatile int exit_req;
++
++static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
++{
++ if (level == LIBBPF_DEBUG && !verbose)
++ return 0;
++ return vfprintf(stderr, format, args);
++}
++
++static void sigint_handler(int dummy)
++{
++ exit_req = 1;
++}
++
++int main(int argc, char **argv)
++{
++ struct scx_central *skel;
++ struct bpf_link *link;
++ __u64 seq = 0, ecode;
++ __s32 opt;
++ cpu_set_t *cpuset;
++
++ libbpf_set_print(libbpf_print_fn);
++ signal(SIGINT, sigint_handler);
++ signal(SIGTERM, sigint_handler);
++restart:
++ skel = SCX_OPS_OPEN(central_ops, scx_central);
++
++ skel->rodata->central_cpu = 0;
++ skel->rodata->nr_cpu_ids = libbpf_num_possible_cpus();
++
++ while ((opt = getopt(argc, argv, "s:c:pvh")) != -1) {
++ switch (opt) {
++ case 's':
++ skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000;
++ break;
++ case 'c':
++ skel->rodata->central_cpu = strtoul(optarg, NULL, 0);
++ break;
++ case 'v':
++ verbose = true;
++ break;
++ default:
++ fprintf(stderr, help_fmt, basename(argv[0]));
++ return opt != 'h';
++ }
++ }
++
++ /* Resize arrays so their element count is equal to cpu count. */
++ RESIZE_ARRAY(skel, data, cpu_gimme_task, skel->rodata->nr_cpu_ids);
++ RESIZE_ARRAY(skel, data, cpu_started_at, skel->rodata->nr_cpu_ids);
++
++ SCX_OPS_LOAD(skel, central_ops, scx_central, uei);
++
++ /*
++ * Affinitize the loading thread to the central CPU, as:
++ * - That's where the BPF timer is first invoked in the BPF program.
++ * - We probably don't want this user space component to take up a core
++ * from a task that would benefit from avoiding preemption on one of
++ * the tickless cores.
++ *
++ * Until BPF supports pinning the timer, it's not guaranteed that it
++ * will always be invoked on the central CPU. In practice, this
++ * suffices the majority of the time.
++ */
++ cpuset = CPU_ALLOC(skel->rodata->nr_cpu_ids);
++ SCX_BUG_ON(!cpuset, "Failed to allocate cpuset");
++ CPU_ZERO(cpuset);
++ CPU_SET(skel->rodata->central_cpu, cpuset);
++ SCX_BUG_ON(sched_setaffinity(0, sizeof(cpuset), cpuset),
++ "Failed to affinitize to central CPU %d (max %d)",
++ skel->rodata->central_cpu, skel->rodata->nr_cpu_ids - 1);
++ CPU_FREE(cpuset);
++
++ link = SCX_OPS_ATTACH(skel, central_ops, scx_central);
++
++ if (!skel->data->timer_pinned)
++ printf("WARNING : BPF_F_TIMER_CPU_PIN not available, timer not pinned to central\n");
++
++ while (!exit_req && !UEI_EXITED(skel, uei)) {
++ printf("[SEQ %llu]\n", seq++);
++ printf("total :%10" PRIu64 " local:%10" PRIu64 " queued:%10" PRIu64 " lost:%10" PRIu64 "\n",
++ skel->bss->nr_total,
++ skel->bss->nr_locals,
++ skel->bss->nr_queued,
++ skel->bss->nr_lost_pids);
++ printf("timer :%10" PRIu64 " dispatch:%10" PRIu64 " mismatch:%10" PRIu64 " retry:%10" PRIu64 "\n",
++ skel->bss->nr_timers,
++ skel->bss->nr_dispatches,
++ skel->bss->nr_mismatches,
++ skel->bss->nr_retries);
++ printf("overflow:%10" PRIu64 "\n",
++ skel->bss->nr_overflows);
++ fflush(stdout);
++ sleep(1);
++ }
++
++ bpf_link__destroy(link);
++ ecode = UEI_REPORT(skel, uei);
++ scx_central__destroy(skel);
++
++ if (UEI_ECODE_RESTART(ecode))
++ goto restart;
++ return 0;
++}
+diff --git a/tools/sched_ext/scx_qmap.bpf.c b/tools/sched_ext/scx_qmap.bpf.c
+new file mode 100644
+index 000000000000..892278f12dce
+--- /dev/null
++++ b/tools/sched_ext/scx_qmap.bpf.c
+@@ -0,0 +1,706 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A simple five-level FIFO queue scheduler.
++ *
++ * There are five FIFOs implemented using BPF_MAP_TYPE_QUEUE. A task gets
++ * assigned to one depending on its compound weight. Each CPU round robins
++ * through the FIFOs and dispatches more from FIFOs with higher indices - 1 from
++ * queue0, 2 from queue1, 4 from queue2 and so on.
++ *
++ * This scheduler demonstrates:
++ *
++ * - BPF-side queueing using PIDs.
++ * - Sleepable per-task storage allocation using ops.prep_enable().
++ * - Using ops.cpu_release() to handle a higher priority scheduling class taking
++ * the CPU away.
++ * - Core-sched support.
++ *
++ * This scheduler is primarily for demonstration and testing of sched_ext
++ * features and unlikely to be useful for actual workloads.
++ *
++ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
++ */
++#include <scx/common.bpf.h>
++
++enum consts {
++ ONE_SEC_IN_NS = 1000000000,
++ SHARED_DSQ = 0,
++};
++
++char _license[] SEC("license") = "GPL";
++
++const volatile u64 slice_ns = SCX_SLICE_DFL;
++const volatile u32 stall_user_nth;
++const volatile u32 stall_kernel_nth;
++const volatile u32 dsp_inf_loop_after;
++const volatile u32 dsp_batch;
++const volatile bool print_shared_dsq;
++const volatile s32 disallow_tgid;
++const volatile bool suppress_dump;
++
++u32 test_error_cnt;
++
++UEI_DEFINE(uei);
++
++struct qmap {
++ __uint(type, BPF_MAP_TYPE_QUEUE);
++ __uint(max_entries, 4096);
++ __type(value, u32);
++} queue0 SEC(".maps"),
++ queue1 SEC(".maps"),
++ queue2 SEC(".maps"),
++ queue3 SEC(".maps"),
++ queue4 SEC(".maps");
++
++struct {
++ __uint(type, BPF_MAP_TYPE_ARRAY_OF_MAPS);
++ __uint(max_entries, 5);
++ __type(key, int);
++ __array(values, struct qmap);
++} queue_arr SEC(".maps") = {
++ .values = {
++ [0] = &queue0,
++ [1] = &queue1,
++ [2] = &queue2,
++ [3] = &queue3,
++ [4] = &queue4,
++ },
++};
++
++/*
++ * If enabled, CPU performance target is set according to the queue index
++ * according to the following table.
++ */
++static const u32 qidx_to_cpuperf_target[] = {
++ [0] = SCX_CPUPERF_ONE * 0 / 4,
++ [1] = SCX_CPUPERF_ONE * 1 / 4,
++ [2] = SCX_CPUPERF_ONE * 2 / 4,
++ [3] = SCX_CPUPERF_ONE * 3 / 4,
++ [4] = SCX_CPUPERF_ONE * 4 / 4,
++};
++
++/*
++ * Per-queue sequence numbers to implement core-sched ordering.
++ *
++ * Tail seq is assigned to each queued task and incremented. Head seq tracks the
++ * sequence number of the latest dispatched task. The distance between the a
++ * task's seq and the associated queue's head seq is called the queue distance
++ * and used when comparing two tasks for ordering. See qmap_core_sched_before().
++ */
++static u64 core_sched_head_seqs[5];
++static u64 core_sched_tail_seqs[5];
++
++/* Per-task scheduling context */
++struct task_ctx {
++ bool force_local; /* Dispatch directly to local_dsq */
++ u64 core_sched_seq;
++};
++
++struct {
++ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
++ __uint(map_flags, BPF_F_NO_PREALLOC);
++ __type(key, int);
++ __type(value, struct task_ctx);
++} task_ctx_stor SEC(".maps");
++
++struct cpu_ctx {
++ u64 dsp_idx; /* dispatch index */
++ u64 dsp_cnt; /* remaining count */
++ u32 avg_weight;
++ u32 cpuperf_target;
++};
++
++struct {
++ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
++ __uint(max_entries, 1);
++ __type(key, u32);
++ __type(value, struct cpu_ctx);
++} cpu_ctx_stor SEC(".maps");
++
++/* Statistics */
++u64 nr_enqueued, nr_dispatched, nr_reenqueued, nr_dequeued, nr_ddsp_from_enq;
++u64 nr_core_sched_execed;
++u32 cpuperf_min, cpuperf_avg, cpuperf_max;
++u32 cpuperf_target_min, cpuperf_target_avg, cpuperf_target_max;
++
++static s32 pick_direct_dispatch_cpu(struct task_struct *p, s32 prev_cpu)
++{
++ s32 cpu;
++
++ if (p->nr_cpus_allowed == 1 ||
++ scx_bpf_test_and_clear_cpu_idle(prev_cpu))
++ return prev_cpu;
++
++ cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0);
++ if (cpu >= 0)
++ return cpu;
++
++ return -1;
++}
++
++s32 BPF_STRUCT_OPS(qmap_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ struct task_ctx *tctx;
++ s32 cpu;
++
++ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
++ if (!tctx) {
++ scx_bpf_error("task_ctx lookup failed");
++ return -ESRCH;
++ }
++
++ cpu = pick_direct_dispatch_cpu(p, prev_cpu);
++
++ if (cpu >= 0) {
++ tctx->force_local = true;
++ return cpu;
++ } else {
++ return prev_cpu;
++ }
++}
++
++static int weight_to_idx(u32 weight)
++{
++ /* Coarsely map the compound weight to a FIFO. */
++ if (weight <= 25)
++ return 0;
++ else if (weight <= 50)
++ return 1;
++ else if (weight < 200)
++ return 2;
++ else if (weight < 400)
++ return 3;
++ else
++ return 4;
++}
++
++void BPF_STRUCT_OPS(qmap_enqueue, struct task_struct *p, u64 enq_flags)
++{
++ static u32 user_cnt, kernel_cnt;
++ struct task_ctx *tctx;
++ u32 pid = p->pid;
++ int idx = weight_to_idx(p->scx.weight);
++ void *ring;
++ s32 cpu;
++
++ if (p->flags & PF_KTHREAD) {
++ if (stall_kernel_nth && !(++kernel_cnt % stall_kernel_nth))
++ return;
++ } else {
++ if (stall_user_nth && !(++user_cnt % stall_user_nth))
++ return;
++ }
++
++ if (test_error_cnt && !--test_error_cnt)
++ scx_bpf_error("test triggering error");
++
++ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
++ if (!tctx) {
++ scx_bpf_error("task_ctx lookup failed");
++ return;
++ }
++
++ /*
++ * All enqueued tasks must have their core_sched_seq updated for correct
++ * core-sched ordering, which is why %SCX_OPS_ENQ_LAST is specified in
++ * qmap_ops.flags.
++ */
++ tctx->core_sched_seq = core_sched_tail_seqs[idx]++;
++
++ /*
++ * If qmap_select_cpu() is telling us to or this is the last runnable
++ * task on the CPU, enqueue locally.
++ */
++ if (tctx->force_local || (enq_flags & SCX_ENQ_LAST)) {
++ tctx->force_local = false;
++ scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, enq_flags);
++ return;
++ }
++
++ /* if !WAKEUP, select_cpu() wasn't called, try direct dispatch */
++ if (!(enq_flags & SCX_ENQ_WAKEUP) &&
++ (cpu = pick_direct_dispatch_cpu(p, scx_bpf_task_cpu(p))) >= 0) {
++ __sync_fetch_and_add(&nr_ddsp_from_enq, 1);
++ scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | cpu, slice_ns, enq_flags);
++ return;
++ }
++
++ /*
++ * If the task was re-enqueued due to the CPU being preempted by a
++ * higher priority scheduling class, just re-enqueue the task directly
++ * on the global DSQ. As we want another CPU to pick it up, find and
++ * kick an idle CPU.
++ */
++ if (enq_flags & SCX_ENQ_REENQ) {
++ s32 cpu;
++
++ scx_bpf_dispatch(p, SHARED_DSQ, 0, enq_flags);
++ cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0);
++ if (cpu >= 0)
++ scx_bpf_kick_cpu(cpu, SCX_KICK_IDLE);
++ return;
++ }
++
++ ring = bpf_map_lookup_elem(&queue_arr, &idx);
++ if (!ring) {
++ scx_bpf_error("failed to find ring %d", idx);
++ return;
++ }
++
++ /* Queue on the selected FIFO. If the FIFO overflows, punt to global. */
++ if (bpf_map_push_elem(ring, &pid, 0)) {
++ scx_bpf_dispatch(p, SHARED_DSQ, slice_ns, enq_flags);
++ return;
++ }
++
++ __sync_fetch_and_add(&nr_enqueued, 1);
++}
++
++/*
++ * The BPF queue map doesn't support removal and sched_ext can handle spurious
++ * dispatches. qmap_dequeue() is only used to collect statistics.
++ */
++void BPF_STRUCT_OPS(qmap_dequeue, struct task_struct *p, u64 deq_flags)
++{
++ __sync_fetch_and_add(&nr_dequeued, 1);
++ if (deq_flags & SCX_DEQ_CORE_SCHED_EXEC)
++ __sync_fetch_and_add(&nr_core_sched_execed, 1);
++}
++
++static void update_core_sched_head_seq(struct task_struct *p)
++{
++ struct task_ctx *tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
++ int idx = weight_to_idx(p->scx.weight);
++
++ if (tctx)
++ core_sched_head_seqs[idx] = tctx->core_sched_seq;
++ else
++ scx_bpf_error("task_ctx lookup failed");
++}
++
++void BPF_STRUCT_OPS(qmap_dispatch, s32 cpu, struct task_struct *prev)
++{
++ struct task_struct *p;
++ struct cpu_ctx *cpuc;
++ u32 zero = 0, batch = dsp_batch ?: 1;
++ void *fifo;
++ s32 i, pid;
++
++ if (scx_bpf_consume(SHARED_DSQ))
++ return;
++
++ if (dsp_inf_loop_after && nr_dispatched > dsp_inf_loop_after) {
++ /*
++ * PID 2 should be kthreadd which should mostly be idle and off
++ * the scheduler. Let's keep dispatching it to force the kernel
++ * to call this function over and over again.
++ */
++ p = bpf_task_from_pid(2);
++ if (p) {
++ scx_bpf_dispatch(p, SCX_DSQ_LOCAL, slice_ns, 0);
++ bpf_task_release(p);
++ return;
++ }
++ }
++
++ if (!(cpuc = bpf_map_lookup_elem(&cpu_ctx_stor, &zero))) {
++ scx_bpf_error("failed to look up cpu_ctx");
++ return;
++ }
++
++ for (i = 0; i < 5; i++) {
++ /* Advance the dispatch cursor and pick the fifo. */
++ if (!cpuc->dsp_cnt) {
++ cpuc->dsp_idx = (cpuc->dsp_idx + 1) % 5;
++ cpuc->dsp_cnt = 1 << cpuc->dsp_idx;
++ }
++
++ fifo = bpf_map_lookup_elem(&queue_arr, &cpuc->dsp_idx);
++ if (!fifo) {
++ scx_bpf_error("failed to find ring %llu", cpuc->dsp_idx);
++ return;
++ }
++
++ /* Dispatch or advance. */
++ bpf_repeat(BPF_MAX_LOOPS) {
++ if (bpf_map_pop_elem(fifo, &pid))
++ break;
++
++ p = bpf_task_from_pid(pid);
++ if (!p)
++ continue;
++
++ update_core_sched_head_seq(p);
++ __sync_fetch_and_add(&nr_dispatched, 1);
++ scx_bpf_dispatch(p, SHARED_DSQ, slice_ns, 0);
++ bpf_task_release(p);
++ batch--;
++ cpuc->dsp_cnt--;
++ if (!batch || !scx_bpf_dispatch_nr_slots()) {
++ scx_bpf_consume(SHARED_DSQ);
++ return;
++ }
++ if (!cpuc->dsp_cnt)
++ break;
++ }
++
++ cpuc->dsp_cnt = 0;
++ }
++}
++
++void BPF_STRUCT_OPS(qmap_tick, struct task_struct *p)
++{
++ struct cpu_ctx *cpuc;
++ u32 zero = 0;
++ int idx;
++
++ if (!(cpuc = bpf_map_lookup_elem(&cpu_ctx_stor, &zero))) {
++ scx_bpf_error("failed to look up cpu_ctx");
++ return;
++ }
++
++ /*
++ * Use the running avg of weights to select the target cpuperf level.
++ * This is a demonstration of the cpuperf feature rather than a
++ * practical strategy to regulate CPU frequency.
++ */
++ cpuc->avg_weight = cpuc->avg_weight * 3 / 4 + p->scx.weight / 4;
++ idx = weight_to_idx(cpuc->avg_weight);
++ cpuc->cpuperf_target = qidx_to_cpuperf_target[idx];
++
++ scx_bpf_cpuperf_set(scx_bpf_task_cpu(p), cpuc->cpuperf_target);
++}
++
++/*
++ * The distance from the head of the queue scaled by the weight of the queue.
++ * The lower the number, the older the task and the higher the priority.
++ */
++static s64 task_qdist(struct task_struct *p)
++{
++ int idx = weight_to_idx(p->scx.weight);
++ struct task_ctx *tctx;
++ s64 qdist;
++
++ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
++ if (!tctx) {
++ scx_bpf_error("task_ctx lookup failed");
++ return 0;
++ }
++
++ qdist = tctx->core_sched_seq - core_sched_head_seqs[idx];
++
++ /*
++ * As queue index increments, the priority doubles. The queue w/ index 3
++ * is dispatched twice more frequently than 2. Reflect the difference by
++ * scaling qdists accordingly. Note that the shift amount needs to be
++ * flipped depending on the sign to avoid flipping priority direction.
++ */
++ if (qdist >= 0)
++ return qdist << (4 - idx);
++ else
++ return qdist << idx;
++}
++
++/*
++ * This is called to determine the task ordering when core-sched is picking
++ * tasks to execute on SMT siblings and should encode about the same ordering as
++ * the regular scheduling path. Use the priority-scaled distances from the head
++ * of the queues to compare the two tasks which should be consistent with the
++ * dispatch path behavior.
++ */
++bool BPF_STRUCT_OPS(qmap_core_sched_before,
++ struct task_struct *a, struct task_struct *b)
++{
++ return task_qdist(a) > task_qdist(b);
++}
++
++void BPF_STRUCT_OPS(qmap_cpu_release, s32 cpu, struct scx_cpu_release_args *args)
++{
++ u32 cnt;
++
++ /*
++ * Called when @cpu is taken by a higher priority scheduling class. This
++ * makes @cpu no longer available for executing sched_ext tasks. As we
++ * don't want the tasks in @cpu's local dsq to sit there until @cpu
++ * becomes available again, re-enqueue them into the global dsq. See
++ * %SCX_ENQ_REENQ handling in qmap_enqueue().
++ */
++ cnt = scx_bpf_reenqueue_local();
++ if (cnt)
++ __sync_fetch_and_add(&nr_reenqueued, cnt);
++}
++
++s32 BPF_STRUCT_OPS(qmap_init_task, struct task_struct *p,
++ struct scx_init_task_args *args)
++{
++ if (p->tgid == disallow_tgid)
++ p->scx.disallow = true;
++
++ /*
++ * @p is new. Let's ensure that its task_ctx is available. We can sleep
++ * in this function and the following will automatically use GFP_KERNEL.
++ */
++ if (bpf_task_storage_get(&task_ctx_stor, p, 0,
++ BPF_LOCAL_STORAGE_GET_F_CREATE))
++ return 0;
++ else
++ return -ENOMEM;
++}
++
++void BPF_STRUCT_OPS(qmap_dump, struct scx_dump_ctx *dctx)
++{
++ s32 i, pid;
++
++ if (suppress_dump)
++ return;
++
++ bpf_for(i, 0, 5) {
++ void *fifo;
++
++ if (!(fifo = bpf_map_lookup_elem(&queue_arr, &i)))
++ return;
++
++ scx_bpf_dump("QMAP FIFO[%d]:", i);
++ bpf_repeat(4096) {
++ if (bpf_map_pop_elem(fifo, &pid))
++ break;
++ scx_bpf_dump(" %d", pid);
++ }
++ scx_bpf_dump("\n");
++ }
++}
++
++void BPF_STRUCT_OPS(qmap_dump_cpu, struct scx_dump_ctx *dctx, s32 cpu, bool idle)
++{
++ u32 zero = 0;
++ struct cpu_ctx *cpuc;
++
++ if (suppress_dump || idle)
++ return;
++ if (!(cpuc = bpf_map_lookup_percpu_elem(&cpu_ctx_stor, &zero, cpu)))
++ return;
++
++ scx_bpf_dump("QMAP: dsp_idx=%llu dsp_cnt=%llu avg_weight=%u cpuperf_target=%u",
++ cpuc->dsp_idx, cpuc->dsp_cnt, cpuc->avg_weight,
++ cpuc->cpuperf_target);
++}
++
++void BPF_STRUCT_OPS(qmap_dump_task, struct scx_dump_ctx *dctx, struct task_struct *p)
++{
++ struct task_ctx *taskc;
++
++ if (suppress_dump)
++ return;
++ if (!(taskc = bpf_task_storage_get(&task_ctx_stor, p, 0, 0)))
++ return;
++
++ scx_bpf_dump("QMAP: force_local=%d core_sched_seq=%llu",
++ taskc->force_local, taskc->core_sched_seq);
++}
++
++/*
++ * Print out the online and possible CPU map using bpf_printk() as a
++ * demonstration of using the cpumask kfuncs and ops.cpu_on/offline().
++ */
++static void print_cpus(void)
++{
++ const struct cpumask *possible, *online;
++ s32 cpu;
++ char buf[128] = "", *p;
++ int idx;
++
++ possible = scx_bpf_get_possible_cpumask();
++ online = scx_bpf_get_online_cpumask();
++
++ idx = 0;
++ bpf_for(cpu, 0, scx_bpf_nr_cpu_ids()) {
++ if (!(p = MEMBER_VPTR(buf, [idx++])))
++ break;
++ if (bpf_cpumask_test_cpu(cpu, online))
++ *p++ = 'O';
++ else if (bpf_cpumask_test_cpu(cpu, possible))
++ *p++ = 'X';
++ else
++ *p++ = ' ';
++
++ if ((cpu & 7) == 7) {
++ if (!(p = MEMBER_VPTR(buf, [idx++])))
++ break;
++ *p++ = '|';
++ }
++ }
++ buf[sizeof(buf) - 1] = '\0';
++
++ scx_bpf_put_cpumask(online);
++ scx_bpf_put_cpumask(possible);
++
++ bpf_printk("CPUS: |%s", buf);
++}
++
++void BPF_STRUCT_OPS(qmap_cpu_online, s32 cpu)
++{
++ bpf_printk("CPU %d coming online", cpu);
++ /* @cpu is already online at this point */
++ print_cpus();
++}
++
++void BPF_STRUCT_OPS(qmap_cpu_offline, s32 cpu)
++{
++ bpf_printk("CPU %d going offline", cpu);
++ /* @cpu is still online at this point */
++ print_cpus();
++}
++
++struct monitor_timer {
++ struct bpf_timer timer;
++};
++
++struct {
++ __uint(type, BPF_MAP_TYPE_ARRAY);
++ __uint(max_entries, 1);
++ __type(key, u32);
++ __type(value, struct monitor_timer);
++} monitor_timer SEC(".maps");
++
++/*
++ * Print out the min, avg and max performance levels of CPUs every second to
++ * demonstrate the cpuperf interface.
++ */
++static void monitor_cpuperf(void)
++{
++ u32 zero = 0, nr_cpu_ids;
++ u64 cap_sum = 0, cur_sum = 0, cur_min = SCX_CPUPERF_ONE, cur_max = 0;
++ u64 target_sum = 0, target_min = SCX_CPUPERF_ONE, target_max = 0;
++ const struct cpumask *online;
++ int i, nr_online_cpus = 0;
++
++ nr_cpu_ids = scx_bpf_nr_cpu_ids();
++ online = scx_bpf_get_online_cpumask();
++
++ bpf_for(i, 0, nr_cpu_ids) {
++ struct cpu_ctx *cpuc;
++ u32 cap, cur;
++
++ if (!bpf_cpumask_test_cpu(i, online))
++ continue;
++ nr_online_cpus++;
++
++ /* collect the capacity and current cpuperf */
++ cap = scx_bpf_cpuperf_cap(i);
++ cur = scx_bpf_cpuperf_cur(i);
++
++ cur_min = cur < cur_min ? cur : cur_min;
++ cur_max = cur > cur_max ? cur : cur_max;
++
++ /*
++ * $cur is relative to $cap. Scale it down accordingly so that
++ * it's in the same scale as other CPUs and $cur_sum/$cap_sum
++ * makes sense.
++ */
++ cur_sum += cur * cap / SCX_CPUPERF_ONE;
++ cap_sum += cap;
++
++ if (!(cpuc = bpf_map_lookup_percpu_elem(&cpu_ctx_stor, &zero, i))) {
++ scx_bpf_error("failed to look up cpu_ctx");
++ goto out;
++ }
++
++ /* collect target */
++ cur = cpuc->cpuperf_target;
++ target_sum += cur;
++ target_min = cur < target_min ? cur : target_min;
++ target_max = cur > target_max ? cur : target_max;
++ }
++
++ cpuperf_min = cur_min;
++ cpuperf_avg = cur_sum * SCX_CPUPERF_ONE / cap_sum;
++ cpuperf_max = cur_max;
++
++ cpuperf_target_min = target_min;
++ cpuperf_target_avg = target_sum / nr_online_cpus;
++ cpuperf_target_max = target_max;
++out:
++ scx_bpf_put_cpumask(online);
++}
++
++/*
++ * Dump the currently queued tasks in the shared DSQ to demonstrate the usage of
++ * scx_bpf_dsq_nr_queued() and DSQ iterator. Raise the dispatch batch count to
++ * see meaningful dumps in the trace pipe.
++ */
++static void dump_shared_dsq(void)
++{
++ struct task_struct *p;
++ s32 nr;
++
++ if (!(nr = scx_bpf_dsq_nr_queued(SHARED_DSQ)))
++ return;
++
++ bpf_printk("Dumping %d tasks in SHARED_DSQ in reverse order", nr);
++
++ bpf_rcu_read_lock();
++ bpf_for_each(scx_dsq, p, SHARED_DSQ, SCX_DSQ_ITER_REV)
++ bpf_printk("%s[%d]", p->comm, p->pid);
++ bpf_rcu_read_unlock();
++}
++
++static int monitor_timerfn(void *map, int *key, struct bpf_timer *timer)
++{
++ monitor_cpuperf();
++
++ if (print_shared_dsq)
++ dump_shared_dsq();
++
++ bpf_timer_start(timer, ONE_SEC_IN_NS, 0);
++ return 0;
++}
++
++s32 BPF_STRUCT_OPS_SLEEPABLE(qmap_init)
++{
++ u32 key = 0;
++ struct bpf_timer *timer;
++ s32 ret;
++
++ print_cpus();
++
++ ret = scx_bpf_create_dsq(SHARED_DSQ, -1);
++ if (ret)
++ return ret;
++
++ timer = bpf_map_lookup_elem(&monitor_timer, &key);
++ if (!timer)
++ return -ESRCH;
++
++ bpf_timer_init(timer, &monitor_timer, CLOCK_MONOTONIC);
++ bpf_timer_set_callback(timer, monitor_timerfn);
++
++ return bpf_timer_start(timer, ONE_SEC_IN_NS, 0);
++}
++
++void BPF_STRUCT_OPS(qmap_exit, struct scx_exit_info *ei)
++{
++ UEI_RECORD(uei, ei);
++}
++
++SCX_OPS_DEFINE(qmap_ops,
++ .select_cpu = (void *)qmap_select_cpu,
++ .enqueue = (void *)qmap_enqueue,
++ .dequeue = (void *)qmap_dequeue,
++ .dispatch = (void *)qmap_dispatch,
++ .tick = (void *)qmap_tick,
++ .core_sched_before = (void *)qmap_core_sched_before,
++ .cpu_release = (void *)qmap_cpu_release,
++ .init_task = (void *)qmap_init_task,
++ .dump = (void *)qmap_dump,
++ .dump_cpu = (void *)qmap_dump_cpu,
++ .dump_task = (void *)qmap_dump_task,
++ .cpu_online = (void *)qmap_cpu_online,
++ .cpu_offline = (void *)qmap_cpu_offline,
++ .init = (void *)qmap_init,
++ .exit = (void *)qmap_exit,
++ .flags = SCX_OPS_ENQ_LAST,
++ .timeout_ms = 5000U,
++ .name = "qmap");
+diff --git a/tools/sched_ext/scx_qmap.c b/tools/sched_ext/scx_qmap.c
+new file mode 100644
+index 000000000000..c9ca30d62b2b
+--- /dev/null
++++ b/tools/sched_ext/scx_qmap.c
+@@ -0,0 +1,144 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
++ */
++#include <stdio.h>
++#include <stdlib.h>
++#include <unistd.h>
++#include <inttypes.h>
++#include <signal.h>
++#include <libgen.h>
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include "scx_qmap.bpf.skel.h"
++
++const char help_fmt[] =
++"A simple five-level FIFO queue sched_ext scheduler.\n"
++"\n"
++"See the top-level comment in .bpf.c for more details.\n"
++"\n"
++"Usage: %s [-s SLICE_US] [-e COUNT] [-t COUNT] [-T COUNT] [-l COUNT] [-b COUNT]\n"
++" [-P] [-d PID] [-D LEN] [-p] [-v]\n"
++"\n"
++" -s SLICE_US Override slice duration\n"
++" -e COUNT Trigger scx_bpf_error() after COUNT enqueues\n"
++" -t COUNT Stall every COUNT'th user thread\n"
++" -T COUNT Stall every COUNT'th kernel thread\n"
++" -l COUNT Trigger dispatch infinite looping after COUNT dispatches\n"
++" -b COUNT Dispatch upto COUNT tasks together\n"
++" -P Print out DSQ content to trace_pipe every second, use with -b\n"
++" -d PID Disallow a process from switching into SCHED_EXT (-1 for self)\n"
++" -D LEN Set scx_exit_info.dump buffer length\n"
++" -S Suppress qmap-specific debug dump\n"
++" -p Switch only tasks on SCHED_EXT policy instead of all\n"
++" -v Print libbpf debug messages\n"
++" -h Display this help and exit\n";
++
++static bool verbose;
++static volatile int exit_req;
++
++static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
++{
++ if (level == LIBBPF_DEBUG && !verbose)
++ return 0;
++ return vfprintf(stderr, format, args);
++}
++
++static void sigint_handler(int dummy)
++{
++ exit_req = 1;
++}
++
++int main(int argc, char **argv)
++{
++ struct scx_qmap *skel;
++ struct bpf_link *link;
++ int opt;
++
++ libbpf_set_print(libbpf_print_fn);
++ signal(SIGINT, sigint_handler);
++ signal(SIGTERM, sigint_handler);
++
++ skel = SCX_OPS_OPEN(qmap_ops, scx_qmap);
++
++ while ((opt = getopt(argc, argv, "s:e:t:T:l:b:Pd:D:Spvh")) != -1) {
++ switch (opt) {
++ case 's':
++ skel->rodata->slice_ns = strtoull(optarg, NULL, 0) * 1000;
++ break;
++ case 'e':
++ skel->bss->test_error_cnt = strtoul(optarg, NULL, 0);
++ break;
++ case 't':
++ skel->rodata->stall_user_nth = strtoul(optarg, NULL, 0);
++ break;
++ case 'T':
++ skel->rodata->stall_kernel_nth = strtoul(optarg, NULL, 0);
++ break;
++ case 'l':
++ skel->rodata->dsp_inf_loop_after = strtoul(optarg, NULL, 0);
++ break;
++ case 'b':
++ skel->rodata->dsp_batch = strtoul(optarg, NULL, 0);
++ break;
++ case 'P':
++ skel->rodata->print_shared_dsq = true;
++ break;
++ case 'd':
++ skel->rodata->disallow_tgid = strtol(optarg, NULL, 0);
++ if (skel->rodata->disallow_tgid < 0)
++ skel->rodata->disallow_tgid = getpid();
++ break;
++ case 'D':
++ skel->struct_ops.qmap_ops->exit_dump_len = strtoul(optarg, NULL, 0);
++ break;
++ case 'S':
++ skel->rodata->suppress_dump = true;
++ break;
++ case 'p':
++ skel->struct_ops.qmap_ops->flags |= SCX_OPS_SWITCH_PARTIAL;
++ break;
++ case 'v':
++ verbose = true;
++ break;
++ default:
++ fprintf(stderr, help_fmt, basename(argv[0]));
++ return opt != 'h';
++ }
++ }
++
++ SCX_OPS_LOAD(skel, qmap_ops, scx_qmap, uei);
++ link = SCX_OPS_ATTACH(skel, qmap_ops, scx_qmap);
++
++ while (!exit_req && !UEI_EXITED(skel, uei)) {
++ long nr_enqueued = skel->bss->nr_enqueued;
++ long nr_dispatched = skel->bss->nr_dispatched;
++
++ printf("stats : enq=%lu dsp=%lu delta=%ld reenq=%"PRIu64" deq=%"PRIu64" core=%"PRIu64" enq_ddsp=%"PRIu64"\n",
++ nr_enqueued, nr_dispatched, nr_enqueued - nr_dispatched,
++ skel->bss->nr_reenqueued, skel->bss->nr_dequeued,
++ skel->bss->nr_core_sched_execed,
++ skel->bss->nr_ddsp_from_enq);
++ if (__COMPAT_has_ksym("scx_bpf_cpuperf_cur"))
++ printf("cpuperf: cur min/avg/max=%u/%u/%u target min/avg/max=%u/%u/%u\n",
++ skel->bss->cpuperf_min,
++ skel->bss->cpuperf_avg,
++ skel->bss->cpuperf_max,
++ skel->bss->cpuperf_target_min,
++ skel->bss->cpuperf_target_avg,
++ skel->bss->cpuperf_target_max);
++ fflush(stdout);
++ sleep(1);
++ }
++
++ bpf_link__destroy(link);
++ UEI_REPORT(skel, uei);
++ scx_qmap__destroy(skel);
++ /*
++ * scx_qmap implements ops.cpu_on/offline() and doesn't need to restart
++ * on CPU hotplug events.
++ */
++ return 0;
++}
+diff --git a/tools/sched_ext/scx_show_state.py b/tools/sched_ext/scx_show_state.py
+new file mode 100644
+index 000000000000..d457d2a74e1e
+--- /dev/null
++++ b/tools/sched_ext/scx_show_state.py
+@@ -0,0 +1,39 @@
++#!/usr/bin/env drgn
++#
++# Copyright (C) 2024 Tejun Heo <tj@kernel.org>
++# Copyright (C) 2024 Meta Platforms, Inc. and affiliates.
++
++desc = """
++This is a drgn script to show the current sched_ext state.
++For more info on drgn, visit https://github.com/osandov/drgn.
++"""
++
++import drgn
++import sys
++
++def err(s):
++ print(s, file=sys.stderr, flush=True)
++ sys.exit(1)
++
++def read_int(name):
++ return int(prog[name].value_())
++
++def read_atomic(name):
++ return prog[name].counter.value_()
++
++def read_static_key(name):
++ return prog[name].key.enabled.counter.value_()
++
++def ops_state_str(state):
++ return prog['scx_ops_enable_state_str'][state].string_().decode()
++
++ops = prog['scx_ops']
++enable_state = read_atomic("scx_ops_enable_state_var")
++
++print(f'ops : {ops.name.string_().decode()}')
++print(f'enabled : {read_static_key("__scx_ops_enabled")}')
++print(f'switching_all : {read_int("scx_switching_all")}')
++print(f'switched_all : {read_static_key("__scx_switched_all")}')
++print(f'enable_state : {ops_state_str(enable_state)} ({enable_state})')
++print(f'bypass_depth : {read_atomic("scx_ops_bypass_depth")}')
++print(f'nr_rejected : {read_atomic("scx_nr_rejected")}')
+diff --git a/tools/sched_ext/scx_simple.bpf.c b/tools/sched_ext/scx_simple.bpf.c
+new file mode 100644
+index 000000000000..ed7e8d535fc5
+--- /dev/null
++++ b/tools/sched_ext/scx_simple.bpf.c
+@@ -0,0 +1,156 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A simple scheduler.
++ *
++ * By default, it operates as a simple global weighted vtime scheduler and can
++ * be switched to FIFO scheduling. It also demonstrates the following niceties.
++ *
++ * - Statistics tracking how many tasks are queued to local and global dsq's.
++ * - Termination notification for userspace.
++ *
++ * While very simple, this scheduler should work reasonably well on CPUs with a
++ * uniform L3 cache topology. While preemption is not implemented, the fact that
++ * the scheduling queue is shared across all CPUs means that whatever is at the
++ * front of the queue is likely to be executed fairly quickly given enough
++ * number of CPUs. The FIFO scheduling mode may be beneficial to some workloads
++ * but comes with the usual problems with FIFO scheduling where saturating
++ * threads can easily drown out interactive ones.
++ *
++ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
++ */
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++const volatile bool fifo_sched;
++
++static u64 vtime_now;
++UEI_DEFINE(uei);
++
++/*
++ * Built-in DSQs such as SCX_DSQ_GLOBAL cannot be used as priority queues
++ * (meaning, cannot be dispatched to with scx_bpf_dispatch_vtime()). We
++ * therefore create a separate DSQ with ID 0 that we dispatch to and consume
++ * from. If scx_simple only supported global FIFO scheduling, then we could
++ * just use SCX_DSQ_GLOBAL.
++ */
++#define SHARED_DSQ 0
++
++struct {
++ __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
++ __uint(key_size, sizeof(u32));
++ __uint(value_size, sizeof(u64));
++ __uint(max_entries, 2); /* [local, global] */
++} stats SEC(".maps");
++
++static void stat_inc(u32 idx)
++{
++ u64 *cnt_p = bpf_map_lookup_elem(&stats, &idx);
++ if (cnt_p)
++ (*cnt_p)++;
++}
++
++static inline bool vtime_before(u64 a, u64 b)
++{
++ return (s64)(a - b) < 0;
++}
++
++s32 BPF_STRUCT_OPS(simple_select_cpu, struct task_struct *p, s32 prev_cpu, u64 wake_flags)
++{
++ bool is_idle = false;
++ s32 cpu;
++
++ cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &is_idle);
++ if (is_idle) {
++ stat_inc(0); /* count local queueing */
++ scx_bpf_dispatch(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL, 0);
++ }
++
++ return cpu;
++}
++
++void BPF_STRUCT_OPS(simple_enqueue, struct task_struct *p, u64 enq_flags)
++{
++ stat_inc(1); /* count global queueing */
++
++ if (fifo_sched) {
++ scx_bpf_dispatch(p, SHARED_DSQ, SCX_SLICE_DFL, enq_flags);
++ } else {
++ u64 vtime = p->scx.dsq_vtime;
++
++ /*
++ * Limit the amount of budget that an idling task can accumulate
++ * to one slice.
++ */
++ if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL))
++ vtime = vtime_now - SCX_SLICE_DFL;
++
++ scx_bpf_dispatch_vtime(p, SHARED_DSQ, SCX_SLICE_DFL, vtime,
++ enq_flags);
++ }
++}
++
++void BPF_STRUCT_OPS(simple_dispatch, s32 cpu, struct task_struct *prev)
++{
++ scx_bpf_consume(SHARED_DSQ);
++}
++
++void BPF_STRUCT_OPS(simple_running, struct task_struct *p)
++{
++ if (fifo_sched)
++ return;
++
++ /*
++ * Global vtime always progresses forward as tasks start executing. The
++ * test and update can be performed concurrently from multiple CPUs and
++ * thus racy. Any error should be contained and temporary. Let's just
++ * live with it.
++ */
++ if (vtime_before(vtime_now, p->scx.dsq_vtime))
++ vtime_now = p->scx.dsq_vtime;
++}
++
++void BPF_STRUCT_OPS(simple_stopping, struct task_struct *p, bool runnable)
++{
++ if (fifo_sched)
++ return;
++
++ /*
++ * Scale the execution time by the inverse of the weight and charge.
++ *
++ * Note that the default yield implementation yields by setting
++ * @p->scx.slice to zero and the following would treat the yielding task
++ * as if it has consumed all its slice. If this penalizes yielding tasks
++ * too much, determine the execution time by taking explicit timestamps
++ * instead of depending on @p->scx.slice.
++ */
++ p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight;
++}
++
++void BPF_STRUCT_OPS(simple_enable, struct task_struct *p)
++{
++ p->scx.dsq_vtime = vtime_now;
++}
++
++s32 BPF_STRUCT_OPS_SLEEPABLE(simple_init)
++{
++ return scx_bpf_create_dsq(SHARED_DSQ, -1);
++}
++
++void BPF_STRUCT_OPS(simple_exit, struct scx_exit_info *ei)
++{
++ UEI_RECORD(uei, ei);
++}
++
++SCX_OPS_DEFINE(simple_ops,
++ .select_cpu = (void *)simple_select_cpu,
++ .enqueue = (void *)simple_enqueue,
++ .dispatch = (void *)simple_dispatch,
++ .running = (void *)simple_running,
++ .stopping = (void *)simple_stopping,
++ .enable = (void *)simple_enable,
++ .init = (void *)simple_init,
++ .exit = (void *)simple_exit,
++ .name = "simple");
+diff --git a/tools/sched_ext/scx_simple.c b/tools/sched_ext/scx_simple.c
+new file mode 100644
+index 000000000000..76d83199545c
+--- /dev/null
++++ b/tools/sched_ext/scx_simple.c
+@@ -0,0 +1,107 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2022 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2022 David Vernet <dvernet@meta.com>
++ */
++#include <stdio.h>
++#include <unistd.h>
++#include <signal.h>
++#include <libgen.h>
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include "scx_simple.bpf.skel.h"
++
++const char help_fmt[] =
++"A simple sched_ext scheduler.\n"
++"\n"
++"See the top-level comment in .bpf.c for more details.\n"
++"\n"
++"Usage: %s [-f] [-v]\n"
++"\n"
++" -f Use FIFO scheduling instead of weighted vtime scheduling\n"
++" -v Print libbpf debug messages\n"
++" -h Display this help and exit\n";
++
++static bool verbose;
++static volatile int exit_req;
++
++static int libbpf_print_fn(enum libbpf_print_level level, const char *format, va_list args)
++{
++ if (level == LIBBPF_DEBUG && !verbose)
++ return 0;
++ return vfprintf(stderr, format, args);
++}
++
++static void sigint_handler(int simple)
++{
++ exit_req = 1;
++}
++
++static void read_stats(struct scx_simple *skel, __u64 *stats)
++{
++ int nr_cpus = libbpf_num_possible_cpus();
++ __u64 cnts[2][nr_cpus];
++ __u32 idx;
++
++ memset(stats, 0, sizeof(stats[0]) * 2);
++
++ for (idx = 0; idx < 2; idx++) {
++ int ret, cpu;
++
++ ret = bpf_map_lookup_elem(bpf_map__fd(skel->maps.stats),
++ &idx, cnts[idx]);
++ if (ret < 0)
++ continue;
++ for (cpu = 0; cpu < nr_cpus; cpu++)
++ stats[idx] += cnts[idx][cpu];
++ }
++}
++
++int main(int argc, char **argv)
++{
++ struct scx_simple *skel;
++ struct bpf_link *link;
++ __u32 opt;
++ __u64 ecode;
++
++ libbpf_set_print(libbpf_print_fn);
++ signal(SIGINT, sigint_handler);
++ signal(SIGTERM, sigint_handler);
++restart:
++ skel = SCX_OPS_OPEN(simple_ops, scx_simple);
++
++ while ((opt = getopt(argc, argv, "fvh")) != -1) {
++ switch (opt) {
++ case 'f':
++ skel->rodata->fifo_sched = true;
++ break;
++ case 'v':
++ verbose = true;
++ break;
++ default:
++ fprintf(stderr, help_fmt, basename(argv[0]));
++ return opt != 'h';
++ }
++ }
++
++ SCX_OPS_LOAD(skel, simple_ops, scx_simple, uei);
++ link = SCX_OPS_ATTACH(skel, simple_ops, scx_simple);
++
++ while (!exit_req && !UEI_EXITED(skel, uei)) {
++ __u64 stats[2];
++
++ read_stats(skel, stats);
++ printf("local=%llu global=%llu\n", stats[0], stats[1]);
++ fflush(stdout);
++ sleep(1);
++ }
++
++ bpf_link__destroy(link);
++ ecode = UEI_REPORT(skel, uei);
++ scx_simple__destroy(skel);
++
++ if (UEI_ECODE_RESTART(ecode))
++ goto restart;
++ return 0;
++}
+diff --git a/tools/testing/selftests/sched_ext/.gitignore b/tools/testing/selftests/sched_ext/.gitignore
+new file mode 100644
+index 000000000000..ae5491a114c0
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/.gitignore
+@@ -0,0 +1,6 @@
++*
++!*.c
++!*.h
++!Makefile
++!.gitignore
++!config
+diff --git a/tools/testing/selftests/sched_ext/Makefile b/tools/testing/selftests/sched_ext/Makefile
+new file mode 100644
+index 000000000000..0754a2c110a1
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/Makefile
+@@ -0,0 +1,218 @@
++# SPDX-License-Identifier: GPL-2.0
++# Copyright (c) 2022 Meta Platforms, Inc. and affiliates.
++include ../../../build/Build.include
++include ../../../scripts/Makefile.arch
++include ../../../scripts/Makefile.include
++include ../lib.mk
++
++ifneq ($(LLVM),)
++ifneq ($(filter %/,$(LLVM)),)
++LLVM_PREFIX := $(LLVM)
++else ifneq ($(filter -%,$(LLVM)),)
++LLVM_SUFFIX := $(LLVM)
++endif
++
++CC := $(LLVM_PREFIX)clang$(LLVM_SUFFIX) $(CLANG_FLAGS) -fintegrated-as
++else
++CC := gcc
++endif # LLVM
++
++ifneq ($(CROSS_COMPILE),)
++$(error CROSS_COMPILE not supported for scx selftests)
++endif # CROSS_COMPILE
++
++CURDIR := $(abspath .)
++REPOROOT := $(abspath ../../../..)
++TOOLSDIR := $(REPOROOT)/tools
++LIBDIR := $(TOOLSDIR)/lib
++BPFDIR := $(LIBDIR)/bpf
++TOOLSINCDIR := $(TOOLSDIR)/include
++BPFTOOLDIR := $(TOOLSDIR)/bpf/bpftool
++APIDIR := $(TOOLSINCDIR)/uapi
++GENDIR := $(REPOROOT)/include/generated
++GENHDR := $(GENDIR)/autoconf.h
++SCXTOOLSDIR := $(TOOLSDIR)/sched_ext
++SCXTOOLSINCDIR := $(TOOLSDIR)/sched_ext/include
++
++OUTPUT_DIR := $(CURDIR)/build
++OBJ_DIR := $(OUTPUT_DIR)/obj
++INCLUDE_DIR := $(OUTPUT_DIR)/include
++BPFOBJ_DIR := $(OBJ_DIR)/libbpf
++SCXOBJ_DIR := $(OBJ_DIR)/sched_ext
++BPFOBJ := $(BPFOBJ_DIR)/libbpf.a
++LIBBPF_OUTPUT := $(OBJ_DIR)/libbpf/libbpf.a
++DEFAULT_BPFTOOL := $(OUTPUT_DIR)/sbin/bpftool
++HOST_BUILD_DIR := $(OBJ_DIR)
++HOST_OUTPUT_DIR := $(OUTPUT_DIR)
++
++VMLINUX_BTF_PATHS ?= ../../../../vmlinux \
++ /sys/kernel/btf/vmlinux \
++ /boot/vmlinux-$(shell uname -r)
++VMLINUX_BTF ?= $(abspath $(firstword $(wildcard $(VMLINUX_BTF_PATHS))))
++ifeq ($(VMLINUX_BTF),)
++$(error Cannot find a vmlinux for VMLINUX_BTF at any of "$(VMLINUX_BTF_PATHS)")
++endif
++
++BPFTOOL ?= $(DEFAULT_BPFTOOL)
++
++ifneq ($(wildcard $(GENHDR)),)
++ GENFLAGS := -DHAVE_GENHDR
++endif
++
++CFLAGS += -g -O2 -rdynamic -pthread -Wall -Werror $(GENFLAGS) \
++ -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \
++ -I$(TOOLSINCDIR) -I$(APIDIR) -I$(CURDIR)/include -I$(SCXTOOLSINCDIR)
++
++# Silence some warnings when compiled with clang
++ifneq ($(LLVM),)
++CFLAGS += -Wno-unused-command-line-argument
++endif
++
++LDFLAGS = -lelf -lz -lpthread -lzstd
++
++IS_LITTLE_ENDIAN = $(shell $(CC) -dM -E - </dev/null | \
++ grep 'define __BYTE_ORDER__ __ORDER_LITTLE_ENDIAN__')
++
++# Get Clang's default includes on this system, as opposed to those seen by
++# '-target bpf'. This fixes "missing" files on some architectures/distros,
++# such as asm/byteorder.h, asm/socket.h, asm/sockios.h, sys/cdefs.h etc.
++#
++# Use '-idirafter': Don't interfere with include mechanics except where the
++# build would have failed anyways.
++define get_sys_includes
++$(shell $(1) -v -E - </dev/null 2>&1 \
++ | sed -n '/<...> search starts here:/,/End of search list./{ s| \(/.*\)|-idirafter \1|p }') \
++$(shell $(1) -dM -E - </dev/null | grep '__riscv_xlen ' | awk '{printf("-D__riscv_xlen=%d -D__BITS_PER_LONG=%d", $$3, $$3)}')
++endef
++
++BPF_CFLAGS = -g -D__TARGET_ARCH_$(SRCARCH) \
++ $(if $(IS_LITTLE_ENDIAN),-mlittle-endian,-mbig-endian) \
++ -I$(CURDIR)/include -I$(CURDIR)/include/bpf-compat \
++ -I$(INCLUDE_DIR) -I$(APIDIR) -I$(SCXTOOLSINCDIR) \
++ -I$(REPOROOT)/include \
++ $(call get_sys_includes,$(CLANG)) \
++ -Wall -Wno-compare-distinct-pointer-types \
++ -Wno-incompatible-function-pointer-types \
++ -O2 -mcpu=v3
++
++# sort removes libbpf duplicates when not cross-building
++MAKE_DIRS := $(sort $(OBJ_DIR)/libbpf $(OBJ_DIR)/libbpf \
++ $(OBJ_DIR)/bpftool $(OBJ_DIR)/resolve_btfids \
++ $(INCLUDE_DIR) $(SCXOBJ_DIR))
++
++$(MAKE_DIRS):
++ $(call msg,MKDIR,,$@)
++ $(Q)mkdir -p $@
++
++$(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \
++ $(APIDIR)/linux/bpf.h \
++ | $(OBJ_DIR)/libbpf
++ $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(OBJ_DIR)/libbpf/ \
++ EXTRA_CFLAGS='-g -O0 -fPIC' \
++ DESTDIR=$(OUTPUT_DIR) prefix= all install_headers
++
++$(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \
++ $(LIBBPF_OUTPUT) | $(OBJ_DIR)/bpftool
++ $(Q)$(MAKE) $(submake_extras) -C $(BPFTOOLDIR) \
++ ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) \
++ EXTRA_CFLAGS='-g -O0' \
++ OUTPUT=$(OBJ_DIR)/bpftool/ \
++ LIBBPF_OUTPUT=$(OBJ_DIR)/libbpf/ \
++ LIBBPF_DESTDIR=$(OUTPUT_DIR)/ \
++ prefix= DESTDIR=$(OUTPUT_DIR)/ install-bin
++
++$(INCLUDE_DIR)/vmlinux.h: $(VMLINUX_BTF) $(BPFTOOL) | $(INCLUDE_DIR)
++ifeq ($(VMLINUX_H),)
++ $(call msg,GEN,,$@)
++ $(Q)$(BPFTOOL) btf dump file $(VMLINUX_BTF) format c > $@
++else
++ $(call msg,CP,,$@)
++ $(Q)cp "$(VMLINUX_H)" $@
++endif
++
++$(SCXOBJ_DIR)/%.bpf.o: %.bpf.c $(INCLUDE_DIR)/vmlinux.h | $(BPFOBJ) $(SCXOBJ_DIR)
++ $(call msg,CLNG-BPF,,$(notdir $@))
++ $(Q)$(CLANG) $(BPF_CFLAGS) -target bpf -c $< -o $@
++
++$(INCLUDE_DIR)/%.bpf.skel.h: $(SCXOBJ_DIR)/%.bpf.o $(INCLUDE_DIR)/vmlinux.h $(BPFTOOL) | $(INCLUDE_DIR)
++ $(eval sched=$(notdir $@))
++ $(call msg,GEN-SKEL,,$(sched))
++ $(Q)$(BPFTOOL) gen object $(<:.o=.linked1.o) $<
++ $(Q)$(BPFTOOL) gen object $(<:.o=.linked2.o) $(<:.o=.linked1.o)
++ $(Q)$(BPFTOOL) gen object $(<:.o=.linked3.o) $(<:.o=.linked2.o)
++ $(Q)diff $(<:.o=.linked2.o) $(<:.o=.linked3.o)
++ $(Q)$(BPFTOOL) gen skeleton $(<:.o=.linked3.o) name $(subst .bpf.skel.h,,$(sched)) > $@
++ $(Q)$(BPFTOOL) gen subskeleton $(<:.o=.linked3.o) name $(subst .bpf.skel.h,,$(sched)) > $(@:.skel.h=.subskel.h)
++
++################
++# C schedulers #
++################
++
++override define CLEAN
++ rm -rf $(OUTPUT_DIR)
++ rm -f *.o *.bpf.o *.bpf.skel.h *.bpf.subskel.h
++ rm -f $(TEST_GEN_PROGS)
++ rm -f runner
++endef
++
++# Every testcase takes all of the BPF progs are dependencies by default. This
++# allows testcases to load any BPF scheduler, which is useful for testcases
++# that don't need their own prog to run their test.
++all_test_bpfprogs := $(foreach prog,$(wildcard *.bpf.c),$(INCLUDE_DIR)/$(patsubst %.c,%.skel.h,$(prog)))
++
++auto-test-targets := \
++ create_dsq \
++ enq_last_no_enq_fails \
++ enq_select_cpu_fails \
++ ddsp_bogus_dsq_fail \
++ ddsp_vtimelocal_fail \
++ dsp_local_on \
++ exit \
++ hotplug \
++ init_enable_count \
++ maximal \
++ maybe_null \
++ minimal \
++ prog_run \
++ reload_loop \
++ select_cpu_dfl \
++ select_cpu_dfl_nodispatch \
++ select_cpu_dispatch \
++ select_cpu_dispatch_bad_dsq \
++ select_cpu_dispatch_dbl_dsp \
++ select_cpu_vtime \
++ test_example \
++
++testcase-targets := $(addsuffix .o,$(addprefix $(SCXOBJ_DIR)/,$(auto-test-targets)))
++
++$(SCXOBJ_DIR)/runner.o: runner.c | $(SCXOBJ_DIR)
++ $(CC) $(CFLAGS) -c $< -o $@
++
++# Create all of the test targets object files, whose testcase objects will be
++# registered into the runner in ELF constructors.
++#
++# Note that we must do double expansion here in order to support conditionally
++# compiling BPF object files only if one is present, as the wildcard Make
++# function doesn't support using implicit rules otherwise.
++$(testcase-targets): $(SCXOBJ_DIR)/%.o: %.c $(SCXOBJ_DIR)/runner.o $(all_test_bpfprogs) | $(SCXOBJ_DIR)
++ $(eval test=$(patsubst %.o,%.c,$(notdir $@)))
++ $(CC) $(CFLAGS) -c $< -o $@ $(SCXOBJ_DIR)/runner.o
++
++$(SCXOBJ_DIR)/util.o: util.c | $(SCXOBJ_DIR)
++ $(CC) $(CFLAGS) -c $< -o $@
++
++runner: $(SCXOBJ_DIR)/runner.o $(SCXOBJ_DIR)/util.o $(BPFOBJ) $(testcase-targets)
++ @echo "$(testcase-targets)"
++ $(CC) $(CFLAGS) -o $@ $^ $(LDFLAGS)
++
++TEST_GEN_PROGS := runner
++
++all: runner
++
++.PHONY: all clean help
++
++.DEFAULT_GOAL := all
++
++.DELETE_ON_ERROR:
++
++.SECONDARY:
+diff --git a/tools/testing/selftests/sched_ext/config b/tools/testing/selftests/sched_ext/config
+new file mode 100644
+index 000000000000..0de9b4ee249d
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/config
+@@ -0,0 +1,9 @@
++CONFIG_SCHED_DEBUG=y
++CONFIG_SCHED_CLASS_EXT=y
++CONFIG_CGROUPS=y
++CONFIG_CGROUP_SCHED=y
++CONFIG_EXT_GROUP_SCHED=y
++CONFIG_BPF=y
++CONFIG_BPF_SYSCALL=y
++CONFIG_DEBUG_INFO=y
++CONFIG_DEBUG_INFO_BTF=y
+diff --git a/tools/testing/selftests/sched_ext/create_dsq.bpf.c b/tools/testing/selftests/sched_ext/create_dsq.bpf.c
+new file mode 100644
+index 000000000000..23f79ed343f0
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/create_dsq.bpf.c
+@@ -0,0 +1,58 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Create and destroy DSQs in a loop.
++ *
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++void BPF_STRUCT_OPS(create_dsq_exit_task, struct task_struct *p,
++ struct scx_exit_task_args *args)
++{
++ scx_bpf_destroy_dsq(p->pid);
++}
++
++s32 BPF_STRUCT_OPS_SLEEPABLE(create_dsq_init_task, struct task_struct *p,
++ struct scx_init_task_args *args)
++{
++ s32 err;
++
++ err = scx_bpf_create_dsq(p->pid, -1);
++ if (err)
++ scx_bpf_error("Failed to create DSQ for %s[%d]",
++ p->comm, p->pid);
++
++ return err;
++}
++
++s32 BPF_STRUCT_OPS_SLEEPABLE(create_dsq_init)
++{
++ u32 i;
++ s32 err;
++
++ bpf_for(i, 0, 1024) {
++ err = scx_bpf_create_dsq(i, -1);
++ if (err) {
++ scx_bpf_error("Failed to create DSQ %d", i);
++ return 0;
++ }
++ }
++
++ bpf_for(i, 0, 1024) {
++ scx_bpf_destroy_dsq(i);
++ }
++
++ return 0;
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops create_dsq_ops = {
++ .init_task = create_dsq_init_task,
++ .exit_task = create_dsq_exit_task,
++ .init = create_dsq_init,
++ .name = "create_dsq",
++};
+diff --git a/tools/testing/selftests/sched_ext/create_dsq.c b/tools/testing/selftests/sched_ext/create_dsq.c
+new file mode 100644
+index 000000000000..fa946d9146d4
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/create_dsq.c
+@@ -0,0 +1,57 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "create_dsq.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct create_dsq *skel;
++
++ skel = create_dsq__open_and_load();
++ if (!skel) {
++ SCX_ERR("Failed to open and load skel");
++ return SCX_TEST_FAIL;
++ }
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct create_dsq *skel = ctx;
++ struct bpf_link *link;
++
++ link = bpf_map__attach_struct_ops(skel->maps.create_dsq_ops);
++ if (!link) {
++ SCX_ERR("Failed to attach scheduler");
++ return SCX_TEST_FAIL;
++ }
++
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct create_dsq *skel = ctx;
++
++ create_dsq__destroy(skel);
++}
++
++struct scx_test create_dsq = {
++ .name = "create_dsq",
++ .description = "Create and destroy a dsq in a loop",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&create_dsq)
+diff --git a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c
+new file mode 100644
+index 000000000000..e97ad41d354a
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.bpf.c
+@@ -0,0 +1,42 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
++ */
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++UEI_DEFINE(uei);
++
++s32 BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0);
++
++ if (cpu >= 0) {
++ /*
++ * If we dispatch to a bogus DSQ that will fall back to the
++ * builtin global DSQ, we fail gracefully.
++ */
++ scx_bpf_dispatch_vtime(p, 0xcafef00d, SCX_SLICE_DFL,
++ p->scx.dsq_vtime, 0);
++ return cpu;
++ }
++
++ return prev_cpu;
++}
++
++void BPF_STRUCT_OPS(ddsp_bogus_dsq_fail_exit, struct scx_exit_info *ei)
++{
++ UEI_RECORD(uei, ei);
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops ddsp_bogus_dsq_fail_ops = {
++ .select_cpu = ddsp_bogus_dsq_fail_select_cpu,
++ .exit = ddsp_bogus_dsq_fail_exit,
++ .name = "ddsp_bogus_dsq_fail",
++ .timeout_ms = 1000U,
++};
+diff --git a/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.c b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.c
+new file mode 100644
+index 000000000000..e65d22f23f3b
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/ddsp_bogus_dsq_fail.c
+@@ -0,0 +1,57 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "ddsp_bogus_dsq_fail.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct ddsp_bogus_dsq_fail *skel;
++
++ skel = ddsp_bogus_dsq_fail__open_and_load();
++ SCX_FAIL_IF(!skel, "Failed to open and load skel");
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct ddsp_bogus_dsq_fail *skel = ctx;
++ struct bpf_link *link;
++
++ link = bpf_map__attach_struct_ops(skel->maps.ddsp_bogus_dsq_fail_ops);
++ SCX_FAIL_IF(!link, "Failed to attach struct_ops");
++
++ sleep(1);
++
++ SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_ERROR));
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct ddsp_bogus_dsq_fail *skel = ctx;
++
++ ddsp_bogus_dsq_fail__destroy(skel);
++}
++
++struct scx_test ddsp_bogus_dsq_fail = {
++ .name = "ddsp_bogus_dsq_fail",
++ .description = "Verify we gracefully fail, and fall back to using a "
++ "built-in DSQ, if we do a direct dispatch to an invalid"
++ " DSQ in ops.select_cpu()",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&ddsp_bogus_dsq_fail)
+diff --git a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c
+new file mode 100644
+index 000000000000..dde7e7dafbfb
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.bpf.c
+@@ -0,0 +1,39 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
++ */
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++UEI_DEFINE(uei);
++
++s32 BPF_STRUCT_OPS(ddsp_vtimelocal_fail_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ s32 cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0);
++
++ if (cpu >= 0) {
++ /* Shouldn't be allowed to vtime dispatch to a builtin DSQ. */
++ scx_bpf_dispatch_vtime(p, SCX_DSQ_LOCAL, SCX_SLICE_DFL,
++ p->scx.dsq_vtime, 0);
++ return cpu;
++ }
++
++ return prev_cpu;
++}
++
++void BPF_STRUCT_OPS(ddsp_vtimelocal_fail_exit, struct scx_exit_info *ei)
++{
++ UEI_RECORD(uei, ei);
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops ddsp_vtimelocal_fail_ops = {
++ .select_cpu = ddsp_vtimelocal_fail_select_cpu,
++ .exit = ddsp_vtimelocal_fail_exit,
++ .name = "ddsp_vtimelocal_fail",
++ .timeout_ms = 1000U,
++};
+diff --git a/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.c b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.c
+new file mode 100644
+index 000000000000..abafee587cd6
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/ddsp_vtimelocal_fail.c
+@@ -0,0 +1,56 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <unistd.h>
++#include "ddsp_vtimelocal_fail.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct ddsp_vtimelocal_fail *skel;
++
++ skel = ddsp_vtimelocal_fail__open_and_load();
++ SCX_FAIL_IF(!skel, "Failed to open and load skel");
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct ddsp_vtimelocal_fail *skel = ctx;
++ struct bpf_link *link;
++
++ link = bpf_map__attach_struct_ops(skel->maps.ddsp_vtimelocal_fail_ops);
++ SCX_FAIL_IF(!link, "Failed to attach struct_ops");
++
++ sleep(1);
++
++ SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_ERROR));
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct ddsp_vtimelocal_fail *skel = ctx;
++
++ ddsp_vtimelocal_fail__destroy(skel);
++}
++
++struct scx_test ddsp_vtimelocal_fail = {
++ .name = "ddsp_vtimelocal_fail",
++ .description = "Verify we gracefully fail, and fall back to using a "
++ "built-in DSQ, if we do a direct vtime dispatch to a "
++ "built-in DSQ from DSQ in ops.select_cpu()",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&ddsp_vtimelocal_fail)
+diff --git a/tools/testing/selftests/sched_ext/dsp_local_on.bpf.c b/tools/testing/selftests/sched_ext/dsp_local_on.bpf.c
+new file mode 100644
+index 000000000000..efb4672decb4
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/dsp_local_on.bpf.c
+@@ -0,0 +1,65 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++const volatile s32 nr_cpus;
++
++UEI_DEFINE(uei);
++
++struct {
++ __uint(type, BPF_MAP_TYPE_QUEUE);
++ __uint(max_entries, 8192);
++ __type(value, s32);
++} queue SEC(".maps");
++
++s32 BPF_STRUCT_OPS(dsp_local_on_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ return prev_cpu;
++}
++
++void BPF_STRUCT_OPS(dsp_local_on_enqueue, struct task_struct *p,
++ u64 enq_flags)
++{
++ s32 pid = p->pid;
++
++ if (bpf_map_push_elem(&queue, &pid, 0))
++ scx_bpf_error("Failed to enqueue %s[%d]", p->comm, p->pid);
++}
++
++void BPF_STRUCT_OPS(dsp_local_on_dispatch, s32 cpu, struct task_struct *prev)
++{
++ s32 pid, target;
++ struct task_struct *p;
++
++ if (bpf_map_pop_elem(&queue, &pid))
++ return;
++
++ p = bpf_task_from_pid(pid);
++ if (!p)
++ return;
++
++ target = bpf_get_prandom_u32() % nr_cpus;
++
++ scx_bpf_dispatch(p, SCX_DSQ_LOCAL_ON | target, SCX_SLICE_DFL, 0);
++ bpf_task_release(p);
++}
++
++void BPF_STRUCT_OPS(dsp_local_on_exit, struct scx_exit_info *ei)
++{
++ UEI_RECORD(uei, ei);
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops dsp_local_on_ops = {
++ .select_cpu = dsp_local_on_select_cpu,
++ .enqueue = dsp_local_on_enqueue,
++ .dispatch = dsp_local_on_dispatch,
++ .exit = dsp_local_on_exit,
++ .name = "dsp_local_on",
++ .timeout_ms = 1000U,
++};
+diff --git a/tools/testing/selftests/sched_ext/dsp_local_on.c b/tools/testing/selftests/sched_ext/dsp_local_on.c
+new file mode 100644
+index 000000000000..472851b56854
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/dsp_local_on.c
+@@ -0,0 +1,58 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <unistd.h>
++#include "dsp_local_on.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct dsp_local_on *skel;
++
++ skel = dsp_local_on__open();
++ SCX_FAIL_IF(!skel, "Failed to open");
++
++ skel->rodata->nr_cpus = libbpf_num_possible_cpus();
++ SCX_FAIL_IF(dsp_local_on__load(skel), "Failed to load skel");
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct dsp_local_on *skel = ctx;
++ struct bpf_link *link;
++
++ link = bpf_map__attach_struct_ops(skel->maps.dsp_local_on_ops);
++ SCX_FAIL_IF(!link, "Failed to attach struct_ops");
++
++ /* Just sleeping is fine, plenty of scheduling events happening */
++ sleep(1);
++
++ SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_ERROR));
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct dsp_local_on *skel = ctx;
++
++ dsp_local_on__destroy(skel);
++}
++
++struct scx_test dsp_local_on = {
++ .name = "dsp_local_on",
++ .description = "Verify we can directly dispatch tasks to a local DSQs "
++ "from osp.dispatch()",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&dsp_local_on)
+diff --git a/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.bpf.c b/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.bpf.c
+new file mode 100644
+index 000000000000..b0b99531d5d5
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.bpf.c
+@@ -0,0 +1,21 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A scheduler that validates the behavior of direct dispatching with a default
++ * select_cpu implementation.
++ *
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++SEC(".struct_ops.link")
++struct sched_ext_ops enq_last_no_enq_fails_ops = {
++ .name = "enq_last_no_enq_fails",
++ /* Need to define ops.enqueue() with SCX_OPS_ENQ_LAST */
++ .flags = SCX_OPS_ENQ_LAST,
++ .timeout_ms = 1000U,
++};
+diff --git a/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.c b/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.c
+new file mode 100644
+index 000000000000..2a3eda5e2c0b
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/enq_last_no_enq_fails.c
+@@ -0,0 +1,60 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "enq_last_no_enq_fails.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct enq_last_no_enq_fails *skel;
++
++ skel = enq_last_no_enq_fails__open_and_load();
++ if (!skel) {
++ SCX_ERR("Failed to open and load skel");
++ return SCX_TEST_FAIL;
++ }
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct enq_last_no_enq_fails *skel = ctx;
++ struct bpf_link *link;
++
++ link = bpf_map__attach_struct_ops(skel->maps.enq_last_no_enq_fails_ops);
++ if (link) {
++ SCX_ERR("Incorrectly succeeded in to attaching scheduler");
++ return SCX_TEST_FAIL;
++ }
++
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct enq_last_no_enq_fails *skel = ctx;
++
++ enq_last_no_enq_fails__destroy(skel);
++}
++
++struct scx_test enq_last_no_enq_fails = {
++ .name = "enq_last_no_enq_fails",
++ .description = "Verify we fail to load a scheduler if we specify "
++ "the SCX_OPS_ENQ_LAST flag without defining "
++ "ops.enqueue()",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&enq_last_no_enq_fails)
+diff --git a/tools/testing/selftests/sched_ext/enq_select_cpu_fails.bpf.c b/tools/testing/selftests/sched_ext/enq_select_cpu_fails.bpf.c
+new file mode 100644
+index 000000000000..b3dfc1033cd6
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/enq_select_cpu_fails.bpf.c
+@@ -0,0 +1,43 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++/* Manually specify the signature until the kfunc is added to the scx repo. */
++s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
++ bool *found) __ksym;
++
++s32 BPF_STRUCT_OPS(enq_select_cpu_fails_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ return prev_cpu;
++}
++
++void BPF_STRUCT_OPS(enq_select_cpu_fails_enqueue, struct task_struct *p,
++ u64 enq_flags)
++{
++ /*
++ * Need to initialize the variable or the verifier will fail to load.
++ * Improving these semantics is actively being worked on.
++ */
++ bool found = false;
++
++ /* Can only call from ops.select_cpu() */
++ scx_bpf_select_cpu_dfl(p, 0, 0, &found);
++
++ scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops enq_select_cpu_fails_ops = {
++ .select_cpu = enq_select_cpu_fails_select_cpu,
++ .enqueue = enq_select_cpu_fails_enqueue,
++ .name = "enq_select_cpu_fails",
++ .timeout_ms = 1000U,
++};
+diff --git a/tools/testing/selftests/sched_ext/enq_select_cpu_fails.c b/tools/testing/selftests/sched_ext/enq_select_cpu_fails.c
+new file mode 100644
+index 000000000000..dd1350e5f002
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/enq_select_cpu_fails.c
+@@ -0,0 +1,61 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "enq_select_cpu_fails.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct enq_select_cpu_fails *skel;
++
++ skel = enq_select_cpu_fails__open_and_load();
++ if (!skel) {
++ SCX_ERR("Failed to open and load skel");
++ return SCX_TEST_FAIL;
++ }
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct enq_select_cpu_fails *skel = ctx;
++ struct bpf_link *link;
++
++ link = bpf_map__attach_struct_ops(skel->maps.enq_select_cpu_fails_ops);
++ if (!link) {
++ SCX_ERR("Failed to attach scheduler");
++ return SCX_TEST_FAIL;
++ }
++
++ sleep(1);
++
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct enq_select_cpu_fails *skel = ctx;
++
++ enq_select_cpu_fails__destroy(skel);
++}
++
++struct scx_test enq_select_cpu_fails = {
++ .name = "enq_select_cpu_fails",
++ .description = "Verify we fail to call scx_bpf_select_cpu_dfl() "
++ "from ops.enqueue()",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&enq_select_cpu_fails)
+diff --git a/tools/testing/selftests/sched_ext/exit.bpf.c b/tools/testing/selftests/sched_ext/exit.bpf.c
+new file mode 100644
+index 000000000000..ae12ddaac921
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/exit.bpf.c
+@@ -0,0 +1,84 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++#include "exit_test.h"
++
++const volatile int exit_point;
++UEI_DEFINE(uei);
++
++#define EXIT_CLEANLY() scx_bpf_exit(exit_point, "%d", exit_point)
++
++s32 BPF_STRUCT_OPS(exit_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ bool found;
++
++ if (exit_point == EXIT_SELECT_CPU)
++ EXIT_CLEANLY();
++
++ return scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags, &found);
++}
++
++void BPF_STRUCT_OPS(exit_enqueue, struct task_struct *p, u64 enq_flags)
++{
++ if (exit_point == EXIT_ENQUEUE)
++ EXIT_CLEANLY();
++
++ scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
++}
++
++void BPF_STRUCT_OPS(exit_dispatch, s32 cpu, struct task_struct *p)
++{
++ if (exit_point == EXIT_DISPATCH)
++ EXIT_CLEANLY();
++
++ scx_bpf_consume(SCX_DSQ_GLOBAL);
++}
++
++void BPF_STRUCT_OPS(exit_enable, struct task_struct *p)
++{
++ if (exit_point == EXIT_ENABLE)
++ EXIT_CLEANLY();
++}
++
++s32 BPF_STRUCT_OPS(exit_init_task, struct task_struct *p,
++ struct scx_init_task_args *args)
++{
++ if (exit_point == EXIT_INIT_TASK)
++ EXIT_CLEANLY();
++
++ return 0;
++}
++
++void BPF_STRUCT_OPS(exit_exit, struct scx_exit_info *ei)
++{
++ UEI_RECORD(uei, ei);
++}
++
++s32 BPF_STRUCT_OPS_SLEEPABLE(exit_init)
++{
++ if (exit_point == EXIT_INIT)
++ EXIT_CLEANLY();
++
++ return 0;
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops exit_ops = {
++ .select_cpu = exit_select_cpu,
++ .enqueue = exit_enqueue,
++ .dispatch = exit_dispatch,
++ .init_task = exit_init_task,
++ .enable = exit_enable,
++ .exit = exit_exit,
++ .init = exit_init,
++ .name = "exit",
++ .timeout_ms = 1000U,
++};
+diff --git a/tools/testing/selftests/sched_ext/exit.c b/tools/testing/selftests/sched_ext/exit.c
+new file mode 100644
+index 000000000000..31bcd06e21cd
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/exit.c
+@@ -0,0 +1,55 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#include <bpf/bpf.h>
++#include <sched.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "exit.bpf.skel.h"
++#include "scx_test.h"
++
++#include "exit_test.h"
++
++static enum scx_test_status run(void *ctx)
++{
++ enum exit_test_case tc;
++
++ for (tc = 0; tc < NUM_EXITS; tc++) {
++ struct exit *skel;
++ struct bpf_link *link;
++ char buf[16];
++
++ skel = exit__open();
++ skel->rodata->exit_point = tc;
++ exit__load(skel);
++ link = bpf_map__attach_struct_ops(skel->maps.exit_ops);
++ if (!link) {
++ SCX_ERR("Failed to attach scheduler");
++ exit__destroy(skel);
++ return SCX_TEST_FAIL;
++ }
++
++ /* Assumes uei.kind is written last */
++ while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE))
++ sched_yield();
++
++ SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF));
++ SCX_EQ(skel->data->uei.exit_code, tc);
++ sprintf(buf, "%d", tc);
++ SCX_ASSERT(!strcmp(skel->data->uei.msg, buf));
++ bpf_link__destroy(link);
++ exit__destroy(skel);
++ }
++
++ return SCX_TEST_PASS;
++}
++
++struct scx_test exit_test = {
++ .name = "exit",
++ .description = "Verify we can cleanly exit a scheduler in multiple places",
++ .run = run,
++};
++REGISTER_SCX_TEST(&exit_test)
+diff --git a/tools/testing/selftests/sched_ext/exit_test.h b/tools/testing/selftests/sched_ext/exit_test.h
+new file mode 100644
+index 000000000000..94f0268b9cb8
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/exit_test.h
+@@ -0,0 +1,20 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++
++#ifndef __EXIT_TEST_H__
++#define __EXIT_TEST_H__
++
++enum exit_test_case {
++ EXIT_SELECT_CPU,
++ EXIT_ENQUEUE,
++ EXIT_DISPATCH,
++ EXIT_ENABLE,
++ EXIT_INIT_TASK,
++ EXIT_INIT,
++ NUM_EXITS,
++};
++
++#endif // # __EXIT_TEST_H__
+diff --git a/tools/testing/selftests/sched_ext/hotplug.bpf.c b/tools/testing/selftests/sched_ext/hotplug.bpf.c
+new file mode 100644
+index 000000000000..8f2601db39f3
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/hotplug.bpf.c
+@@ -0,0 +1,61 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++#include "hotplug_test.h"
++
++UEI_DEFINE(uei);
++
++void BPF_STRUCT_OPS(hotplug_exit, struct scx_exit_info *ei)
++{
++ UEI_RECORD(uei, ei);
++}
++
++static void exit_from_hotplug(s32 cpu, bool onlining)
++{
++ /*
++ * Ignored, just used to verify that we can invoke blocking kfuncs
++ * from the hotplug path.
++ */
++ scx_bpf_create_dsq(0, -1);
++
++ s64 code = SCX_ECODE_ACT_RESTART | HOTPLUG_EXIT_RSN;
++
++ if (onlining)
++ code |= HOTPLUG_ONLINING;
++
++ scx_bpf_exit(code, "hotplug event detected (%d going %s)", cpu,
++ onlining ? "online" : "offline");
++}
++
++void BPF_STRUCT_OPS_SLEEPABLE(hotplug_cpu_online, s32 cpu)
++{
++ exit_from_hotplug(cpu, true);
++}
++
++void BPF_STRUCT_OPS_SLEEPABLE(hotplug_cpu_offline, s32 cpu)
++{
++ exit_from_hotplug(cpu, false);
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops hotplug_cb_ops = {
++ .cpu_online = hotplug_cpu_online,
++ .cpu_offline = hotplug_cpu_offline,
++ .exit = hotplug_exit,
++ .name = "hotplug_cbs",
++ .timeout_ms = 1000U,
++};
++
++SEC(".struct_ops.link")
++struct sched_ext_ops hotplug_nocb_ops = {
++ .exit = hotplug_exit,
++ .name = "hotplug_nocbs",
++ .timeout_ms = 1000U,
++};
+diff --git a/tools/testing/selftests/sched_ext/hotplug.c b/tools/testing/selftests/sched_ext/hotplug.c
+new file mode 100644
+index 000000000000..87bf220b1bce
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/hotplug.c
+@@ -0,0 +1,168 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#include <bpf/bpf.h>
++#include <sched.h>
++#include <scx/common.h>
++#include <sched.h>
++#include <sys/wait.h>
++#include <unistd.h>
++
++#include "hotplug_test.h"
++#include "hotplug.bpf.skel.h"
++#include "scx_test.h"
++#include "util.h"
++
++const char *online_path = "/sys/devices/system/cpu/cpu1/online";
++
++static bool is_cpu_online(void)
++{
++ return file_read_long(online_path) > 0;
++}
++
++static void toggle_online_status(bool online)
++{
++ long val = online ? 1 : 0;
++ int ret;
++
++ ret = file_write_long(online_path, val);
++ if (ret != 0)
++ fprintf(stderr, "Failed to bring CPU %s (%s)",
++ online ? "online" : "offline", strerror(errno));
++}
++
++static enum scx_test_status setup(void **ctx)
++{
++ if (!is_cpu_online())
++ return SCX_TEST_SKIP;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status test_hotplug(bool onlining, bool cbs_defined)
++{
++ struct hotplug *skel;
++ struct bpf_link *link;
++ long kind, code;
++
++ SCX_ASSERT(is_cpu_online());
++
++ skel = hotplug__open_and_load();
++ SCX_ASSERT(skel);
++
++ /* Testing the offline -> online path, so go offline before starting */
++ if (onlining)
++ toggle_online_status(0);
++
++ if (cbs_defined) {
++ kind = SCX_KIND_VAL(SCX_EXIT_UNREG_BPF);
++ code = SCX_ECODE_VAL(SCX_ECODE_ACT_RESTART) | HOTPLUG_EXIT_RSN;
++ if (onlining)
++ code |= HOTPLUG_ONLINING;
++ } else {
++ kind = SCX_KIND_VAL(SCX_EXIT_UNREG_KERN);
++ code = SCX_ECODE_VAL(SCX_ECODE_ACT_RESTART) |
++ SCX_ECODE_VAL(SCX_ECODE_RSN_HOTPLUG);
++ }
++
++ if (cbs_defined)
++ link = bpf_map__attach_struct_ops(skel->maps.hotplug_cb_ops);
++ else
++ link = bpf_map__attach_struct_ops(skel->maps.hotplug_nocb_ops);
++
++ if (!link) {
++ SCX_ERR("Failed to attach scheduler");
++ hotplug__destroy(skel);
++ return SCX_TEST_FAIL;
++ }
++
++ toggle_online_status(onlining ? 1 : 0);
++
++ while (!UEI_EXITED(skel, uei))
++ sched_yield();
++
++ SCX_EQ(skel->data->uei.kind, kind);
++ SCX_EQ(UEI_REPORT(skel, uei), code);
++
++ if (!onlining)
++ toggle_online_status(1);
++
++ bpf_link__destroy(link);
++ hotplug__destroy(skel);
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status test_hotplug_attach(void)
++{
++ struct hotplug *skel;
++ struct bpf_link *link;
++ enum scx_test_status status = SCX_TEST_PASS;
++ long kind, code;
++
++ SCX_ASSERT(is_cpu_online());
++ SCX_ASSERT(scx_hotplug_seq() > 0);
++
++ skel = SCX_OPS_OPEN(hotplug_nocb_ops, hotplug);
++ SCX_ASSERT(skel);
++
++ SCX_OPS_LOAD(skel, hotplug_nocb_ops, hotplug, uei);
++
++ /*
++ * Take the CPU offline to increment the global hotplug seq, which
++ * should cause attach to fail due to us setting the hotplug seq above
++ */
++ toggle_online_status(0);
++ link = bpf_map__attach_struct_ops(skel->maps.hotplug_nocb_ops);
++
++ toggle_online_status(1);
++
++ SCX_ASSERT(link);
++ while (!UEI_EXITED(skel, uei))
++ sched_yield();
++
++ kind = SCX_KIND_VAL(SCX_EXIT_UNREG_KERN);
++ code = SCX_ECODE_VAL(SCX_ECODE_ACT_RESTART) |
++ SCX_ECODE_VAL(SCX_ECODE_RSN_HOTPLUG);
++ SCX_EQ(skel->data->uei.kind, kind);
++ SCX_EQ(UEI_REPORT(skel, uei), code);
++
++ bpf_link__destroy(link);
++ hotplug__destroy(skel);
++
++ return status;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++
++#define HP_TEST(__onlining, __cbs_defined) ({ \
++ if (test_hotplug(__onlining, __cbs_defined) != SCX_TEST_PASS) \
++ return SCX_TEST_FAIL; \
++})
++
++ HP_TEST(true, true);
++ HP_TEST(false, true);
++ HP_TEST(true, false);
++ HP_TEST(false, false);
++
++#undef HP_TEST
++
++ return test_hotplug_attach();
++}
++
++static void cleanup(void *ctx)
++{
++ toggle_online_status(1);
++}
++
++struct scx_test hotplug_test = {
++ .name = "hotplug",
++ .description = "Verify hotplug behavior",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&hotplug_test)
+diff --git a/tools/testing/selftests/sched_ext/hotplug_test.h b/tools/testing/selftests/sched_ext/hotplug_test.h
+new file mode 100644
+index 000000000000..73d236f90787
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/hotplug_test.h
+@@ -0,0 +1,15 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++
++#ifndef __HOTPLUG_TEST_H__
++#define __HOTPLUG_TEST_H__
++
++enum hotplug_test_flags {
++ HOTPLUG_EXIT_RSN = 1LLU << 0,
++ HOTPLUG_ONLINING = 1LLU << 1,
++};
++
++#endif // # __HOTPLUG_TEST_H__
+diff --git a/tools/testing/selftests/sched_ext/init_enable_count.bpf.c b/tools/testing/selftests/sched_ext/init_enable_count.bpf.c
+new file mode 100644
+index 000000000000..47ea89a626c3
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/init_enable_count.bpf.c
+@@ -0,0 +1,53 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A scheduler that verifies that we do proper counting of init, enable, etc
++ * callbacks.
++ *
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++u64 init_task_cnt, exit_task_cnt, enable_cnt, disable_cnt;
++u64 init_fork_cnt, init_transition_cnt;
++
++s32 BPF_STRUCT_OPS_SLEEPABLE(cnt_init_task, struct task_struct *p,
++ struct scx_init_task_args *args)
++{
++ __sync_fetch_and_add(&init_task_cnt, 1);
++
++ if (args->fork)
++ __sync_fetch_and_add(&init_fork_cnt, 1);
++ else
++ __sync_fetch_and_add(&init_transition_cnt, 1);
++
++ return 0;
++}
++
++void BPF_STRUCT_OPS(cnt_exit_task, struct task_struct *p)
++{
++ __sync_fetch_and_add(&exit_task_cnt, 1);
++}
++
++void BPF_STRUCT_OPS(cnt_enable, struct task_struct *p)
++{
++ __sync_fetch_and_add(&enable_cnt, 1);
++}
++
++void BPF_STRUCT_OPS(cnt_disable, struct task_struct *p)
++{
++ __sync_fetch_and_add(&disable_cnt, 1);
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops init_enable_count_ops = {
++ .init_task = cnt_init_task,
++ .exit_task = cnt_exit_task,
++ .enable = cnt_enable,
++ .disable = cnt_disable,
++ .name = "init_enable_count",
++};
+diff --git a/tools/testing/selftests/sched_ext/init_enable_count.c b/tools/testing/selftests/sched_ext/init_enable_count.c
+new file mode 100644
+index 000000000000..97d45f1e5597
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/init_enable_count.c
+@@ -0,0 +1,166 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++#include <stdio.h>
++#include <unistd.h>
++#include <sched.h>
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include "scx_test.h"
++#include "init_enable_count.bpf.skel.h"
++
++#define SCHED_EXT 7
++
++static struct init_enable_count *
++open_load_prog(bool global)
++{
++ struct init_enable_count *skel;
++
++ skel = init_enable_count__open();
++ SCX_BUG_ON(!skel, "Failed to open skel");
++
++ if (!global)
++ skel->struct_ops.init_enable_count_ops->flags |= SCX_OPS_SWITCH_PARTIAL;
++
++ SCX_BUG_ON(init_enable_count__load(skel), "Failed to load skel");
++
++ return skel;
++}
++
++static enum scx_test_status run_test(bool global)
++{
++ struct init_enable_count *skel;
++ struct bpf_link *link;
++ const u32 num_children = 5, num_pre_forks = 1024;
++ int ret, i, status;
++ struct sched_param param = {};
++ pid_t pids[num_pre_forks];
++
++ skel = open_load_prog(global);
++
++ /*
++ * Fork a bunch of children before we attach the scheduler so that we
++ * ensure (at least in practical terms) that there are more tasks that
++ * transition from SCHED_OTHER -> SCHED_EXT than there are tasks that
++ * take the fork() path either below or in other processes.
++ */
++ for (i = 0; i < num_pre_forks; i++) {
++ pids[i] = fork();
++ SCX_FAIL_IF(pids[i] < 0, "Failed to fork child");
++ if (pids[i] == 0) {
++ sleep(1);
++ exit(0);
++ }
++ }
++
++ link = bpf_map__attach_struct_ops(skel->maps.init_enable_count_ops);
++ SCX_FAIL_IF(!link, "Failed to attach struct_ops");
++
++ for (i = 0; i < num_pre_forks; i++) {
++ SCX_FAIL_IF(waitpid(pids[i], &status, 0) != pids[i],
++ "Failed to wait for pre-forked child\n");
++
++ SCX_FAIL_IF(status != 0, "Pre-forked child %d exited with status %d\n", i,
++ status);
++ }
++
++ bpf_link__destroy(link);
++ SCX_GE(skel->bss->init_task_cnt, num_pre_forks);
++ SCX_GE(skel->bss->exit_task_cnt, num_pre_forks);
++
++ link = bpf_map__attach_struct_ops(skel->maps.init_enable_count_ops);
++ SCX_FAIL_IF(!link, "Failed to attach struct_ops");
++
++ /* SCHED_EXT children */
++ for (i = 0; i < num_children; i++) {
++ pids[i] = fork();
++ SCX_FAIL_IF(pids[i] < 0, "Failed to fork child");
++
++ if (pids[i] == 0) {
++ ret = sched_setscheduler(0, SCHED_EXT, &param);
++ SCX_BUG_ON(ret, "Failed to set sched to sched_ext");
++
++ /*
++ * Reset to SCHED_OTHER for half of them. Counts for
++ * everything should still be the same regardless, as
++ * ops.disable() is invoked even if a task is still on
++ * SCHED_EXT before it exits.
++ */
++ if (i % 2 == 0) {
++ ret = sched_setscheduler(0, SCHED_OTHER, &param);
++ SCX_BUG_ON(ret, "Failed to reset sched to normal");
++ }
++ exit(0);
++ }
++ }
++ for (i = 0; i < num_children; i++) {
++ SCX_FAIL_IF(waitpid(pids[i], &status, 0) != pids[i],
++ "Failed to wait for SCX child\n");
++
++ SCX_FAIL_IF(status != 0, "SCX child %d exited with status %d\n", i,
++ status);
++ }
++
++ /* SCHED_OTHER children */
++ for (i = 0; i < num_children; i++) {
++ pids[i] = fork();
++ if (pids[i] == 0)
++ exit(0);
++ }
++
++ for (i = 0; i < num_children; i++) {
++ SCX_FAIL_IF(waitpid(pids[i], &status, 0) != pids[i],
++ "Failed to wait for normal child\n");
++
++ SCX_FAIL_IF(status != 0, "Normal child %d exited with status %d\n", i,
++ status);
++ }
++
++ bpf_link__destroy(link);
++
++ SCX_GE(skel->bss->init_task_cnt, 2 * num_children);
++ SCX_GE(skel->bss->exit_task_cnt, 2 * num_children);
++
++ if (global) {
++ SCX_GE(skel->bss->enable_cnt, 2 * num_children);
++ SCX_GE(skel->bss->disable_cnt, 2 * num_children);
++ } else {
++ SCX_EQ(skel->bss->enable_cnt, num_children);
++ SCX_EQ(skel->bss->disable_cnt, num_children);
++ }
++ /*
++ * We forked a ton of tasks before we attached the scheduler above, so
++ * this should be fine. Technically it could be flaky if a ton of forks
++ * are happening at the same time in other processes, but that should
++ * be exceedingly unlikely.
++ */
++ SCX_GT(skel->bss->init_transition_cnt, skel->bss->init_fork_cnt);
++ SCX_GE(skel->bss->init_fork_cnt, 2 * num_children);
++
++ init_enable_count__destroy(skel);
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ enum scx_test_status status;
++
++ status = run_test(true);
++ if (status != SCX_TEST_PASS)
++ return status;
++
++ return run_test(false);
++}
++
++struct scx_test init_enable_count = {
++ .name = "init_enable_count",
++ .description = "Verify we do the correct amount of counting of init, "
++ "enable, etc callbacks.",
++ .run = run,
++};
++REGISTER_SCX_TEST(&init_enable_count)
+diff --git a/tools/testing/selftests/sched_ext/maximal.bpf.c b/tools/testing/selftests/sched_ext/maximal.bpf.c
+new file mode 100644
+index 000000000000..44612fdaf399
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/maximal.bpf.c
+@@ -0,0 +1,132 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A scheduler with every callback defined.
++ *
++ * This scheduler defines every callback.
++ *
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++s32 BPF_STRUCT_OPS(maximal_select_cpu, struct task_struct *p, s32 prev_cpu,
++ u64 wake_flags)
++{
++ return prev_cpu;
++}
++
++void BPF_STRUCT_OPS(maximal_enqueue, struct task_struct *p, u64 enq_flags)
++{
++ scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
++}
++
++void BPF_STRUCT_OPS(maximal_dequeue, struct task_struct *p, u64 deq_flags)
++{}
++
++void BPF_STRUCT_OPS(maximal_dispatch, s32 cpu, struct task_struct *prev)
++{
++ scx_bpf_consume(SCX_DSQ_GLOBAL);
++}
++
++void BPF_STRUCT_OPS(maximal_runnable, struct task_struct *p, u64 enq_flags)
++{}
++
++void BPF_STRUCT_OPS(maximal_running, struct task_struct *p)
++{}
++
++void BPF_STRUCT_OPS(maximal_stopping, struct task_struct *p, bool runnable)
++{}
++
++void BPF_STRUCT_OPS(maximal_quiescent, struct task_struct *p, u64 deq_flags)
++{}
++
++bool BPF_STRUCT_OPS(maximal_yield, struct task_struct *from,
++ struct task_struct *to)
++{
++ return false;
++}
++
++bool BPF_STRUCT_OPS(maximal_core_sched_before, struct task_struct *a,
++ struct task_struct *b)
++{
++ return false;
++}
++
++void BPF_STRUCT_OPS(maximal_set_weight, struct task_struct *p, u32 weight)
++{}
++
++void BPF_STRUCT_OPS(maximal_set_cpumask, struct task_struct *p,
++ const struct cpumask *cpumask)
++{}
++
++void BPF_STRUCT_OPS(maximal_update_idle, s32 cpu, bool idle)
++{}
++
++void BPF_STRUCT_OPS(maximal_cpu_acquire, s32 cpu,
++ struct scx_cpu_acquire_args *args)
++{}
++
++void BPF_STRUCT_OPS(maximal_cpu_release, s32 cpu,
++ struct scx_cpu_release_args *args)
++{}
++
++void BPF_STRUCT_OPS(maximal_cpu_online, s32 cpu)
++{}
++
++void BPF_STRUCT_OPS(maximal_cpu_offline, s32 cpu)
++{}
++
++s32 BPF_STRUCT_OPS(maximal_init_task, struct task_struct *p,
++ struct scx_init_task_args *args)
++{
++ return 0;
++}
++
++void BPF_STRUCT_OPS(maximal_enable, struct task_struct *p)
++{}
++
++void BPF_STRUCT_OPS(maximal_exit_task, struct task_struct *p,
++ struct scx_exit_task_args *args)
++{}
++
++void BPF_STRUCT_OPS(maximal_disable, struct task_struct *p)
++{}
++
++s32 BPF_STRUCT_OPS_SLEEPABLE(maximal_init)
++{
++ return 0;
++}
++
++void BPF_STRUCT_OPS(maximal_exit, struct scx_exit_info *info)
++{}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops maximal_ops = {
++ .select_cpu = maximal_select_cpu,
++ .enqueue = maximal_enqueue,
++ .dequeue = maximal_dequeue,
++ .dispatch = maximal_dispatch,
++ .runnable = maximal_runnable,
++ .running = maximal_running,
++ .stopping = maximal_stopping,
++ .quiescent = maximal_quiescent,
++ .yield = maximal_yield,
++ .core_sched_before = maximal_core_sched_before,
++ .set_weight = maximal_set_weight,
++ .set_cpumask = maximal_set_cpumask,
++ .update_idle = maximal_update_idle,
++ .cpu_acquire = maximal_cpu_acquire,
++ .cpu_release = maximal_cpu_release,
++ .cpu_online = maximal_cpu_online,
++ .cpu_offline = maximal_cpu_offline,
++ .init_task = maximal_init_task,
++ .enable = maximal_enable,
++ .exit_task = maximal_exit_task,
++ .disable = maximal_disable,
++ .init = maximal_init,
++ .exit = maximal_exit,
++ .name = "maximal",
++};
+diff --git a/tools/testing/selftests/sched_ext/maximal.c b/tools/testing/selftests/sched_ext/maximal.c
+new file mode 100644
+index 000000000000..f38fc973c380
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/maximal.c
+@@ -0,0 +1,51 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "maximal.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct maximal *skel;
++
++ skel = maximal__open_and_load();
++ SCX_FAIL_IF(!skel, "Failed to open and load skel");
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct maximal *skel = ctx;
++ struct bpf_link *link;
++
++ link = bpf_map__attach_struct_ops(skel->maps.maximal_ops);
++ SCX_FAIL_IF(!link, "Failed to attach scheduler");
++
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct maximal *skel = ctx;
++
++ maximal__destroy(skel);
++}
++
++struct scx_test maximal = {
++ .name = "maximal",
++ .description = "Verify we can load a scheduler with every callback defined",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&maximal)
+diff --git a/tools/testing/selftests/sched_ext/maybe_null.bpf.c b/tools/testing/selftests/sched_ext/maybe_null.bpf.c
+new file mode 100644
+index 000000000000..27d0f386acfb
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/maybe_null.bpf.c
+@@ -0,0 +1,36 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++u64 vtime_test;
++
++void BPF_STRUCT_OPS(maybe_null_running, struct task_struct *p)
++{}
++
++void BPF_STRUCT_OPS(maybe_null_success_dispatch, s32 cpu, struct task_struct *p)
++{
++ if (p != NULL)
++ vtime_test = p->scx.dsq_vtime;
++}
++
++bool BPF_STRUCT_OPS(maybe_null_success_yield, struct task_struct *from,
++ struct task_struct *to)
++{
++ if (to)
++ bpf_printk("Yielding to %s[%d]", to->comm, to->pid);
++
++ return false;
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops maybe_null_success = {
++ .dispatch = maybe_null_success_dispatch,
++ .yield = maybe_null_success_yield,
++ .enable = maybe_null_running,
++ .name = "minimal",
++};
+diff --git a/tools/testing/selftests/sched_ext/maybe_null.c b/tools/testing/selftests/sched_ext/maybe_null.c
+new file mode 100644
+index 000000000000..31cfafb0cf65
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/maybe_null.c
+@@ -0,0 +1,49 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "maybe_null.bpf.skel.h"
++#include "maybe_null_fail_dsp.bpf.skel.h"
++#include "maybe_null_fail_yld.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status run(void *ctx)
++{
++ struct maybe_null *skel;
++ struct maybe_null_fail_dsp *fail_dsp;
++ struct maybe_null_fail_yld *fail_yld;
++
++ skel = maybe_null__open_and_load();
++ if (!skel) {
++ SCX_ERR("Failed to open and load maybe_null skel");
++ return SCX_TEST_FAIL;
++ }
++ maybe_null__destroy(skel);
++
++ fail_dsp = maybe_null_fail_dsp__open_and_load();
++ if (fail_dsp) {
++ maybe_null_fail_dsp__destroy(fail_dsp);
++ SCX_ERR("Should failed to open and load maybe_null_fail_dsp skel");
++ return SCX_TEST_FAIL;
++ }
++
++ fail_yld = maybe_null_fail_yld__open_and_load();
++ if (fail_yld) {
++ maybe_null_fail_yld__destroy(fail_yld);
++ SCX_ERR("Should failed to open and load maybe_null_fail_yld skel");
++ return SCX_TEST_FAIL;
++ }
++
++ return SCX_TEST_PASS;
++}
++
++struct scx_test maybe_null = {
++ .name = "maybe_null",
++ .description = "Verify if PTR_MAYBE_NULL work for .dispatch",
++ .run = run,
++};
++REGISTER_SCX_TEST(&maybe_null)
+diff --git a/tools/testing/selftests/sched_ext/maybe_null_fail_dsp.bpf.c b/tools/testing/selftests/sched_ext/maybe_null_fail_dsp.bpf.c
+new file mode 100644
+index 000000000000..c0641050271d
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/maybe_null_fail_dsp.bpf.c
+@@ -0,0 +1,25 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++u64 vtime_test;
++
++void BPF_STRUCT_OPS(maybe_null_running, struct task_struct *p)
++{}
++
++void BPF_STRUCT_OPS(maybe_null_fail_dispatch, s32 cpu, struct task_struct *p)
++{
++ vtime_test = p->scx.dsq_vtime;
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops maybe_null_fail = {
++ .dispatch = maybe_null_fail_dispatch,
++ .enable = maybe_null_running,
++ .name = "maybe_null_fail_dispatch",
++};
+diff --git a/tools/testing/selftests/sched_ext/maybe_null_fail_yld.bpf.c b/tools/testing/selftests/sched_ext/maybe_null_fail_yld.bpf.c
+new file mode 100644
+index 000000000000..3c1740028e3b
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/maybe_null_fail_yld.bpf.c
+@@ -0,0 +1,28 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++u64 vtime_test;
++
++void BPF_STRUCT_OPS(maybe_null_running, struct task_struct *p)
++{}
++
++bool BPF_STRUCT_OPS(maybe_null_fail_yield, struct task_struct *from,
++ struct task_struct *to)
++{
++ bpf_printk("Yielding to %s[%d]", to->comm, to->pid);
++
++ return false;
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops maybe_null_fail = {
++ .yield = maybe_null_fail_yield,
++ .enable = maybe_null_running,
++ .name = "maybe_null_fail_yield",
++};
+diff --git a/tools/testing/selftests/sched_ext/minimal.bpf.c b/tools/testing/selftests/sched_ext/minimal.bpf.c
+new file mode 100644
+index 000000000000..6a7eccef0104
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/minimal.bpf.c
+@@ -0,0 +1,21 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A completely minimal scheduler.
++ *
++ * This scheduler defines the absolute minimal set of struct sched_ext_ops
++ * fields: its name. It should _not_ fail to be loaded, and can be used to
++ * exercise the default scheduling paths in ext.c.
++ *
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++SEC(".struct_ops.link")
++struct sched_ext_ops minimal_ops = {
++ .name = "minimal",
++};
+diff --git a/tools/testing/selftests/sched_ext/minimal.c b/tools/testing/selftests/sched_ext/minimal.c
+new file mode 100644
+index 000000000000..6c5db8ebbf8a
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/minimal.c
+@@ -0,0 +1,58 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "minimal.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct minimal *skel;
++
++ skel = minimal__open_and_load();
++ if (!skel) {
++ SCX_ERR("Failed to open and load skel");
++ return SCX_TEST_FAIL;
++ }
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct minimal *skel = ctx;
++ struct bpf_link *link;
++
++ link = bpf_map__attach_struct_ops(skel->maps.minimal_ops);
++ if (!link) {
++ SCX_ERR("Failed to attach scheduler");
++ return SCX_TEST_FAIL;
++ }
++
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct minimal *skel = ctx;
++
++ minimal__destroy(skel);
++}
++
++struct scx_test minimal = {
++ .name = "minimal",
++ .description = "Verify we can load a fully minimal scheduler",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&minimal)
+diff --git a/tools/testing/selftests/sched_ext/prog_run.bpf.c b/tools/testing/selftests/sched_ext/prog_run.bpf.c
+new file mode 100644
+index 000000000000..fd2c8f12af16
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/prog_run.bpf.c
+@@ -0,0 +1,32 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A scheduler that validates that we can invoke sched_ext kfuncs in
++ * BPF_PROG_TYPE_SYSCALL programs.
++ *
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++
++#include <scx/common.bpf.h>
++
++UEI_DEFINE(uei);
++
++char _license[] SEC("license") = "GPL";
++
++SEC("syscall")
++int BPF_PROG(prog_run_syscall)
++{
++ scx_bpf_exit(0xdeadbeef, "Exited from PROG_RUN");
++ return 0;
++}
++
++void BPF_STRUCT_OPS(prog_run_exit, struct scx_exit_info *ei)
++{
++ UEI_RECORD(uei, ei);
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops prog_run_ops = {
++ .exit = prog_run_exit,
++ .name = "prog_run",
++};
+diff --git a/tools/testing/selftests/sched_ext/prog_run.c b/tools/testing/selftests/sched_ext/prog_run.c
+new file mode 100644
+index 000000000000..3cd57ef8daaa
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/prog_run.c
+@@ -0,0 +1,78 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#include <bpf/bpf.h>
++#include <sched.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "prog_run.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct prog_run *skel;
++
++ skel = prog_run__open_and_load();
++ if (!skel) {
++ SCX_ERR("Failed to open and load skel");
++ return SCX_TEST_FAIL;
++ }
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct prog_run *skel = ctx;
++ struct bpf_link *link;
++ int prog_fd, err = 0;
++
++ prog_fd = bpf_program__fd(skel->progs.prog_run_syscall);
++ if (prog_fd < 0) {
++ SCX_ERR("Failed to get BPF_PROG_RUN prog");
++ return SCX_TEST_FAIL;
++ }
++
++ LIBBPF_OPTS(bpf_test_run_opts, topts);
++
++ link = bpf_map__attach_struct_ops(skel->maps.prog_run_ops);
++ if (!link) {
++ SCX_ERR("Failed to attach scheduler");
++ close(prog_fd);
++ return SCX_TEST_FAIL;
++ }
++
++ err = bpf_prog_test_run_opts(prog_fd, &topts);
++ SCX_EQ(err, 0);
++
++ /* Assumes uei.kind is written last */
++ while (skel->data->uei.kind == EXIT_KIND(SCX_EXIT_NONE))
++ sched_yield();
++
++ SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_UNREG_BPF));
++ SCX_EQ(skel->data->uei.exit_code, 0xdeadbeef);
++ close(prog_fd);
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct prog_run *skel = ctx;
++
++ prog_run__destroy(skel);
++}
++
++struct scx_test prog_run = {
++ .name = "prog_run",
++ .description = "Verify we can call into a scheduler with BPF_PROG_RUN, and invoke kfuncs",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&prog_run)
+diff --git a/tools/testing/selftests/sched_ext/reload_loop.c b/tools/testing/selftests/sched_ext/reload_loop.c
+new file mode 100644
+index 000000000000..5cfba2d6e056
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/reload_loop.c
+@@ -0,0 +1,75 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#include <bpf/bpf.h>
++#include <pthread.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "maximal.bpf.skel.h"
++#include "scx_test.h"
++
++static struct maximal *skel;
++static pthread_t threads[2];
++
++bool force_exit = false;
++
++static enum scx_test_status setup(void **ctx)
++{
++ skel = maximal__open_and_load();
++ if (!skel) {
++ SCX_ERR("Failed to open and load skel");
++ return SCX_TEST_FAIL;
++ }
++
++ return SCX_TEST_PASS;
++}
++
++static void *do_reload_loop(void *arg)
++{
++ u32 i;
++
++ for (i = 0; i < 1024 && !force_exit; i++) {
++ struct bpf_link *link;
++
++ link = bpf_map__attach_struct_ops(skel->maps.maximal_ops);
++ if (link)
++ bpf_link__destroy(link);
++ }
++
++ return NULL;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ int err;
++ void *ret;
++
++ err = pthread_create(&threads[0], NULL, do_reload_loop, NULL);
++ SCX_FAIL_IF(err, "Failed to create thread 0");
++
++ err = pthread_create(&threads[1], NULL, do_reload_loop, NULL);
++ SCX_FAIL_IF(err, "Failed to create thread 1");
++
++ SCX_FAIL_IF(pthread_join(threads[0], &ret), "thread 0 failed");
++ SCX_FAIL_IF(pthread_join(threads[1], &ret), "thread 1 failed");
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ force_exit = true;
++ maximal__destroy(skel);
++}
++
++struct scx_test reload_loop = {
++ .name = "reload_loop",
++ .description = "Stress test loading and unloading schedulers repeatedly in a tight loop",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&reload_loop)
+diff --git a/tools/testing/selftests/sched_ext/runner.c b/tools/testing/selftests/sched_ext/runner.c
+new file mode 100644
+index 000000000000..eab48c7ff309
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/runner.c
+@@ -0,0 +1,201 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
++ */
++#include <stdio.h>
++#include <unistd.h>
++#include <signal.h>
++#include <libgen.h>
++#include <bpf/bpf.h>
++#include "scx_test.h"
++
++const char help_fmt[] =
++"The runner for sched_ext tests.\n"
++"\n"
++"The runner is statically linked against all testcases, and runs them all serially.\n"
++"It's required for the testcases to be serial, as only a single host-wide sched_ext\n"
++"scheduler may be loaded at any given time."
++"\n"
++"Usage: %s [-t TEST] [-h]\n"
++"\n"
++" -t TEST Only run tests whose name includes this string\n"
++" -s Include print output for skipped tests\n"
++" -q Don't print the test descriptions during run\n"
++" -h Display this help and exit\n";
++
++static volatile int exit_req;
++static bool quiet, print_skipped;
++
++#define MAX_SCX_TESTS 2048
++
++static struct scx_test __scx_tests[MAX_SCX_TESTS];
++static unsigned __scx_num_tests = 0;
++
++static void sigint_handler(int simple)
++{
++ exit_req = 1;
++}
++
++static void print_test_preamble(const struct scx_test *test, bool quiet)
++{
++ printf("===== START =====\n");
++ printf("TEST: %s\n", test->name);
++ if (!quiet)
++ printf("DESCRIPTION: %s\n", test->description);
++ printf("OUTPUT:\n");
++}
++
++static const char *status_to_result(enum scx_test_status status)
++{
++ switch (status) {
++ case SCX_TEST_PASS:
++ case SCX_TEST_SKIP:
++ return "ok";
++ case SCX_TEST_FAIL:
++ return "not ok";
++ default:
++ return "<UNKNOWN>";
++ }
++}
++
++static void print_test_result(const struct scx_test *test,
++ enum scx_test_status status,
++ unsigned int testnum)
++{
++ const char *result = status_to_result(status);
++ const char *directive = status == SCX_TEST_SKIP ? "SKIP " : "";
++
++ printf("%s %u %s # %s\n", result, testnum, test->name, directive);
++ printf("===== END =====\n");
++}
++
++static bool should_skip_test(const struct scx_test *test, const char * filter)
++{
++ return !strstr(test->name, filter);
++}
++
++static enum scx_test_status run_test(const struct scx_test *test)
++{
++ enum scx_test_status status;
++ void *context = NULL;
++
++ if (test->setup) {
++ status = test->setup(&context);
++ if (status != SCX_TEST_PASS)
++ return status;
++ }
++
++ status = test->run(context);
++
++ if (test->cleanup)
++ test->cleanup(context);
++
++ return status;
++}
++
++static bool test_valid(const struct scx_test *test)
++{
++ if (!test) {
++ fprintf(stderr, "NULL test detected\n");
++ return false;
++ }
++
++ if (!test->name) {
++ fprintf(stderr,
++ "Test with no name found. Must specify test name.\n");
++ return false;
++ }
++
++ if (!test->description) {
++ fprintf(stderr, "Test %s requires description.\n", test->name);
++ return false;
++ }
++
++ if (!test->run) {
++ fprintf(stderr, "Test %s has no run() callback\n", test->name);
++ return false;
++ }
++
++ return true;
++}
++
++int main(int argc, char **argv)
++{
++ const char *filter = NULL;
++ unsigned testnum = 0, i;
++ unsigned passed = 0, skipped = 0, failed = 0;
++ int opt;
++
++ signal(SIGINT, sigint_handler);
++ signal(SIGTERM, sigint_handler);
++
++ libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
++
++ while ((opt = getopt(argc, argv, "qst:h")) != -1) {
++ switch (opt) {
++ case 'q':
++ quiet = true;
++ break;
++ case 's':
++ print_skipped = true;
++ break;
++ case 't':
++ filter = optarg;
++ break;
++ default:
++ fprintf(stderr, help_fmt, basename(argv[0]));
++ return opt != 'h';
++ }
++ }
++
++ for (i = 0; i < __scx_num_tests; i++) {
++ enum scx_test_status status;
++ struct scx_test *test = &__scx_tests[i];
++
++ if (filter && should_skip_test(test, filter)) {
++ /*
++ * Printing the skipped tests and their preambles can
++ * add a lot of noise to the runner output. Printing
++ * this is only really useful for CI, so let's skip it
++ * by default.
++ */
++ if (print_skipped) {
++ print_test_preamble(test, quiet);
++ print_test_result(test, SCX_TEST_SKIP, ++testnum);
++ }
++ continue;
++ }
++
++ print_test_preamble(test, quiet);
++ status = run_test(test);
++ print_test_result(test, status, ++testnum);
++ switch (status) {
++ case SCX_TEST_PASS:
++ passed++;
++ break;
++ case SCX_TEST_SKIP:
++ skipped++;
++ break;
++ case SCX_TEST_FAIL:
++ failed++;
++ break;
++ }
++ }
++ printf("\n\n=============================\n\n");
++ printf("RESULTS:\n\n");
++ printf("PASSED: %u\n", passed);
++ printf("SKIPPED: %u\n", skipped);
++ printf("FAILED: %u\n", failed);
++
++ return 0;
++}
++
++void scx_test_register(struct scx_test *test)
++{
++ SCX_BUG_ON(!test_valid(test), "Invalid test found");
++ SCX_BUG_ON(__scx_num_tests >= MAX_SCX_TESTS, "Maximum tests exceeded");
++
++ __scx_tests[__scx_num_tests++] = *test;
++}
+diff --git a/tools/testing/selftests/sched_ext/scx_test.h b/tools/testing/selftests/sched_ext/scx_test.h
+new file mode 100644
+index 000000000000..90b8d6915bb7
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/scx_test.h
+@@ -0,0 +1,131 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ */
++
++#ifndef __SCX_TEST_H__
++#define __SCX_TEST_H__
++
++#include <errno.h>
++#include <scx/common.h>
++#include <scx/compat.h>
++
++enum scx_test_status {
++ SCX_TEST_PASS = 0,
++ SCX_TEST_SKIP,
++ SCX_TEST_FAIL,
++};
++
++#define EXIT_KIND(__ent) __COMPAT_ENUM_OR_ZERO("scx_exit_kind", #__ent)
++
++struct scx_test {
++ /**
++ * name - The name of the testcase.
++ */
++ const char *name;
++
++ /**
++ * description - A description of your testcase: what it tests and is
++ * meant to validate.
++ */
++ const char *description;
++
++ /*
++ * setup - Setup the test.
++ * @ctx: A pointer to a context object that will be passed to run and
++ * cleanup.
++ *
++ * An optional callback that allows a testcase to perform setup for its
++ * run. A test may return SCX_TEST_SKIP to skip the run.
++ */
++ enum scx_test_status (*setup)(void **ctx);
++
++ /*
++ * run - Run the test.
++ * @ctx: Context set in the setup() callback. If @ctx was not set in
++ * setup(), it is NULL.
++ *
++ * The main test. Callers should return one of:
++ *
++ * - SCX_TEST_PASS: Test passed
++ * - SCX_TEST_SKIP: Test should be skipped
++ * - SCX_TEST_FAIL: Test failed
++ *
++ * This callback must be defined.
++ */
++ enum scx_test_status (*run)(void *ctx);
++
++ /*
++ * cleanup - Perform cleanup following the test
++ * @ctx: Context set in the setup() callback. If @ctx was not set in
++ * setup(), it is NULL.
++ *
++ * An optional callback that allows a test to perform cleanup after
++ * being run. This callback is run even if the run() callback returns
++ * SCX_TEST_SKIP or SCX_TEST_FAIL. It is not run if setup() returns
++ * SCX_TEST_SKIP or SCX_TEST_FAIL.
++ */
++ void (*cleanup)(void *ctx);
++};
++
++void scx_test_register(struct scx_test *test);
++
++#define REGISTER_SCX_TEST(__test) \
++ __attribute__((constructor)) \
++ static void ___scxregister##__LINE__(void) \
++ { \
++ scx_test_register(__test); \
++ }
++
++#define SCX_ERR(__fmt, ...) \
++ do { \
++ fprintf(stderr, "ERR: %s:%d\n", __FILE__, __LINE__); \
++ fprintf(stderr, __fmt"\n", ##__VA_ARGS__); \
++ } while (0)
++
++#define SCX_FAIL(__fmt, ...) \
++ do { \
++ SCX_ERR(__fmt, ##__VA_ARGS__); \
++ return SCX_TEST_FAIL; \
++ } while (0)
++
++#define SCX_FAIL_IF(__cond, __fmt, ...) \
++ do { \
++ if (__cond) \
++ SCX_FAIL(__fmt, ##__VA_ARGS__); \
++ } while (0)
++
++#define SCX_GT(_x, _y) SCX_FAIL_IF((_x) <= (_y), "Expected %s > %s (%lu > %lu)", \
++ #_x, #_y, (u64)(_x), (u64)(_y))
++#define SCX_GE(_x, _y) SCX_FAIL_IF((_x) < (_y), "Expected %s >= %s (%lu >= %lu)", \
++ #_x, #_y, (u64)(_x), (u64)(_y))
++#define SCX_LT(_x, _y) SCX_FAIL_IF((_x) >= (_y), "Expected %s < %s (%lu < %lu)", \
++ #_x, #_y, (u64)(_x), (u64)(_y))
++#define SCX_LE(_x, _y) SCX_FAIL_IF((_x) > (_y), "Expected %s <= %s (%lu <= %lu)", \
++ #_x, #_y, (u64)(_x), (u64)(_y))
++#define SCX_EQ(_x, _y) SCX_FAIL_IF((_x) != (_y), "Expected %s == %s (%lu == %lu)", \
++ #_x, #_y, (u64)(_x), (u64)(_y))
++#define SCX_ASSERT(_x) SCX_FAIL_IF(!(_x), "Expected %s to be true (%lu)", \
++ #_x, (u64)(_x))
++
++#define SCX_ECODE_VAL(__ecode) ({ \
++ u64 __val = 0; \
++ bool __found = false; \
++ \
++ __found = __COMPAT_read_enum("scx_exit_code", #__ecode, &__val); \
++ SCX_ASSERT(__found); \
++ (s64)__val; \
++})
++
++#define SCX_KIND_VAL(__kind) ({ \
++ u64 __val = 0; \
++ bool __found = false; \
++ \
++ __found = __COMPAT_read_enum("scx_exit_kind", #__kind, &__val); \
++ SCX_ASSERT(__found); \
++ __val; \
++})
++
++#endif // # __SCX_TEST_H__
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_dfl.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_dfl.bpf.c
+new file mode 100644
+index 000000000000..2ed2991afafe
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_dfl.bpf.c
+@@ -0,0 +1,40 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A scheduler that validates the behavior of direct dispatching with a default
++ * select_cpu implementation.
++ *
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++bool saw_local = false;
++
++static bool task_is_test(const struct task_struct *p)
++{
++ return !bpf_strncmp(p->comm, 9, "select_cpu");
++}
++
++void BPF_STRUCT_OPS(select_cpu_dfl_enqueue, struct task_struct *p,
++ u64 enq_flags)
++{
++ const struct cpumask *idle_mask = scx_bpf_get_idle_cpumask();
++
++ if (task_is_test(p) &&
++ bpf_cpumask_test_cpu(scx_bpf_task_cpu(p), idle_mask)) {
++ saw_local = true;
++ }
++ scx_bpf_put_idle_cpumask(idle_mask);
++
++ scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, enq_flags);
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops select_cpu_dfl_ops = {
++ .enqueue = select_cpu_dfl_enqueue,
++ .name = "select_cpu_dfl",
++};
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_dfl.c b/tools/testing/selftests/sched_ext/select_cpu_dfl.c
+new file mode 100644
+index 000000000000..a53a40c2d2f0
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_dfl.c
+@@ -0,0 +1,72 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "select_cpu_dfl.bpf.skel.h"
++#include "scx_test.h"
++
++#define NUM_CHILDREN 1028
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct select_cpu_dfl *skel;
++
++ skel = select_cpu_dfl__open_and_load();
++ SCX_FAIL_IF(!skel, "Failed to open and load skel");
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct select_cpu_dfl *skel = ctx;
++ struct bpf_link *link;
++ pid_t pids[NUM_CHILDREN];
++ int i, status;
++
++ link = bpf_map__attach_struct_ops(skel->maps.select_cpu_dfl_ops);
++ SCX_FAIL_IF(!link, "Failed to attach scheduler");
++
++ for (i = 0; i < NUM_CHILDREN; i++) {
++ pids[i] = fork();
++ if (pids[i] == 0) {
++ sleep(1);
++ exit(0);
++ }
++ }
++
++ for (i = 0; i < NUM_CHILDREN; i++) {
++ SCX_EQ(waitpid(pids[i], &status, 0), pids[i]);
++ SCX_EQ(status, 0);
++ }
++
++ SCX_ASSERT(!skel->bss->saw_local);
++
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct select_cpu_dfl *skel = ctx;
++
++ select_cpu_dfl__destroy(skel);
++}
++
++struct scx_test select_cpu_dfl = {
++ .name = "select_cpu_dfl",
++ .description = "Verify the default ops.select_cpu() dispatches tasks "
++ "when idles cores are found, and skips ops.enqueue()",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&select_cpu_dfl)
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.bpf.c
+new file mode 100644
+index 000000000000..4bb5abb2d369
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.bpf.c
+@@ -0,0 +1,89 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A scheduler that validates the behavior of direct dispatching with a default
++ * select_cpu implementation, and with the SCX_OPS_ENQ_DFL_NO_DISPATCH ops flag
++ * specified.
++ *
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++bool saw_local = false;
++
++/* Per-task scheduling context */
++struct task_ctx {
++ bool force_local; /* CPU changed by ops.select_cpu() */
++};
++
++struct {
++ __uint(type, BPF_MAP_TYPE_TASK_STORAGE);
++ __uint(map_flags, BPF_F_NO_PREALLOC);
++ __type(key, int);
++ __type(value, struct task_ctx);
++} task_ctx_stor SEC(".maps");
++
++/* Manually specify the signature until the kfunc is added to the scx repo. */
++s32 scx_bpf_select_cpu_dfl(struct task_struct *p, s32 prev_cpu, u64 wake_flags,
++ bool *found) __ksym;
++
++s32 BPF_STRUCT_OPS(select_cpu_dfl_nodispatch_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ struct task_ctx *tctx;
++ s32 cpu;
++
++ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
++ if (!tctx) {
++ scx_bpf_error("task_ctx lookup failed");
++ return -ESRCH;
++ }
++
++ cpu = scx_bpf_select_cpu_dfl(p, prev_cpu, wake_flags,
++ &tctx->force_local);
++
++ return cpu;
++}
++
++void BPF_STRUCT_OPS(select_cpu_dfl_nodispatch_enqueue, struct task_struct *p,
++ u64 enq_flags)
++{
++ u64 dsq_id = SCX_DSQ_GLOBAL;
++ struct task_ctx *tctx;
++
++ tctx = bpf_task_storage_get(&task_ctx_stor, p, 0, 0);
++ if (!tctx) {
++ scx_bpf_error("task_ctx lookup failed");
++ return;
++ }
++
++ if (tctx->force_local) {
++ dsq_id = SCX_DSQ_LOCAL;
++ tctx->force_local = false;
++ saw_local = true;
++ }
++
++ scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, enq_flags);
++}
++
++s32 BPF_STRUCT_OPS(select_cpu_dfl_nodispatch_init_task,
++ struct task_struct *p, struct scx_init_task_args *args)
++{
++ if (bpf_task_storage_get(&task_ctx_stor, p, 0,
++ BPF_LOCAL_STORAGE_GET_F_CREATE))
++ return 0;
++ else
++ return -ENOMEM;
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops select_cpu_dfl_nodispatch_ops = {
++ .select_cpu = select_cpu_dfl_nodispatch_select_cpu,
++ .enqueue = select_cpu_dfl_nodispatch_enqueue,
++ .init_task = select_cpu_dfl_nodispatch_init_task,
++ .name = "select_cpu_dfl_nodispatch",
++};
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.c b/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.c
+new file mode 100644
+index 000000000000..1d85bf4bf3a3
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_dfl_nodispatch.c
+@@ -0,0 +1,72 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "select_cpu_dfl_nodispatch.bpf.skel.h"
++#include "scx_test.h"
++
++#define NUM_CHILDREN 1028
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct select_cpu_dfl_nodispatch *skel;
++
++ skel = select_cpu_dfl_nodispatch__open_and_load();
++ SCX_FAIL_IF(!skel, "Failed to open and load skel");
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct select_cpu_dfl_nodispatch *skel = ctx;
++ struct bpf_link *link;
++ pid_t pids[NUM_CHILDREN];
++ int i, status;
++
++ link = bpf_map__attach_struct_ops(skel->maps.select_cpu_dfl_nodispatch_ops);
++ SCX_FAIL_IF(!link, "Failed to attach scheduler");
++
++ for (i = 0; i < NUM_CHILDREN; i++) {
++ pids[i] = fork();
++ if (pids[i] == 0) {
++ sleep(1);
++ exit(0);
++ }
++ }
++
++ for (i = 0; i < NUM_CHILDREN; i++) {
++ SCX_EQ(waitpid(pids[i], &status, 0), pids[i]);
++ SCX_EQ(status, 0);
++ }
++
++ SCX_ASSERT(skel->bss->saw_local);
++
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct select_cpu_dfl_nodispatch *skel = ctx;
++
++ select_cpu_dfl_nodispatch__destroy(skel);
++}
++
++struct scx_test select_cpu_dfl_nodispatch = {
++ .name = "select_cpu_dfl_nodispatch",
++ .description = "Verify behavior of scx_bpf_select_cpu_dfl() in "
++ "ops.select_cpu()",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&select_cpu_dfl_nodispatch)
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch.bpf.c
+new file mode 100644
+index 000000000000..f0b96a4a04b2
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch.bpf.c
+@@ -0,0 +1,41 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A scheduler that validates the behavior of direct dispatching with a default
++ * select_cpu implementation.
++ *
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++s32 BPF_STRUCT_OPS(select_cpu_dispatch_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ u64 dsq_id = SCX_DSQ_LOCAL;
++ s32 cpu = prev_cpu;
++
++ if (scx_bpf_test_and_clear_cpu_idle(cpu))
++ goto dispatch;
++
++ cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0);
++ if (cpu >= 0)
++ goto dispatch;
++
++ dsq_id = SCX_DSQ_GLOBAL;
++ cpu = prev_cpu;
++
++dispatch:
++ scx_bpf_dispatch(p, dsq_id, SCX_SLICE_DFL, 0);
++ return cpu;
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops select_cpu_dispatch_ops = {
++ .select_cpu = select_cpu_dispatch_select_cpu,
++ .name = "select_cpu_dispatch",
++ .timeout_ms = 1000U,
++};
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch.c
+new file mode 100644
+index 000000000000..0309ca8785b3
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch.c
+@@ -0,0 +1,70 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "select_cpu_dispatch.bpf.skel.h"
++#include "scx_test.h"
++
++#define NUM_CHILDREN 1028
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct select_cpu_dispatch *skel;
++
++ skel = select_cpu_dispatch__open_and_load();
++ SCX_FAIL_IF(!skel, "Failed to open and load skel");
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct select_cpu_dispatch *skel = ctx;
++ struct bpf_link *link;
++ pid_t pids[NUM_CHILDREN];
++ int i, status;
++
++ link = bpf_map__attach_struct_ops(skel->maps.select_cpu_dispatch_ops);
++ SCX_FAIL_IF(!link, "Failed to attach scheduler");
++
++ for (i = 0; i < NUM_CHILDREN; i++) {
++ pids[i] = fork();
++ if (pids[i] == 0) {
++ sleep(1);
++ exit(0);
++ }
++ }
++
++ for (i = 0; i < NUM_CHILDREN; i++) {
++ SCX_EQ(waitpid(pids[i], &status, 0), pids[i]);
++ SCX_EQ(status, 0);
++ }
++
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct select_cpu_dispatch *skel = ctx;
++
++ select_cpu_dispatch__destroy(skel);
++}
++
++struct scx_test select_cpu_dispatch = {
++ .name = "select_cpu_dispatch",
++ .description = "Test direct dispatching to built-in DSQs from "
++ "ops.select_cpu()",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&select_cpu_dispatch)
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.bpf.c
+new file mode 100644
+index 000000000000..7b42ddce0f56
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.bpf.c
+@@ -0,0 +1,37 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A scheduler that validates the behavior of direct dispatching with a default
++ * select_cpu implementation.
++ *
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++UEI_DEFINE(uei);
++
++s32 BPF_STRUCT_OPS(select_cpu_dispatch_bad_dsq_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ /* Dispatching to a random DSQ should fail. */
++ scx_bpf_dispatch(p, 0xcafef00d, SCX_SLICE_DFL, 0);
++
++ return prev_cpu;
++}
++
++void BPF_STRUCT_OPS(select_cpu_dispatch_bad_dsq_exit, struct scx_exit_info *ei)
++{
++ UEI_RECORD(uei, ei);
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops select_cpu_dispatch_bad_dsq_ops = {
++ .select_cpu = select_cpu_dispatch_bad_dsq_select_cpu,
++ .exit = select_cpu_dispatch_bad_dsq_exit,
++ .name = "select_cpu_dispatch_bad_dsq",
++ .timeout_ms = 1000U,
++};
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.c
+new file mode 100644
+index 000000000000..47eb6ed7627d
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch_bad_dsq.c
+@@ -0,0 +1,56 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "select_cpu_dispatch_bad_dsq.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct select_cpu_dispatch_bad_dsq *skel;
++
++ skel = select_cpu_dispatch_bad_dsq__open_and_load();
++ SCX_FAIL_IF(!skel, "Failed to open and load skel");
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct select_cpu_dispatch_bad_dsq *skel = ctx;
++ struct bpf_link *link;
++
++ link = bpf_map__attach_struct_ops(skel->maps.select_cpu_dispatch_bad_dsq_ops);
++ SCX_FAIL_IF(!link, "Failed to attach scheduler");
++
++ sleep(1);
++
++ SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_ERROR));
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct select_cpu_dispatch_bad_dsq *skel = ctx;
++
++ select_cpu_dispatch_bad_dsq__destroy(skel);
++}
++
++struct scx_test select_cpu_dispatch_bad_dsq = {
++ .name = "select_cpu_dispatch_bad_dsq",
++ .description = "Verify graceful failure if we direct-dispatch to a "
++ "bogus DSQ in ops.select_cpu()",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&select_cpu_dispatch_bad_dsq)
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.bpf.c
+new file mode 100644
+index 000000000000..653e3dc0b4dc
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.bpf.c
+@@ -0,0 +1,38 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A scheduler that validates the behavior of direct dispatching with a default
++ * select_cpu implementation.
++ *
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++UEI_DEFINE(uei);
++
++s32 BPF_STRUCT_OPS(select_cpu_dispatch_dbl_dsp_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ /* Dispatching twice in a row is disallowed. */
++ scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
++ scx_bpf_dispatch(p, SCX_DSQ_GLOBAL, SCX_SLICE_DFL, 0);
++
++ return prev_cpu;
++}
++
++void BPF_STRUCT_OPS(select_cpu_dispatch_dbl_dsp_exit, struct scx_exit_info *ei)
++{
++ UEI_RECORD(uei, ei);
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops select_cpu_dispatch_dbl_dsp_ops = {
++ .select_cpu = select_cpu_dispatch_dbl_dsp_select_cpu,
++ .exit = select_cpu_dispatch_dbl_dsp_exit,
++ .name = "select_cpu_dispatch_dbl_dsp",
++ .timeout_ms = 1000U,
++};
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.c b/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.c
+new file mode 100644
+index 000000000000..48ff028a3c46
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_dispatch_dbl_dsp.c
+@@ -0,0 +1,56 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2023 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2023 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2023 Tejun Heo <tj@kernel.org>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "select_cpu_dispatch_dbl_dsp.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct select_cpu_dispatch_dbl_dsp *skel;
++
++ skel = select_cpu_dispatch_dbl_dsp__open_and_load();
++ SCX_FAIL_IF(!skel, "Failed to open and load skel");
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct select_cpu_dispatch_dbl_dsp *skel = ctx;
++ struct bpf_link *link;
++
++ link = bpf_map__attach_struct_ops(skel->maps.select_cpu_dispatch_dbl_dsp_ops);
++ SCX_FAIL_IF(!link, "Failed to attach scheduler");
++
++ sleep(1);
++
++ SCX_EQ(skel->data->uei.kind, EXIT_KIND(SCX_EXIT_ERROR));
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct select_cpu_dispatch_dbl_dsp *skel = ctx;
++
++ select_cpu_dispatch_dbl_dsp__destroy(skel);
++}
++
++struct scx_test select_cpu_dispatch_dbl_dsp = {
++ .name = "select_cpu_dispatch_dbl_dsp",
++ .description = "Verify graceful failure if we dispatch twice to a "
++ "DSQ in ops.select_cpu()",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&select_cpu_dispatch_dbl_dsp)
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_vtime.bpf.c b/tools/testing/selftests/sched_ext/select_cpu_vtime.bpf.c
+new file mode 100644
+index 000000000000..7f3ebf4fc2ea
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_vtime.bpf.c
+@@ -0,0 +1,92 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * A scheduler that validates that enqueue flags are properly stored and
++ * applied at dispatch time when a task is directly dispatched from
++ * ops.select_cpu(). We validate this by using scx_bpf_dispatch_vtime(), and
++ * making the test a very basic vtime scheduler.
++ *
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
++ */
++
++#include <scx/common.bpf.h>
++
++char _license[] SEC("license") = "GPL";
++
++volatile bool consumed;
++
++static u64 vtime_now;
++
++#define VTIME_DSQ 0
++
++static inline bool vtime_before(u64 a, u64 b)
++{
++ return (s64)(a - b) < 0;
++}
++
++static inline u64 task_vtime(const struct task_struct *p)
++{
++ u64 vtime = p->scx.dsq_vtime;
++
++ if (vtime_before(vtime, vtime_now - SCX_SLICE_DFL))
++ return vtime_now - SCX_SLICE_DFL;
++ else
++ return vtime;
++}
++
++s32 BPF_STRUCT_OPS(select_cpu_vtime_select_cpu, struct task_struct *p,
++ s32 prev_cpu, u64 wake_flags)
++{
++ s32 cpu;
++
++ cpu = scx_bpf_pick_idle_cpu(p->cpus_ptr, 0);
++ if (cpu >= 0)
++ goto ddsp;
++
++ cpu = prev_cpu;
++ scx_bpf_test_and_clear_cpu_idle(cpu);
++ddsp:
++ scx_bpf_dispatch_vtime(p, VTIME_DSQ, SCX_SLICE_DFL, task_vtime(p), 0);
++ return cpu;
++}
++
++void BPF_STRUCT_OPS(select_cpu_vtime_dispatch, s32 cpu, struct task_struct *p)
++{
++ if (scx_bpf_consume(VTIME_DSQ))
++ consumed = true;
++}
++
++void BPF_STRUCT_OPS(select_cpu_vtime_running, struct task_struct *p)
++{
++ if (vtime_before(vtime_now, p->scx.dsq_vtime))
++ vtime_now = p->scx.dsq_vtime;
++}
++
++void BPF_STRUCT_OPS(select_cpu_vtime_stopping, struct task_struct *p,
++ bool runnable)
++{
++ p->scx.dsq_vtime += (SCX_SLICE_DFL - p->scx.slice) * 100 / p->scx.weight;
++}
++
++void BPF_STRUCT_OPS(select_cpu_vtime_enable, struct task_struct *p)
++{
++ p->scx.dsq_vtime = vtime_now;
++}
++
++s32 BPF_STRUCT_OPS_SLEEPABLE(select_cpu_vtime_init)
++{
++ return scx_bpf_create_dsq(VTIME_DSQ, -1);
++}
++
++SEC(".struct_ops.link")
++struct sched_ext_ops select_cpu_vtime_ops = {
++ .select_cpu = select_cpu_vtime_select_cpu,
++ .dispatch = select_cpu_vtime_dispatch,
++ .running = select_cpu_vtime_running,
++ .stopping = select_cpu_vtime_stopping,
++ .enable = select_cpu_vtime_enable,
++ .init = select_cpu_vtime_init,
++ .name = "select_cpu_vtime",
++ .timeout_ms = 1000U,
++};
+diff --git a/tools/testing/selftests/sched_ext/select_cpu_vtime.c b/tools/testing/selftests/sched_ext/select_cpu_vtime.c
+new file mode 100644
+index 000000000000..b4629c2364f5
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/select_cpu_vtime.c
+@@ -0,0 +1,59 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include <sys/wait.h>
++#include <unistd.h>
++#include "select_cpu_vtime.bpf.skel.h"
++#include "scx_test.h"
++
++static enum scx_test_status setup(void **ctx)
++{
++ struct select_cpu_vtime *skel;
++
++ skel = select_cpu_vtime__open_and_load();
++ SCX_FAIL_IF(!skel, "Failed to open and load skel");
++ *ctx = skel;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ struct select_cpu_vtime *skel = ctx;
++ struct bpf_link *link;
++
++ SCX_ASSERT(!skel->bss->consumed);
++
++ link = bpf_map__attach_struct_ops(skel->maps.select_cpu_vtime_ops);
++ SCX_FAIL_IF(!link, "Failed to attach scheduler");
++
++ sleep(1);
++
++ SCX_ASSERT(skel->bss->consumed);
++
++ bpf_link__destroy(link);
++
++ return SCX_TEST_PASS;
++}
++
++static void cleanup(void *ctx)
++{
++ struct select_cpu_vtime *skel = ctx;
++
++ select_cpu_vtime__destroy(skel);
++}
++
++struct scx_test select_cpu_vtime = {
++ .name = "select_cpu_vtime",
++ .description = "Test doing direct vtime-dispatching from "
++ "ops.select_cpu(), to a non-built-in DSQ",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&select_cpu_vtime)
+diff --git a/tools/testing/selftests/sched_ext/test_example.c b/tools/testing/selftests/sched_ext/test_example.c
+new file mode 100644
+index 000000000000..ce36cdf03cdc
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/test_example.c
+@@ -0,0 +1,49 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 Tejun Heo <tj@kernel.org>
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#include <bpf/bpf.h>
++#include <scx/common.h>
++#include "scx_test.h"
++
++static bool setup_called = false;
++static bool run_called = false;
++static bool cleanup_called = false;
++
++static int context = 10;
++
++static enum scx_test_status setup(void **ctx)
++{
++ setup_called = true;
++ *ctx = &context;
++
++ return SCX_TEST_PASS;
++}
++
++static enum scx_test_status run(void *ctx)
++{
++ int *arg = ctx;
++
++ SCX_ASSERT(setup_called);
++ SCX_ASSERT(!run_called && !cleanup_called);
++ SCX_EQ(*arg, context);
++
++ run_called = true;
++ return SCX_TEST_PASS;
++}
++
++static void cleanup (void *ctx)
++{
++ SCX_BUG_ON(!run_called || cleanup_called, "Wrong callbacks invoked");
++}
++
++struct scx_test example = {
++ .name = "example",
++ .description = "Validate the basic function of the test suite itself",
++ .setup = setup,
++ .run = run,
++ .cleanup = cleanup,
++};
++REGISTER_SCX_TEST(&example)
+diff --git a/tools/testing/selftests/sched_ext/util.c b/tools/testing/selftests/sched_ext/util.c
+new file mode 100644
+index 000000000000..e47769c91918
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/util.c
+@@ -0,0 +1,71 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <dvernet@meta.com>
++ */
++#include <errno.h>
++#include <fcntl.h>
++#include <stdio.h>
++#include <stdlib.h>
++#include <string.h>
++#include <unistd.h>
++
++/* Returns read len on success, or -errno on failure. */
++static ssize_t read_text(const char *path, char *buf, size_t max_len)
++{
++ ssize_t len;
++ int fd;
++
++ fd = open(path, O_RDONLY);
++ if (fd < 0)
++ return -errno;
++
++ len = read(fd, buf, max_len - 1);
++
++ if (len >= 0)
++ buf[len] = 0;
++
++ close(fd);
++ return len < 0 ? -errno : len;
++}
++
++/* Returns written len on success, or -errno on failure. */
++static ssize_t write_text(const char *path, char *buf, ssize_t len)
++{
++ int fd;
++ ssize_t written;
++
++ fd = open(path, O_WRONLY | O_APPEND);
++ if (fd < 0)
++ return -errno;
++
++ written = write(fd, buf, len);
++ close(fd);
++ return written < 0 ? -errno : written;
++}
++
++long file_read_long(const char *path)
++{
++ char buf[128];
++
++
++ if (read_text(path, buf, sizeof(buf)) <= 0)
++ return -1;
++
++ return atol(buf);
++}
++
++int file_write_long(const char *path, long val)
++{
++ char buf[64];
++ int ret;
++
++ ret = sprintf(buf, "%lu", val);
++ if (ret < 0)
++ return ret;
++
++ if (write_text(path, buf, sizeof(buf)) <= 0)
++ return -1;
++
++ return 0;
++}
+diff --git a/tools/testing/selftests/sched_ext/util.h b/tools/testing/selftests/sched_ext/util.h
+new file mode 100644
+index 000000000000..bc13dfec1267
+--- /dev/null
++++ b/tools/testing/selftests/sched_ext/util.h
+@@ -0,0 +1,13 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++/*
++ * Copyright (c) 2024 Meta Platforms, Inc. and affiliates.
++ * Copyright (c) 2024 David Vernet <void@manifault.com>
++ */
++
++#ifndef __SCX_TEST_UTIL_H__
++#define __SCX_TEST_UTIL_H__
++
++long file_read_long(const char *path);
++int file_write_long(const char *path, long val);
++
++#endif // __SCX_TEST_H__