From a590ad4bd15279875de5537e5bcfb47ae7263e7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rom=C3=A1n=20C=C3=A1rdenas=20Rodr=C3=ADguez?= Date: Fri, 20 Nov 2026 11:38:18 +0900 Subject: [PATCH] feat(scheduler): separated EDF from common clustered scheduling logic --- awkernel_async_lib/src/scheduler.rs | 1 + awkernel_async_lib/src/scheduler/clustered.rs | 239 +++++++++++++++++ awkernel_async_lib/src/scheduler/edf.rs | 251 ++---------------- mdbook/src/internal/scheduler.md | 6 +- 4 files changed, 261 insertions(+), 236 deletions(-) create mode 100644 awkernel_async_lib/src/scheduler/clustered.rs diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 064b3f88e..3d7c7e3c1 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -19,6 +19,7 @@ use awkernel_lib::{ #[cfg(not(feature = "std"))] use alloc::boxed::Box; +mod clustered; mod edf; pub(super) mod panicked; mod prioritized_fifo; diff --git a/awkernel_async_lib/src/scheduler/clustered.rs b/awkernel_async_lib/src/scheduler/clustered.rs new file mode 100644 index 000000000..1e495c4d8 --- /dev/null +++ b/awkernel_async_lib/src/scheduler/clustered.rs @@ -0,0 +1,239 @@ +//! Shared affinity queue and preemption code for the clustered schedulers. +//! +//! Each task carries a [`CpuSet`] describing the CPU cores it may run on. +//! Runnable tasks are kept in a single affinity-aware priority queue +//! ([`AffinityBTreeQueue`]) per scheduler type. + +use super::{Scheduler, SchedulerType, Task}; +use crate::{ + scheduler::{ + get_priority, peek_preemption_pending, push_preemption_pending, ClusteredTask, + GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED, + }, + task::{ + get_task, get_task_running, set_current_task, set_need_preemption, State, TaskInfo, + MAX_TASK_PRIORITY, + }, +}; +use core::marker::PhantomData; + +use affinity_btree_queue::{AffinityBTreeQueue, DEFAULT_MIN_DEGREE}; +use alloc::sync::Arc; +use awkernel_lib::{ + cpu::{masked_workers, num_cpu, CpuSet, CPU_SET_WORDS}, + sync::mutex::{MCSNode, Mutex}, +}; +/// The run queue. Priority is `(ordering_time, wake_time)`; smaller values +/// dequeue first, and entries with fully equal keys are FIFO-ordered by the +/// queue's internal sequence number. +type ReadyQueue = + AffinityBTreeQueue<(u64, u64), ClusteredTask>, DEFAULT_MIN_DEGREE, CPU_SET_WORDS>; + +pub(super) trait ClusteredOrdering { + const KIND: SchedulerType; + + /// Called once per wake under the task info lock. Return the real + /// deadline and queue ordering time. + fn ready_priority(info: &TaskInfo, wake_time: u64) -> (u64, u64); +} + +pub(super) struct ClusteredScheduler { + // `AffinityBTreeQueue::new` is not a const fn, so the queue is lazily + // initialized with `num_cpu()` on first use. + data: Mutex>, + priority: u8, + ordering: PhantomData O>, +} + +impl Scheduler for ClusteredScheduler { + // Queue operations can allocate and contend. Victim selection scans at + // most the CPUs in the affinity set. + fn wake_task(&self, task: Arc) { + let key = { + let mut node_inner = MCSNode::new(); + let mut info = task.info.lock(&mut node_inner); + let wake_time = awkernel_lib::delay::uptime(); + let (absolute_deadline, ordering_time) = O::ready_priority(&info, wake_time); + task.priority + .update_priority_info(self.priority, MAX_TASK_PRIORITY - ordering_time); + info.update_absolute_deadline(absolute_deadline); + (ordering_time, wake_time) + }; + + // `Tasks::spawn` is the single point that validates and normalizes the + // set (masking to worker cores, falling back to all workers when empty). + // Reaching here with an invalid set means that invariant was violated + // internally, so fail loud rather than enqueue an unrunnable task. + let cpu_set = task.cpu_set.expect("clustered task has no CPU set"); + if cpu_set.is_empty() || masked_workers(cpu_set, num_cpu()) != cpu_set { + unreachable!("clustered scheduler: cpu_set {cpu_set:?} was not normalized by spawn"); + } + + let mut node = MCSNode::new(); + let _guard = GLOBAL_WAKE_GET_MUTEX.lock(&mut node); + if !self.invoke_preemption(task.clone()) { + let mut node_inner = MCSNode::new(); + let mut data = self.data.lock(&mut node_inner); + let queue = data.get_or_insert_with(|| ReadyQueue::new(num_cpu())); + queue + .push(key, cpu_set, ClusteredTask::new(task)) + .expect("clustered scheduler: failed to push a task"); + drop(data); + + // Every enqueue goes through this function, including the re-wakes + // that bypass `ArcWake::wake`, so notify the primary CPU here to + // wake an idle worker in the set. + awkernel_lib::cpu::wake_cpu(0); + } + } + + fn get_next(&self, execution_ensured: bool) -> Option> { + let cpu_id = awkernel_lib::cpu::cpu_id(); + + let mut node = MCSNode::new(); + let mut data = self.data.lock(&mut node); + let queue = (*data).as_mut()?; + + // Note: spawn removes CPU 0 from every set, so `pop_for_cpu(0)` always + // returns `None` and the `get_next_task(false)` path executed on the + // primary CPU can never dequeue a clustered task. + loop { + let (_, _, mut entry) = queue.pop_for_cpu(cpu_id)?; + + // take() decrements the counters; entry then drops with inner == None. + let Some(task) = entry.take() else { + continue; + }; + + { + let mut node = MCSNode::new(); + let mut task_info = task.info.lock(&mut node); + + if matches!(task_info.state, State::Terminated | State::Panicked) { + continue; + } + + if task_info.state == State::Preempted { + task_info.need_preemption = false; + } + if execution_ensured { + task_info.state = State::Running; + set_current_task(awkernel_lib::cpu::cpu_id(), task.id); + } + } + + return Some(task); + } + } + + fn scheduler_name(&self) -> SchedulerType { + O::KIND + } + + fn priority(&self) -> u8 { + self.priority + } + + fn queued_cpu_mask(&self) -> CpuSet { + let mut node = MCSNode::new(); + let data = self.data.lock(&mut node); + + // O(1): the queue maintains the OR of all queued affinities in its + // B-tree root. + match &*data { + Some(queue) => queue.affinity_mask(), + None => CpuSet::empty(), + } + } +} + +impl ClusteredScheduler { + pub(super) const fn new() -> Self { + Self { + data: Mutex::new(None), + priority: get_priority(&O::KIND), + ordering: PhantomData, + } + } + + fn invoke_preemption(&self, task: Arc) -> bool { + if !PREEMPTION_ENABLED { + return false; + } + + let cpu_set = task.cpu_set.expect("Task has no CPU set"); + + // Find the CPU whose target (running or pending-preemption) task has + // the lowest priority among the busy CPUs of the set. + let mut victim: Option<(usize, Arc)> = None; + + for cpu_id in cpu_set.iter() { + let task_running = get_task_running(cpu_id); + + // If the task has already been running, preempt is not required. + if task_running.task_id == task.id { + return false; + } + + // An idle CPU does not stop the preemption. A task queued before + // or after this one can take the idle CPU first, and a sleeping + // CPU runs a queued task only after the primary CPU wakes it. So + // this task can preempt the lowest-priority busy CPU even if the + // set has an idle CPU. The cost is that the preemption can be + // unneeded, and the victim waits for the primary CPU to wake a + // sleeping CPU. If the set of the victim holds no idle CPU, the + // victim waits for its own CPU while the idle CPU stays idle. + if task_running.task_id == 0 { + continue; + } + + // A running task that is missing from the task table finished + // after the read above, so this CPU counts as idle too. + let Some(task_running) = get_task(task_running.task_id) else { + continue; + }; + + let target_task = match peek_preemption_pending(cpu_id) { + Some(pending) => task_running.max(pending), + None => task_running, + }; + + match victim { + Some((_, ref lowest)) if target_task >= *lowest => (), + _ => victim = Some((cpu_id, target_task)), + } + } + + let Some((victim_cpu, target_task)) = victim else { + return false; + }; + + if task.preempts(&target_task) { + push_preemption_pending(victim_cpu, task); + let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); + set_need_preemption(target_task.id, victim_cpu); + awkernel_lib::interrupt::send_ipi(preempt_irq, victim_cpu as u32); + + // NOTE: The task is now pinned to `victim_cpu`'s preemption-pending + // heap and is not in the affinity queue, so it is invisible to the + // other CPUs in its `cpu_set` (neither `pop_for_cpu`, the clustered + // task counter, nor `queued_cpu_mask` sees it). If another CPU in + // the set is or becomes idle while the IPI is in flight, it cannot pick + // the task up and may go to sleep — a latency window bounded by the + // IPI delivery plus the victim's interrupt-disabled sections. This + // is not a lost wakeup: the task is eventually run either by the + // victim's preemption handler (`do_preemption`), or re-woken by the + // victim's `run_main` if its running task finishes first, in which + // case this function decides again. `run_main` checks the pending + // heap again before the victim sleeps. Future work: + // an enqueue-instead-of-pin strategy (push to the affinity queue + // and treat the IPI as a hint) would close this window, but it + // requires redesigning the duplicate-IPI suppression that the + // pending heap currently provides via `peek_preemption_pending`. + + return true; + } + + false + } +} diff --git a/awkernel_async_lib/src/scheduler/edf.rs b/awkernel_async_lib/src/scheduler/edf.rs index d7811a52d..4f0d44fbb 100644 --- a/awkernel_async_lib/src/scheduler/edf.rs +++ b/awkernel_async_lib/src/scheduler/edf.rs @@ -1,244 +1,27 @@ -//! An EDF scheduler. -//! -//! Each task carries a [`CpuSet`] describing the CPU cores it may run on. -//! A one-core set gives partitioned EDF, and `CpuSet::all()` gives global EDF. -//! Runnable tasks are kept in a single affinity-aware priority queue -//! ([`AffinityBTreeQueue`]); `get_next` on CPU `c` pops the earliest-deadline -//! task whose `cpu_set` contains `c`. +//! Earliest Deadline First, ordered by the real job deadline. -use super::{Scheduler, SchedulerType, Task}; -use crate::{ - dag::calculate_and_update_dag_deadline, - scheduler::{ - get_priority, peek_preemption_pending, push_preemption_pending, ClusteredTask, - GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED, - }, - task::{ - get_task, get_task_running, set_current_task, set_need_preemption, State, MAX_TASK_PRIORITY, - }, +use super::{ + clustered::{ClusteredOrdering, ClusteredScheduler}, + SchedulerType, }; -use affinity_btree_queue::{AffinityBTreeQueue, DEFAULT_MIN_DEGREE}; -use alloc::sync::Arc; -use awkernel_lib::{ - cpu::{masked_workers, num_cpu, CpuSet, CPU_SET_WORDS}, - sync::mutex::{MCSNode, Mutex}, -}; - -/// The run queue. Priority is `(absolute_deadline, wake_time)`; smaller values -/// dequeue first, and entries with fully equal keys are FIFO-ordered by the -/// queue's internal sequence number. -type EDFQueue = - AffinityBTreeQueue<(u64, u64), ClusteredTask>, DEFAULT_MIN_DEGREE, CPU_SET_WORDS>; +use crate::{dag::calculate_and_update_dag_deadline, task::TaskInfo}; +use awkernel_lib::cpu::CpuSet; -pub struct EDFScheduler { - // `AffinityBTreeQueue::new` is not a const fn, so the queue is lazily - // initialized with `num_cpu()` on first use. - data: Mutex>, - priority: u8, -} +pub(super) struct Edf; -impl Scheduler for EDFScheduler { - fn wake_task(&self, task: Arc) { - let (wake_time, absolute_deadline) = { - let mut node_inner = MCSNode::new(); - let mut info = task.info.lock(&mut node_inner); - let dag_info = info.get_dag_info(); - match info.scheduler_type { - SchedulerType::EDF(relative_deadline, _) => { - let wake_time = awkernel_lib::delay::uptime(); - let absolute_deadline = if let Some(ref dag_info) = dag_info { - calculate_and_update_dag_deadline(dag_info, wake_time, info.state) - } else { - // If dag_info is not present, the task is treated as a regular task, and - // the absolute_deadline is calculated using the scheduler's relative_deadline. - wake_time + relative_deadline - }; +pub(super) static SCHEDULER: ClusteredScheduler = ClusteredScheduler::new(); - task.priority - .update_priority_info(self.priority, MAX_TASK_PRIORITY - absolute_deadline); - info.update_absolute_deadline(absolute_deadline); +impl ClusteredOrdering for Edf { + const KIND: SchedulerType = SchedulerType::EDF(0, CpuSet::empty()); - (wake_time, absolute_deadline) - } - _ => unreachable!(), - } + fn ready_priority(info: &TaskInfo, wake_time: u64) -> (u64, u64) { + let SchedulerType::EDF(relative_deadline, _) = info.scheduler_type else { + unreachable!("non-EDF task routed to EDF"); }; - - // `Tasks::spawn` is the single point that validates and normalizes the - // set (masking to worker cores, falling back to all workers when empty). - // Reaching here with an invalid set means that invariant was violated - // internally, so fail loud rather than enqueue an unrunnable task. - let cpu_set = task.cpu_set.expect("EDF task has no CPU set"); - if cpu_set.is_empty() || masked_workers(cpu_set, num_cpu()) != cpu_set { - unreachable!("EDF: cpu_set {cpu_set:?} was not normalized by spawn"); - } - - let mut node = MCSNode::new(); - let _guard = GLOBAL_WAKE_GET_MUTEX.lock(&mut node); - if !self.invoke_preemption(task.clone()) { - let mut node_inner = MCSNode::new(); - let mut data = self.data.lock(&mut node_inner); - let queue = data.get_or_insert_with(|| EDFQueue::new(num_cpu())); - queue - .push( - (absolute_deadline, wake_time), - cpu_set, - ClusteredTask::new(task), - ) - .expect("EDF: failed to push a task"); - drop(data); - - // Every enqueue goes through this function, including the re-wakes - // that bypass `ArcWake::wake`, so notify the primary CPU here to - // wake an idle worker in the set. - awkernel_lib::cpu::wake_cpu(0); - } - } - - fn get_next(&self, execution_ensured: bool) -> Option> { - let cpu_id = awkernel_lib::cpu::cpu_id(); - - let mut node = MCSNode::new(); - let mut data = self.data.lock(&mut node); - let queue = (*data).as_mut()?; - - // Note: spawn removes CPU 0 from every set, so `pop_for_cpu(0)` always - // returns `None` and the `get_next_task(false)` path executed on the - // primary CPU can never dequeue a clustered task. - loop { - let (_, _, mut entry) = queue.pop_for_cpu(cpu_id)?; - - // take() decrements the counters; entry then drops with inner == None. - let Some(task) = entry.take() else { - continue; - }; - - { - let mut node = MCSNode::new(); - let mut task_info = task.info.lock(&mut node); - - if matches!(task_info.state, State::Terminated | State::Panicked) { - continue; - } - - if task_info.state == State::Preempted { - task_info.need_preemption = false; - } - if execution_ensured { - task_info.state = State::Running; - set_current_task(awkernel_lib::cpu::cpu_id(), task.id); - } - } - - return Some(task); - } - } - - fn scheduler_name(&self) -> SchedulerType { - SchedulerType::EDF(0, CpuSet::empty()) - } - - fn priority(&self) -> u8 { - self.priority - } - - fn queued_cpu_mask(&self) -> CpuSet { - let mut node = MCSNode::new(); - let data = self.data.lock(&mut node); - - // O(1): the queue maintains the OR of all queued affinities in its - // B-tree root. - match &*data { - Some(queue) => queue.affinity_mask(), - None => CpuSet::empty(), - } - } -} - -pub static SCHEDULER: EDFScheduler = EDFScheduler { - data: Mutex::new(None), - priority: get_priority(&SchedulerType::EDF(0, CpuSet::empty())), -}; - -impl EDFScheduler { - fn invoke_preemption(&self, task: Arc) -> bool { - if !PREEMPTION_ENABLED { - return false; - } - - let cpu_set = task.cpu_set.expect("Task has no CPU set"); - - // Find the CPU whose target (running or pending-preemption) task has - // the lowest priority among the busy CPUs of the set. - let mut victim: Option<(usize, Arc)> = None; - - for cpu_id in cpu_set.iter() { - let task_running = get_task_running(cpu_id); - - // If the task has already been running, preempt is not required. - if task_running.task_id == task.id { - return false; - } - - // An idle CPU does not stop the preemption. A task queued before - // or after this one can take the idle CPU first, and a sleeping - // CPU runs a queued task only after the primary CPU wakes it. So - // this task can preempt the lowest-priority busy CPU even if the - // set has an idle CPU. The cost is that the preemption can be - // unneeded, and the victim waits for the primary CPU to wake a - // sleeping CPU. If the set of the victim holds no idle CPU, the - // victim waits for its own CPU while the idle CPU stays idle. - if task_running.task_id == 0 { - continue; - } - - // A running task that is missing from the task table finished - // after the read above, so this CPU counts as idle too. - let Some(task_running) = get_task(task_running.task_id) else { - continue; - }; - - let target_task = match peek_preemption_pending(cpu_id) { - Some(pending) => task_running.max(pending), - None => task_running, - }; - - match victim { - Some((_, ref lowest)) if target_task >= *lowest => (), - _ => victim = Some((cpu_id, target_task)), - } - } - - let Some((victim_cpu, target_task)) = victim else { - return false; + let deadline = match &info.dag_info { + Some(dag_info) => calculate_and_update_dag_deadline(dag_info, wake_time, info.state), + None => wake_time + relative_deadline, }; - - if task.preempts(&target_task) { - push_preemption_pending(victim_cpu, task); - let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); - set_need_preemption(target_task.id, victim_cpu); - awkernel_lib::interrupt::send_ipi(preempt_irq, victim_cpu as u32); - - // NOTE: The task is now pinned to `victim_cpu`'s preemption-pending - // heap and is not in the affinity queue, so it is invisible to the - // other CPUs in its `cpu_set` (neither `pop_for_cpu`, the clustered - // task counter, nor `queued_cpu_mask` sees it). If another CPU in - // the set is or becomes idle while the IPI is in flight, it cannot pick - // the task up and may go to sleep — a latency window bounded by the - // IPI delivery plus the victim's interrupt-disabled sections. This - // is not a lost wakeup: the task is eventually run either by the - // victim's preemption handler (`do_preemption`), or re-woken by the - // victim's `run_main` if its running task finishes first, in which - // case this function decides again. `run_main` checks the pending - // heap again before the victim sleeps. Future work: - // an enqueue-instead-of-pin strategy (push to the affinity queue - // and treat the IPI as a hint) would close this window, but it - // requires redesigning the duplicate-IPI suppression that the - // pending heap currently provides via `peek_preemption_pending`. - - return true; - } - - false + (deadline, deadline) } } diff --git a/mdbook/src/internal/scheduler.md b/mdbook/src/internal/scheduler.md index 9ce66e97d..18adea826 100644 --- a/mdbook/src/internal/scheduler.md +++ b/mdbook/src/internal/scheduler.md @@ -62,7 +62,7 @@ Some schedulers are implemented under the folder [awkernel_async_lib/src/schedul ```shell $ ls awkernel_async_lib/src/scheduler -> edf.rs panicked.rs prioritized_fifo.rs prioritized_rr.rs +> clustered.rs edf.rs panicked.rs prioritized_fifo.rs prioritized_rr.rs ``` A scheduler can be implemented by implementing `Scheduler` Trait. @@ -75,7 +75,9 @@ The Earliest Deadline First (EDF) scheduler is implemented in [edf.rs](https://g The scheduler holds a single affinity-aware priority queue, [`AffinityBTreeQueue`](https://crates.io/crates/affinity_btree_queue), backed by an augmented B-tree. Each entry carries `(priority, affinity, task)`, where the priority is the pair `(absolute_deadline, wake_time)`; smaller values dequeue first, so tasks are ordered by earliest deadline with wake time as a tie-breaker. Every B-tree node stores the OR of the affinities in its subtree, which lets `pop_for_cpu(cpu)` find the earliest-deadline task runnable on `cpu` in logarithmic time while skipping subtrees with no eligible entry. -When a task is enqueued via `wake_task()`, the scheduler reads the `SchedulerType::EDF(relative_deadline, cpu_set)` attached to the task and calculates the absolute deadline as `uptime + relative_deadline`. If the task is part of a DAG, `calculate_and_update_dag_deadline()` is used instead to propagate deadlines through the DAG. The task priority (`PriorityInfo`) becomes `MAX_TASK_PRIORITY - absolute_deadline`, so an earlier deadline gives a higher priority. `invoke_preemption()` handles preemption. If the task is already running, the scheduler does not preempt. Otherwise, the scheduler chooses the core running the lowest-priority task among the busy cores of the set. If the new task has a higher priority than the task running (or pending preemption) on that core, the scheduler sends an IPI to it. In that case, the scheduler pins the new task to the preemption-pending heap of that core and does not push it into the queue. If the scheduler sends no IPI, it pushes the task into the queue with its `cpu_set` as the affinity mask. One such case is a set with no busy core. Against another EDF task, a higher priority means an earlier deadline. An EDF task preempts a task of a lower-priority scheduler whatever its deadline. +EDF defines its ordering in `edf.rs` and uses the affinity queue and preemption implementation in `clustered.rs`. `SchedulerType::EDF(relative_deadline, cpu_set)` uses `uptime + relative_deadline` for standalone tasks and delegates DAG tasks to `calculate_and_update_dag_deadline()`. + +The scheduler uses that same deadline for the queue key and for `PriorityInfo`, which becomes `MAX_TASK_PRIORITY - absolute_deadline`, so an earlier deadline gives a higher priority. `invoke_preemption()` handles preemption. If the task is already running, the scheduler does not preempt. Otherwise, the scheduler chooses the core running the lowest-priority task among the busy cores of the set. If the new task has a higher priority than the task running (or pending preemption) on that core, the scheduler sends an IPI to it. In that case, the scheduler pins the new task to the preemption-pending heap of that core and does not push it into the queue. If the scheduler sends no IPI, it pushes the task into the queue with its `cpu_set` as the affinity mask. One such case is a set with no busy core. Against another EDF task, a higher priority means an earlier deadline. An EDF task preempts a task of a lower-priority scheduler whatever its deadline. An idle core in the set does not stop the preemption. A task queued before or after the new task can take the idle core first. Also, a sleeping core runs a queued task only after the primary CPU wakes it. The cost is that the preemption can be unneeded, and the preempted task waits for the primary CPU to wake a sleeping core. If the set of the preempted task holds no idle core, the preempted task waits for its own core while the idle core stays idle.