From 2a2df8b9c6ae56014b4e8ecf1850f80bc5759e2f Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Tue, 17 Nov 2026 17:26:37 +0900 Subject: [PATCH 1/2] fix(sched): do not dequeue a lower-priority task on the PrioritizedRR tick (#729) `invoke_preemption_tick()` dequeued the head of the run queues whenever the quantum of the running task had expired, without comparing priorities. A lower-priority head was sent to the target CPU, rejected by `do_preemption()`, and re-queued at the tail of its level. Nothing restarted the quantum, so this repeated on every primary loop iteration: one IPI per iteration, and the FIFO order of the lower level was lost. - Add `Scheduler::get_next_if()` and `get_next_task_if()`, which dequeue the head only if a predicate accepts it, under the run queue lock. `PriorityQueue::peek()` is added for it. - The tick dequeues a task only if its priority is at least that of the running task and no preemption is pending on the CPU. Otherwise it restarts the quantum, or sends the IPI again if one is pending. - Panicked tasks are never taken by the tick: `Task::priority` keeps the priority they had before the panic. - The tick returns the time until the quantum expires, and `scheduler::wake_task()` caps the sleep of the primary CPU with it while a task is queued, so a restarted quantum is still checked on time. - Start the quantum of a preempted task that `do_preemption()` resumes directly; it did not pass through `run_main`, so its quantum looked expired at once. Co-Authored-By: Claude Fable 5.2 --- awkernel_async_lib/src/scheduler.rs | 116 +++++++++++++++++- awkernel_async_lib/src/scheduler/edf.rs | 8 +- awkernel_async_lib/src/scheduler/panicked.rs | 9 +- .../src/scheduler/prioritized_fifo.rs | 12 +- .../src/scheduler/prioritized_rr.rs | 92 +++++++++++--- awkernel_async_lib/src/task.rs | 8 +- awkernel_async_lib/src/task/preempt.rs | 18 ++- awkernel_lib/src/priority_queue.rs | 49 ++++++++ mdbook/src/internal/scheduler.md | 2 +- 9 files changed, 282 insertions(+), 32 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 064b3f88e..93c7a494a 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -5,13 +5,14 @@ use core::sync::atomic::Ordering; use core::time::Duration; use crate::{ - task::{get_current_task, get_scheduler_type_by_task_id, Task}, + task::{get_current_task, get_scheduler_type_by_task_id, State, Task, NUM_TASK_IN_QUEUE}, time::Instant, }; use alloc::collections::{binary_heap::BinaryHeap, btree_map::BTreeMap}; use alloc::sync::Arc; use awkernel_lib::{ cpu::{num_cpu, CpuSet}, + priority_queue::PriorityQueue, sync::mutex::{MCSNode, Mutex}, verified::delta_list::DeltaList, }; @@ -202,6 +203,17 @@ pub(crate) trait Scheduler { /// Get the next executable task. fn get_next(&self, execution_ensured: bool) -> Option>; + /// Dequeue the task that `get_next(false)` would return, but only if + /// `pred` accepts it. A rejected task stays at the head of the run queue. + /// + /// The check and the dequeue must be done without releasing the lock of + /// the run queue, so that the dequeued task is the one `pred` accepted. + /// + /// This is used only by `get_next_task_if()`, which runs on the primary + /// CPU (CPU 0). A scheduler whose tasks must never be taken that way + /// returns [`ConditionalNext::Empty`]. + fn get_next_if(&self, pred: &dyn Fn(&Task) -> bool) -> ConditionalNext; + /// Get the scheduler name. fn scheduler_name(&self) -> SchedulerType; @@ -258,6 +270,87 @@ pub(crate) fn get_next_task(execution_ensured: bool) -> Option> { .find_map(|scheduler_type| get_scheduler(scheduler_type).get_next(execution_ensured)) } +/// The result of [`Scheduler::get_next_if`]. +pub(crate) enum ConditionalNext { + /// The run queue has no executable task. + Empty, + + /// The head of the run queue was rejected and is left in the queue. + Rejected, + + /// The head of the run queue was accepted and dequeued. + Taken(Arc), +} + +/// Dequeue the next executable task only if `pred` accepts it. +/// +/// This examines the task that `get_next_task(false)` would return, except +/// that panicked tasks are never taken. If `pred` returns `false` for it, the +/// run queues are left untouched, so the task keeps its position in the +/// queue, and `None` is returned. +/// +/// `pred` is called with `GLOBAL_WAKE_GET_MUTEX`, the lock of the run queue, +/// and the lock of the task information held. +/// +/// This function must be called on the primary CPU, which can never dequeue +/// a clustered task (see `get_next_task`), so only the non-clustered +/// schedulers are examined. +pub(crate) fn get_next_task_if(pred: impl Fn(&Task) -> bool) -> Option> { + let mut node = MCSNode::new(); + let _guard = GLOBAL_WAKE_GET_MUTEX.lock(&mut node); + + for scheduler_type in &PRIORITY_LIST[get_num_clustered_schedulers()..] { + match get_scheduler(scheduler_type).get_next_if(&pred) { + ConditionalNext::Empty => (), + ConditionalNext::Rejected => return None, + ConditionalNext::Taken(task) => return Some(task), + } + } + + None +} + +/// The body of [`Scheduler::get_next_if`] for the schedulers whose run queue +/// is a [`PriorityQueue`]. `task_of` returns the task of a queue entry. +/// +/// Finished tasks at the head of the queue are discarded as `get_next()` +/// does. The lock of the run queue must be held by the caller. +fn take_head_if( + queue: &mut PriorityQueue, + task_of: impl Fn(&T) -> &Arc, + pred: &dyn Fn(&Task) -> bool, +) -> ConditionalNext { + loop { + let Some(task) = queue.peek().map(|entry| task_of(entry).clone()) else { + return ConditionalNext::Empty; + }; + + let mut node = MCSNode::new(); + let mut task_info = task.info.lock(&mut node); + + if matches!(task_info.state, State::Terminated | State::Panicked) { + drop(task_info); + queue.pop(); + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); + continue; + } + + if !pred(&task) { + return ConditionalNext::Rejected; + } + + queue.pop(); + NUM_TASK_IN_QUEUE.fetch_sub(1, Ordering::Relaxed); + + if task_info.state == State::Preempted { + task_info.need_preemption = false; + } + + drop(task_info); + return ConditionalNext::Taken(task); + } +} + /// Get a scheduler. /// /// Takes `&SchedulerType` so the caller does not copy the full payload (the @@ -427,20 +520,37 @@ pub(crate) fn sleep_until_task(sleep_handler: Box, next: In /// If there are no sleeping tasks, this function returns `None`. pub fn wake_task() -> Option { // Check whether each running task exceeds the time quantum. + let mut quantum_wait: Option = None; + if PREEMPTION_ENABLED { for cpu_id in 1..num_cpu() { if let Some(task_id) = get_current_task(cpu_id) { if let Some(SchedulerType::PrioritizedRR(_)) = get_scheduler_type_by_task_id(task_id) { - prioritized_rr::SCHEDULER.invoke_preemption_tick(cpu_id, task_id) + let wait = prioritized_rr::SCHEDULER.invoke_preemption_tick(cpu_id, task_id); + quantum_wait = match (quantum_wait, wait) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + }; } } } } + // A queued task may be waiting for a time quantum to expire. Nothing + // else wakes the primary CPU at that time, so do not sleep beyond it. + // If no task is queued, the next enqueue wakes the primary CPU. + if NUM_TASK_IN_QUEUE.load(Ordering::Relaxed) == 0 { + quantum_wait = None; + } + let mut node = MCSNode::new(); let mut guard = SLEEPING.lock(&mut node); guard.wake_task(); - guard.time_to_wait() + + match (guard.time_to_wait(), quantum_wait) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + } } diff --git a/awkernel_async_lib/src/scheduler/edf.rs b/awkernel_async_lib/src/scheduler/edf.rs index d7811a52d..fce33bb1d 100644 --- a/awkernel_async_lib/src/scheduler/edf.rs +++ b/awkernel_async_lib/src/scheduler/edf.rs @@ -6,7 +6,7 @@ //! ([`AffinityBTreeQueue`]); `get_next` on CPU `c` pops the earliest-deadline //! task whose `cpu_set` contains `c`. -use super::{Scheduler, SchedulerType, Task}; +use super::{ConditionalNext, Scheduler, SchedulerType, Task}; use crate::{ dag::calculate_and_update_dag_deadline, scheduler::{ @@ -134,6 +134,12 @@ impl Scheduler for EDFScheduler { } } + fn get_next_if(&self, _pred: &dyn Fn(&Task) -> bool) -> ConditionalNext { + // `get_next_if` runs on the primary CPU. Spawn removes CPU 0 from + // every set, so no queued task is runnable there (see `get_next`). + ConditionalNext::Empty + } + fn scheduler_name(&self) -> SchedulerType { SchedulerType::EDF(0, CpuSet::empty()) } diff --git a/awkernel_async_lib/src/scheduler/panicked.rs b/awkernel_async_lib/src/scheduler/panicked.rs index 213d550e6..4d40cb023 100644 --- a/awkernel_async_lib/src/scheduler/panicked.rs +++ b/awkernel_async_lib/src/scheduler/panicked.rs @@ -1,7 +1,7 @@ //! A scheduler for panicked tasks. //! Panicked tasks will be the lowest priority. -use super::{Scheduler, SchedulerType, Task}; +use super::{ConditionalNext, Scheduler, SchedulerType, Task}; use crate::{ scheduler::get_priority, task::{set_current_task, State, NUM_TASK_IN_QUEUE}, @@ -82,6 +82,13 @@ impl Scheduler for PanickedScheduler { } } + fn get_next_if(&self, _pred: &dyn Fn(&Task) -> bool) -> ConditionalNext { + // Panicked tasks have the lowest priority, but `Task::priority` still + // holds the priority a task had before it panicked, so `pred` cannot + // judge them. They are never taken ahead of a running task. + ConditionalNext::Empty + } + fn scheduler_name(&self) -> SchedulerType { SchedulerType::Panicked } diff --git a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs index a38b2f5a7..d699bdfe6 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_fifo.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_fifo.rs @@ -2,7 +2,7 @@ use core::cmp::max; -use super::{Scheduler, SchedulerType, Task}; +use super::{take_head_if, ConditionalNext, Scheduler, SchedulerType, Task}; use crate::scheduler::{ peek_preemption_pending, push_preemption_pending, GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED, }; @@ -109,6 +109,16 @@ impl Scheduler for PrioritizedFIFOScheduler { } } + fn get_next_if(&self, pred: &dyn Fn(&Task) -> bool) -> ConditionalNext { + let mut node = MCSNode::new(); + let mut data = self.data.lock(&mut node); + + match (*data).as_mut() { + Some(data) => take_head_if(&mut data.queue, |entry| &entry.task, pred), + None => ConditionalNext::Empty, + } + } + fn scheduler_name(&self) -> SchedulerType { SchedulerType::PrioritizedFIFO(0) } diff --git a/awkernel_async_lib/src/scheduler/prioritized_rr.rs b/awkernel_async_lib/src/scheduler/prioritized_rr.rs index 5c22e3d6b..4d553e5fc 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_rr.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_rr.rs @@ -2,21 +2,22 @@ use core::cmp::max; -use super::{Scheduler, SchedulerType, Task}; +use super::{take_head_if, ConditionalNext, Scheduler, SchedulerType, Task}; use crate::{ scheduler::{ - get_next_task, get_priority, peek_preemption_pending, push_preemption_pending, + get_next_task_if, get_priority, peek_preemption_pending, push_preemption_pending, GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED, }, task::{ - get_last_executed_by_task_id, get_task, get_tasks_running, set_current_task, - set_need_preemption, State, NUM_TASK_IN_QUEUE, + get_task, get_tasks_running, set_current_task, set_need_preemption, State, + NUM_TASK_IN_QUEUE, }, }; use alloc::{sync::Arc, vec::Vec}; use awkernel_lib::priority_queue::PriorityQueue; use awkernel_lib::sync::mutex::{MCSNode, Mutex}; use core::sync::atomic::Ordering; +use core::time::Duration; pub struct PrioritizedRRScheduler { // Time quantum @@ -111,6 +112,16 @@ impl Scheduler for PrioritizedRRScheduler { None } + fn get_next_if(&self, pred: &dyn Fn(&Task) -> bool) -> ConditionalNext { + let mut node = MCSNode::new(); + let mut data = self.data.lock(&mut node); + + match (*data).as_mut() { + Some(data) => take_head_if(&mut data.queue, |entry| &entry.task, pred), + None => ConditionalNext::Empty, + } + } + fn scheduler_name(&self) -> SchedulerType { SchedulerType::PrioritizedRR(0) } @@ -128,19 +139,68 @@ pub static SCHEDULER: PrioritizedRRScheduler = PrioritizedRRScheduler { }; impl PrioritizedRRScheduler { - // Invoke a preemption if the task exceeds the time quantum - pub fn invoke_preemption_tick(&self, cpu_id: usize, task_id: u32) { - if let Some(last_executed) = get_last_executed_by_task_id(task_id) { - let elapsed = last_executed.elapsed().as_micros() as u64; - if elapsed > self.interval { - if let Some(next_task) = get_next_task(false) { - push_preemption_pending(cpu_id, next_task); - let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); - set_need_preemption(task_id, cpu_id); - awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); - } - } + /// Rotate the PrioritizedRR task `task_id` running on `cpu_id` if it has + /// used up its time quantum. + /// + /// The next runnable task is dequeued and sent to `cpu_id` only if its + /// priority is at least that of the running task: an equal priority is + /// the quantum rotation, and a higher priority is a plain preemption. + /// A lower-priority task could not preempt the running task anyway, so + /// it is left in the run queue to keep its position, and the quantum of + /// the running task is restarted instead. + /// + /// Returns the time after which this function must be called again for + /// this CPU, or `None` if a preemption was requested: the preempted task + /// is enqueued again, and that wakes the primary CPU. + /// + /// This function must be called on the primary CPU. + pub fn invoke_preemption_tick(&self, cpu_id: usize, task_id: u32) -> Option { + let current_task = get_task(task_id)?; + let interval = Duration::from_micros(self.interval); + + let elapsed = { + let mut node = MCSNode::new(); + let info = current_task.info.lock(&mut node); + info.get_last_executed().elapsed() + }; + + if elapsed <= interval { + return Some(interval - elapsed); + } + + // A task is dequeued only while no preemption is pending on this + // CPU: `do_preemption` runs one pending task and re-queues the others + // at the tail of their level. The predicate runs under + // `GLOBAL_WAKE_GET_MUTEX`, so no wake path can push a pending task + // between this check and the dequeue. + let next_task = get_next_task_if(|next| { + peek_preemption_pending(cpu_id).is_none() && !current_task.preempts(next) + }); + + if let Some(next_task) = next_task { + push_preemption_pending(cpu_id, next_task); + let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); + set_need_preemption(task_id, cpu_id); + awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); + return None; } + + if peek_preemption_pending(cpu_id).is_some() { + // A preemption has already been requested on this CPU, but it + // has not been performed yet. Request it again in case the IPI + // was not handled. + let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); + set_need_preemption(task_id, cpu_id); + awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); + } else { + // There is no task to rotate with. Restart the quantum so that + // the run queues are not examined again on every tick. + let mut node = MCSNode::new(); + let mut info = current_task.info.lock(&mut node); + info.update_last_executed(); + } + + Some(interval) } fn invoke_preemption_wake(&self, task: Arc) -> bool { diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 7ac806ba5..598902f2b 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -93,10 +93,10 @@ impl Task { /// /// The one exception is the PrioritizedRR time quantum: /// `PrioritizedRRScheduler::invoke_preemption_tick` pushes the next - /// runnable task to the preemption-pending heap without a priority check, - /// and `preempt::do_preemption` lets such an equal-priority pending task - /// preempt a running PrioritizedRR task so that the quantum rotates tasks - /// of the same priority. + /// runnable task to the preemption-pending heap if its priority is at + /// least that of the running task, and `preempt::do_preemption` lets such + /// an equal-priority pending task preempt a running PrioritizedRR task so + /// that the quantum rotates tasks of the same priority. /// /// This is deliberately not `self > other`: [`Ord`] for `Task` breaks /// priority ties by task ID to give the preemption-pending heaps a total diff --git a/awkernel_async_lib/src/task/preempt.rs b/awkernel_async_lib/src/task/preempt.rs index df30685fc..c6119414d 100644 --- a/awkernel_async_lib/src/task/preempt.rs +++ b/awkernel_async_lib/src/task/preempt.rs @@ -164,11 +164,13 @@ unsafe fn do_preemption() { // path requires a strictly higher priority (`Task::preempts`), so a // pending task of equal priority can only come from // `PrioritizedRRScheduler::invoke_preemption_tick`, which pushes the - // next runnable task without a priority check when the quantum of - // the running task has expired. Let it through when the running task + // next runnable task when the quantum of the running task has + // expired and that task does not have a lower priority than the + // running one. Let it through when the running task // is PrioritizedRR so that the quantum rotates equal-priority tasks. - // `PriorityInfo` includes the scheduler priority, so equal priority - // implies that `next` is a PrioritizedRR task too. + // `PriorityInfo` includes the scheduler priority, and the tick never + // pushes a panicked task, so equal priority implies that `next` is a + // PrioritizedRR task too. let quantum_rotation = !current_task.preempts(&next) && matches!( current_task.scheduler_name(), @@ -206,7 +208,13 @@ unsafe fn do_preemption() { if let Some(next_thread) = { let mut node = MCSNode::new(); let mut task_info = next.info.lock(&mut node); - task_info.take_preempt_context() + let ctx = task_info.take_preempt_context(); + if ctx.is_some() { + // `next` resumes on its own thread without passing through + // `run_main`, so start its time quantum here. + task_info.update_last_executed(); + } + ctx } { // If the next task is a preempted task, yield to it. yield_preempted_and_wake_task(current_task, next_thread); diff --git a/awkernel_lib/src/priority_queue.rs b/awkernel_lib/src/priority_queue.rs index 92fd24b23..98009f627 100644 --- a/awkernel_lib/src/priority_queue.rs +++ b/awkernel_lib/src/priority_queue.rs @@ -42,6 +42,17 @@ impl PriorityQueue { next } + + /// Return a reference to the value that `pop` would return, without removing it + #[inline(always)] + pub fn peek(&self) -> Option<&T> { + if self.has_entry == 0 { + return None; + } + let next_priority = (u32::BITS - 1) - self.has_entry.leading_zeros(); + + self.queue[next_priority as usize].front() + } } impl Default for PriorityQueue { @@ -95,6 +106,44 @@ mod tests { // Queue should be empty assert_eq!(q.pop(), None); } + + #[test] + fn test_priority_queue_peek() { + let mut q = PriorityQueue::new(); + + // Nothing to peek in an empty queue + assert_eq!(q.peek(), None); + + // `peek` returns the head of the FIFO order and does not remove it + for id in 0..10 { + q.push(0, id); + } + + for expected in 0..10 { + assert_eq!(q.peek(), Some(&expected)); + assert_eq!(q.peek(), Some(&expected)); + assert_eq!(q.pop(), Some(expected)); + } + + assert_eq!(q.peek(), None); + + // `peek` returns the value with the highest priority + q.push(1, 10); + assert_eq!(q.peek(), Some(&10)); + q.push(3, 30); + assert_eq!(q.peek(), Some(&30)); + q.push(2, 20); + assert_eq!(q.peek(), Some(&30)); + q.push(3, 31); + assert_eq!(q.peek(), Some(&30)); + + // `peek` always agrees with the following `pop` + while let Some(expected) = q.peek().copied() { + assert_eq!(q.pop(), Some(expected)); + } + + assert_eq!(q.pop(), None); + } } // TODO: Verification diff --git a/mdbook/src/internal/scheduler.md b/mdbook/src/internal/scheduler.md index 9ce66e97d..90a8ae47f 100644 --- a/mdbook/src/internal/scheduler.md +++ b/mdbook/src/internal/scheduler.md @@ -99,7 +99,7 @@ The PrioritizedRR (Prioritized Round Robin) scheduler is implemented in [priorit The scheduler maintains a `PriorityQueue` similar to PrioritizedFIFO, but adds time quantum management with a default interval of 4ms (4,000 microseconds). The scheduler provides two preemption mechanisms: `invoke_preemption_wake()` for priority-based preemption when tasks are awakened, and `invoke_preemption_tick()` for time quantum-based preemption. -The `invoke_preemption_tick()` method is called periodically on primary CPU to check if the currently running task has exceeded its time quantum. It compares the elapsed execution time against the configured interval and triggers preemption by sending an IPI if the quantum is exceeded. The task pushed by the tick usually has the same priority as the running one; `do_preemption()` lets such an equal-priority pending task preempt a running PrioritizedRR task, which is the only case where equal priority preempts. +The `invoke_preemption_tick()` method is called periodically on primary CPU to check if the currently running task has exceeded its time quantum. It compares the elapsed execution time against the configured interval. If the quantum is exceeded, it peeks at the next runnable task and dequeues it only if its priority is at least that of the running task; it then triggers preemption by sending an IPI. The task pushed by the tick usually has the same priority as the running one; `do_preemption()` lets such an equal-priority pending task preempt a running PrioritizedRR task, which is the only case where equal priority preempts. A lower-priority task stays in the run queue and keeps its position, and the quantum of the running task is restarted, as it is when the run queues are empty. Panicked tasks are never taken by the tick. While a preemption requested earlier is still pending on that CPU, the tick dequeues nothing and only sends the IPI again. The tick returns the time until the quantum expires, and the primary CPU does not sleep beyond it while a task is queued. ### Panicked Scheduler From 00fed6c10b566fdbb18a58508fffba01e2ab34c7 Mon Sep 17 00:00:00 2001 From: Yuuki Takano Date: Fri, 20 Nov 2026 18:10:21 +0900 Subject: [PATCH 2/2] fix(sched): address review findings on the PrioritizedRR tick - wake_task(): read NUM_TASK_IN_QUEUE after the sleep handlers have run, so that a task enqueued by a handler on the primary CPU keeps the quantum cap on the sleep of the primary CPU. - invoke_preemption_tick(): check under GLOBAL_WAKE_GET_MUTEX that the task still runs on the CPU, and push the dequeued task to the preemption-pending heap before the mutex is released. The idle check in run_main() then sees the push, and the task is not left pending on a sleeping CPU. get_next_task_if() takes an on_taken callback for this. - Start the time quantum before a task becomes visible as the running task, in do_preemption() (both branches) and in PrioritizedRRScheduler::get_next(). A stale last_executed_time let the tick rotate the task out before it ran. - Reword the comments that referred to get_next(false) and get_next_task(false), which have no callers anymore. - Remove the unused task::get_last_executed_by_task_id(). Co-Authored-By: Claude Fable 5.2 --- awkernel_async_lib/src/scheduler.rs | 50 ++++++++++++------- awkernel_async_lib/src/scheduler/edf.rs | 4 +- .../src/scheduler/prioritized_rr.rs | 35 +++++++++---- awkernel_async_lib/src/task.rs | 12 ----- awkernel_async_lib/src/task/preempt.rs | 19 ++++--- 5 files changed, 73 insertions(+), 47 deletions(-) diff --git a/awkernel_async_lib/src/scheduler.rs b/awkernel_async_lib/src/scheduler.rs index 93c7a494a..3ca2be7f0 100644 --- a/awkernel_async_lib/src/scheduler.rs +++ b/awkernel_async_lib/src/scheduler.rs @@ -203,8 +203,9 @@ pub(crate) trait Scheduler { /// Get the next executable task. fn get_next(&self, execution_ensured: bool) -> Option>; - /// Dequeue the task that `get_next(false)` would return, but only if - /// `pred` accepts it. A rejected task stays at the head of the run queue. + /// Dequeue the task at the head of the run queue, but only if `pred` + /// accepts it. A rejected task stays at the head of the run queue. + /// Unlike `get_next(true)`, the dequeued task is not marked as running. /// /// The check and the dequeue must be done without releasing the lock of /// the run queue, so that the dequeued task is the one `pred` accepted. @@ -284,30 +285,41 @@ pub(crate) enum ConditionalNext { /// Dequeue the next executable task only if `pred` accepts it. /// -/// This examines the task that `get_next_task(false)` would return, except -/// that panicked tasks are never taken. If `pred` returns `false` for it, the -/// run queues are left untouched, so the task keeps its position in the -/// queue, and `None` is returned. +/// This examines the head of the run queues of the non-clustered schedulers +/// in the order of their priorities, except that panicked tasks are never +/// taken. If `pred` returns `false` for it, the run queues are left +/// untouched, so the task keeps its position in the queue, and `false` is +/// returned. The dequeued task is not marked as running. +/// +/// If a task is dequeued, it is passed to `on_taken` and `true` is returned. /// /// `pred` is called with `GLOBAL_WAKE_GET_MUTEX`, the lock of the run queue, -/// and the lock of the task information held. +/// and the lock of the task information held. `on_taken` is called with only +/// `GLOBAL_WAKE_GET_MUTEX` held, so what `pred` observed under that mutex +/// still holds, and a worker that takes the mutex afterwards sees the effect +/// of `on_taken`. /// /// This function must be called on the primary CPU, which can never dequeue -/// a clustered task (see `get_next_task`), so only the non-clustered -/// schedulers are examined. -pub(crate) fn get_next_task_if(pred: impl Fn(&Task) -> bool) -> Option> { +/// a clustered task, so only the non-clustered schedulers are examined. +pub(crate) fn get_next_task_if( + pred: impl Fn(&Task) -> bool, + on_taken: impl FnOnce(Arc), +) -> bool { let mut node = MCSNode::new(); let _guard = GLOBAL_WAKE_GET_MUTEX.lock(&mut node); for scheduler_type in &PRIORITY_LIST[get_num_clustered_schedulers()..] { match get_scheduler(scheduler_type).get_next_if(&pred) { ConditionalNext::Empty => (), - ConditionalNext::Rejected => return None, - ConditionalNext::Taken(task) => return Some(task), + ConditionalNext::Rejected => return false, + ConditionalNext::Taken(task) => { + on_taken(task); + return true; + } } } - None + false } /// The body of [`Scheduler::get_next_if`] for the schedulers whose run queue @@ -538,17 +550,21 @@ pub fn wake_task() -> Option { } } + let mut node = MCSNode::new(); + let mut guard = SLEEPING.lock(&mut node); + guard.wake_task(); + // A queued task may be waiting for a time quantum to expire. Nothing // else wakes the primary CPU at that time, so do not sleep beyond it. // If no task is queued, the next enqueue wakes the primary CPU. + // + // This is checked after `guard.wake_task()`: a sleep handler enqueues + // its task on this CPU, and `wake_cpu(0)` does nothing here, so such a + // task must be counted now. if NUM_TASK_IN_QUEUE.load(Ordering::Relaxed) == 0 { quantum_wait = None; } - let mut node = MCSNode::new(); - let mut guard = SLEEPING.lock(&mut node); - guard.wake_task(); - match (guard.time_to_wait(), quantum_wait) { (Some(a), Some(b)) => Some(a.min(b)), (a, b) => a.or(b), diff --git a/awkernel_async_lib/src/scheduler/edf.rs b/awkernel_async_lib/src/scheduler/edf.rs index fce33bb1d..72cf5510e 100644 --- a/awkernel_async_lib/src/scheduler/edf.rs +++ b/awkernel_async_lib/src/scheduler/edf.rs @@ -103,8 +103,8 @@ impl Scheduler for EDFScheduler { let queue = (*data).as_mut()?; // Note: spawn removes CPU 0 from every set, so `pop_for_cpu(0)` always - // returns `None` and the `get_next_task(false)` path executed on the - // primary CPU can never dequeue a clustered task. + // returns `None` and the primary CPU can never dequeue a clustered + // task. loop { let (_, _, mut entry) = queue.pop_for_cpu(cpu_id)?; diff --git a/awkernel_async_lib/src/scheduler/prioritized_rr.rs b/awkernel_async_lib/src/scheduler/prioritized_rr.rs index 4d553e5fc..d73cb30f6 100644 --- a/awkernel_async_lib/src/scheduler/prioritized_rr.rs +++ b/awkernel_async_lib/src/scheduler/prioritized_rr.rs @@ -9,8 +9,8 @@ use crate::{ GLOBAL_WAKE_GET_MUTEX, PREEMPTION_ENABLED, }, task::{ - get_task, get_tasks_running, set_current_task, set_need_preemption, State, - NUM_TASK_IN_QUEUE, + get_current_task, get_task, get_tasks_running, set_current_task, set_need_preemption, + State, NUM_TASK_IN_QUEUE, }, }; use alloc::{sync::Arc, vec::Vec}; @@ -102,6 +102,13 @@ impl Scheduler for PrioritizedRRScheduler { } if execution_ensured { task_info.state = State::Running; + + // Start the time quantum before the task becomes + // visible as the running task. `run_main` updates the + // time only later, and a stale time would let + // `invoke_preemption_tick` rotate the task out before + // it runs. + task_info.update_last_executed(); set_current_task(awkernel_lib::cpu::cpu_id(), rr_task.task.id); } } @@ -173,12 +180,22 @@ impl PrioritizedRRScheduler { // at the tail of their level. The predicate runs under // `GLOBAL_WAKE_GET_MUTEX`, so no wake path can push a pending task // between this check and the dequeue. - let next_task = get_next_task_if(|next| { - peek_preemption_pending(cpu_id).is_none() && !current_task.preempts(next) - }); - - if let Some(next_task) = next_task { - push_preemption_pending(cpu_id, next_task); + // + // The predicate also checks that `task_id` still runs on `cpu_id`, + // and the dequeued task is pushed before the mutex is released. A + // worker clears its running task before it takes the mutex in + // `get_next_task`, so the idle check in `run_main` sees this push + // and the task is never left pending on a sleeping CPU. + let requested = get_next_task_if( + |next| { + get_current_task(cpu_id) == Some(task_id) + && peek_preemption_pending(cpu_id).is_none() + && !current_task.preempts(next) + }, + |next_task| push_preemption_pending(cpu_id, next_task), + ); + + if requested { let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); set_need_preemption(task_id, cpu_id); awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); @@ -192,7 +209,7 @@ impl PrioritizedRRScheduler { let preempt_irq = awkernel_lib::interrupt::get_preempt_irq(); set_need_preemption(task_id, cpu_id); awkernel_lib::interrupt::send_ipi(preempt_irq, cpu_id as u32); - } else { + } else if get_current_task(cpu_id) == Some(task_id) { // There is no task to rotate with. Restart the quantum so that // the run queues are not examined again on every tick. let mut node = MCSNode::new(); diff --git a/awkernel_async_lib/src/task.rs b/awkernel_async_lib/src/task.rs index 598902f2b..02c7f26c7 100644 --- a/awkernel_async_lib/src/task.rs +++ b/awkernel_async_lib/src/task.rs @@ -1507,18 +1507,6 @@ pub fn get_task(task_id: u32) -> Option> { tasks.id_to_task.get(&task_id).cloned() } -#[inline(always)] -pub fn get_last_executed_by_task_id(task_id: u32) -> Option { - let mut node = MCSNode::new(); - let tasks = TASKS.lock(&mut node); - - tasks.id_to_task.get(&task_id).map(|task| { - let mut node = MCSNode::new(); - let info = task.info.lock(&mut node); - info.get_last_executed() - }) -} - #[inline(always)] pub fn get_scheduler_type_by_task_id(task_id: u32) -> Option { let mut node = MCSNode::new(); diff --git a/awkernel_async_lib/src/task/preempt.rs b/awkernel_async_lib/src/task/preempt.rs index c6119414d..c103a8165 100644 --- a/awkernel_async_lib/src/task/preempt.rs +++ b/awkernel_async_lib/src/task/preempt.rs @@ -189,6 +189,17 @@ unsafe fn do_preemption() { let mut moved_preemption_pending = move_preemption_pending(cpu_id).unwrap(); next = moved_preemption_pending.pop().unwrap(); // Use latest highest priority task. + // Start the time quantum of `next` before it becomes visible as the + // running task: `next` may resume on its own thread without passing + // through `run_main`, and otherwise `run_main` updates the time only + // after a new thread has started. A stale time would let + // `invoke_preemption_tick` rotate `next` out before it runs. + { + let mut node = MCSNode::new(); + let mut task_info = next.info.lock(&mut node); + task_info.update_last_executed(); + } + set_current_task(cpu_id, next.id); // Re-wake the remaining all preemption-pending tasks with lower priorities than `next`. @@ -208,13 +219,7 @@ unsafe fn do_preemption() { if let Some(next_thread) = { let mut node = MCSNode::new(); let mut task_info = next.info.lock(&mut node); - let ctx = task_info.take_preempt_context(); - if ctx.is_some() { - // `next` resumes on its own thread without passing through - // `run_main`, so start its time quantum here. - task_info.update_last_executed(); - } - ctx + task_info.take_preempt_context() } { // If the next task is a preempted task, yield to it. yield_preempted_and_wake_task(current_task, next_thread);