diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e746cd4..78e841b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,18 @@ ## Unreleased +### After-Response Tasks (#476) + +- Add `RequestContext::after_response(budget, task)` and the adapter-facing + `edgezero_core::after_response::{AfterResponseScheduler, AfterResponseHandle}`. A task starts + after the response attempt ends and is cancelled at its budget. Adapters without support return + `501 Not Implemented` at registration. +- Add the `after-response-tasks` capability: Axum and Fastly `BoundedCooperative`, + Cloudflare and Spin `Unsupported` (tracked in #478 and #483). +- Axum production, `run_app` and `AxumDevServer::run` shutdown now waits for registered after-response tasks within the + existing shutdown grace, after connections drain. Tasks still pending at the grace make the + process exit non-zero with `grace exhausted`. + ### Portable HTTP And Request Lifecycles (PR #275) The changes below cover [PR #275](https://github.com/IABTechLab/edgezero/pull/275), including its diff --git a/crates/edgezero-adapter-axum/src/after_response.rs b/crates/edgezero-adapter-axum/src/after_response.rs new file mode 100644 index 00000000..c8dd90b0 --- /dev/null +++ b/crates/edgezero-adapter-axum/src/after_response.rs @@ -0,0 +1,250 @@ +//! Axum after-response tasks: spawned on the server's `LocalSet`, released when the response +//! body leaves Hyper, bounded by their budget and drained on shutdown. + +use std::sync::Arc; +use std::time::Duration; + +use edgezero_core::OutboundScopeOwner; +use edgezero_core::after_response::{AfterResponseHandle, AfterResponseScheduler}; +use edgezero_core::error::EdgeError; +use futures_util::future::LocalBoxFuture; +use tokio::sync::watch; +use tokio::time::timeout; + +use crate::outbound_executor::AxumOutboundExecutorHandle; + +/// Counts after-response tasks that were registered but have not yet finished. +#[derive(Clone, Debug)] +pub(crate) struct AfterResponseTracker(Arc>); + +impl Default for AfterResponseTracker { + fn default() -> Self { + Self(Arc::new(watch::Sender::new(0))) + } +} + +impl AfterResponseTracker { + /// Number of tasks still registered. + pub(crate) fn pending(&self) -> usize { + *self.0.borrow() + } + + fn register(&self) -> TrackedTask { + self.0.send_modify(|count| *count = count.saturating_add(1)); + TrackedTask(self.clone()) + } + + /// Resolves once every registered task has finished, timed out or been cancelled. + pub(crate) async fn wait_idle(&self) { + let mut idle = self.0.subscribe(); + // The sender lives in `self`, so the channel cannot close while this waits. + let _idle = idle.wait_for(|count| *count == 0).await; + } + + /// Logs how many registered tasks shutdown is about to cancel, if any. + pub(crate) fn warn_abandoned(&self) { + let abandoned = self.pending(); + if abandoned > 0 { + log::warn!("{abandoned} after-response task(s) abandoned at shutdown"); + } + } +} + +/// Decrements the tracker when its task ends for any reason, including spawn failure. +struct TrackedTask(AfterResponseTracker); + +#[expect( + clippy::missing_trait_methods, + reason = "`Drop::pin_drop` is unstable (`pin_ergonomics`) and conflicts with `drop`" +)] +impl Drop for TrackedTask { + fn drop(&mut self) { + self.0 + .0 + .send_modify(|count| *count = count.saturating_sub(1)); + } +} + +/// Releases a request's tasks when dropped, which happens when Hyper drops the response body. +/// +/// The sender is never used to send; only its drop is observed. +pub(crate) struct AfterResponseRelease( + #[expect(dead_code, reason = "held only to be dropped")] watch::Sender<()>, +); + +/// Per-request scheduler installed by the Axum service. +struct AxumAfterResponse { + delivered: watch::Receiver<()>, + executor: AxumOutboundExecutorHandle, + tracker: AfterResponseTracker, +} + +impl AfterResponseScheduler for AxumAfterResponse { + fn schedule( + &self, + budget: Duration, + task: LocalBoxFuture<'static, ()>, + ) -> Result<(), EdgeError> { + // The registration window closes when the request's tasks are released. + if self.delivered.has_changed().is_err() { + return Err(EdgeError::internal(anyhow::anyhow!( + "after-response tasks can only be registered while the request is handled" + ))); + } + let tracked = self.tracker.register(); + let mut delivered = self.delivered.clone(); + self.executor + .spawn(Box::pin(async move { + let _tracked = tracked; + // Only the sender's drop ends this wait; the request never sends a value. + let _released = delivered.changed().await; + let scope = OutboundScopeOwner::callback(); + if timeout(budget, scope.run(task)).await.is_err() { + log::warn!( + "after-response task exceeded its {budget:?} budget and was cancelled" + ); + } + })) + .map(|_detached| ()) + } +} + +/// Creates the scheduler handle for one request and the release that starts its tasks. +pub(crate) fn request_tasks( + executor: AxumOutboundExecutorHandle, + tracker: AfterResponseTracker, +) -> (AfterResponseHandle, AfterResponseRelease) { + let (release, delivered) = watch::channel(()); + let handle = AfterResponseHandle::new(AxumAfterResponse { + delivered, + executor, + tracker, + }); + (handle, AfterResponseRelease(release)) +} + +#[cfg(test)] +mod tests { + use super::{AfterResponseTracker, request_tasks}; + use crate::outbound_executor::AxumOutboundExecutor; + use futures_util::future::pending; + use std::cell::Cell; + use std::rc::Rc; + use std::time::Duration; + use tokio::task::yield_now; + use tokio::time::sleep; + + struct DropProbe(Rc>); + + #[expect( + clippy::missing_trait_methods, + reason = "`Drop::pin_drop` is unstable (`pin_ergonomics`) and conflicts with `drop`" + )] + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.set(true); + } + } + + #[tokio::test(flavor = "current_thread")] + async fn tasks_wait_for_release_then_run_and_untrack() { + let mut owner = AxumOutboundExecutor::new(); + let tracker = AfterResponseTracker::default(); + let (handle, release) = request_tasks(owner.handle(), tracker.clone()); + let ran = Rc::new(Cell::new(false)); + let observed = Rc::clone(&ran); + handle + .schedule(Duration::from_secs(5), async move { observed.set(true) }) + .unwrap(); + assert_eq!(tracker.pending(), 1); + owner + .run_until(async { + for _ in 0_u8..8 { + yield_now().await; + } + assert!(!ran.get(), "a task never starts before its release"); + drop(release); + tracker.wait_idle().await; + }) + .await; + assert!(ran.get()); + assert_eq!(tracker.pending(), 0); + } + + #[tokio::test(flavor = "current_thread", start_paused = true)] + async fn budget_cancels_a_pending_task() { + let mut owner = AxumOutboundExecutor::new(); + let tracker = AfterResponseTracker::default(); + let (handle, release) = request_tasks(owner.handle(), tracker.clone()); + let dropped = Rc::new(Cell::new(false)); + let started = Rc::new(Cell::new(false)); + let probe = DropProbe(Rc::clone(&dropped)); + let observed = Rc::clone(&started); + handle + .schedule(Duration::from_millis(50), async move { + let _probe = probe; + observed.set(true); + pending::<()>().await; + }) + .unwrap(); + drop(release); + owner + .run_until(async { + sleep(Duration::from_millis(49)).await; + assert!(started.get(), "the released task starts"); + assert!(!dropped.get(), "the task survives within its budget"); + tracker.wait_idle().await; + }) + .await; + assert!(dropped.get(), "the budget drops the pending task"); + assert_eq!(tracker.pending(), 0); + } + + #[tokio::test(flavor = "current_thread")] + async fn registration_closes_when_tasks_are_released() { + let owner = AxumOutboundExecutor::new(); + let tracker = AfterResponseTracker::default(); + let (handle, release) = request_tasks(owner.handle(), tracker.clone()); + drop(release); + let polled = Rc::new(Cell::new(false)); + let observed = Rc::clone(&polled); + let result = handle.schedule(Duration::from_secs(1), async move { observed.set(true) }); + assert!(result.is_err(), "a released request accepts no more tasks"); + assert_eq!(tracker.pending(), 0); + assert!(!polled.get()); + } + + #[tokio::test(flavor = "current_thread", start_paused = true)] + async fn budget_starts_at_release_not_at_registration() { + let mut owner = AxumOutboundExecutor::new(); + let tracker = AfterResponseTracker::default(); + let (handle, release) = request_tasks(owner.handle(), tracker.clone()); + let ran = Rc::new(Cell::new(false)); + let observed = Rc::clone(&ran); + handle + .schedule(Duration::from_millis(50), async move { + sleep(Duration::from_millis(10)).await; + observed.set(true); + }) + .unwrap(); + owner + .run_until(async { + // A slow response outlasts the budget before the task may start. + sleep(Duration::from_millis(100)).await; + drop(release); + tracker.wait_idle().await; + }) + .await; + assert!(ran.get(), "the budget is measured from release"); + } + + #[test] + fn spawn_failure_untracks_the_task() { + let owner = AxumOutboundExecutor::new(); + let tracker = AfterResponseTracker::default(); + let (handle, _release) = request_tasks(owner.handle(), tracker.clone()); + // No Tokio runtime is current, so the executor refuses the spawn. + assert!(handle.schedule(Duration::from_secs(1), async {}).is_err()); + assert_eq!(tracker.pending(), 0); + } +} diff --git a/crates/edgezero-adapter-axum/src/cli.rs b/crates/edgezero-adapter-axum/src/cli.rs index 646a75c0..e1d52206 100644 --- a/crates/edgezero-adapter-axum/src/cli.rs +++ b/crates/edgezero-adapter-axum/src/cli.rs @@ -138,6 +138,7 @@ struct EdgezeroAxumConfig { impl Adapter for AxumCliAdapter { fn capability(&self, capability: Capability) -> CapabilitySupport { match capability { + Capability::AfterResponseTasks => CapabilitySupport::BoundedCooperative, Capability::ConfigReadDeadlines | Capability::OutboundConnectTimeout | Capability::OutboundFailureDiagnostics @@ -890,10 +891,18 @@ mod tests { } #[test] + #[expect( + clippy::too_many_lines, + reason = "keep the complete capability data table together" + )] fn adapter_capability_matrix_matches_contracts() { let connect = AXUM_ADAPTER.capability(Capability::OutboundConnectTimeout); assert_eq!(connect, CapabilitySupport::BestEffort); for (capability, support) in [ + ( + Capability::AfterResponseTasks, + CapabilitySupport::BoundedCooperative, + ), ( Capability::ConfigReadAllocationBounds, CapabilitySupport::Unsupported, diff --git a/crates/edgezero-adapter-axum/src/dev_server.rs b/crates/edgezero-adapter-axum/src/dev_server.rs index e5dec322..f9fef79f 100644 --- a/crates/edgezero-adapter-axum/src/dev_server.rs +++ b/crates/edgezero-adapter-axum/src/dev_server.rs @@ -20,6 +20,7 @@ use std::pin::Pin; use std::sync::Arc; use std::time::Duration; +use crate::after_response::AfterResponseTracker; use crate::outbound_executor::{AxumOutboundExecutor, AxumOutboundExecutorHandle}; pub use crate::run_options::AxumRunOptions; use crate::run_options::{failure, shutdown_grace}; @@ -637,8 +638,10 @@ where { log::warn!(target: BOOT_LOG_TARGET, "application head limits exceed Axum raw ingress limits; the parser rejects first"); } - let mut service = - AxumServiceState::with_transport(app, transport, executor).with_phase(reader.clone()); + let after_response = AfterResponseTracker::default(); + let mut service = AxumServiceState::with_transport(app, transport, executor) + .with_phase(reader.clone()) + .with_after_response_tracker(after_response.clone()); if let Some(registry) = stores.config { service = service.with_config_registry(registry); } @@ -699,8 +702,10 @@ where tasks.clear(); outcome } else { - drain_tasks(&mut tasks, grace, signals, outcome).await + drain_tasks(&mut tasks, &after_response, grace, signals, outcome).await }; + // Pending tasks are cancelled when the executor drops, whichever way shutdown ended. + after_response.warn_abandoned(); // Service/registry clones are dropped only after every owned connection settles. drop(service); result @@ -712,6 +717,7 @@ where )] async fn drain_tasks( tasks: &mut Connections, + after_response: &AfterResponseTracker, grace: Duration, signals: &mut Option, mut outcome: anyhow::Result<()>, @@ -737,6 +743,20 @@ async fn drain_tasks( } } tasks.clear(); + // Every connection has settled, so no handler can register more work. Tasks share the grace. + // An idle tracker wins over a deadline that passed while the select was being set up. + if outcome.is_ok() { + tokio::select! { + biased; + signal = next_signal(signals) => { + outcome = Err(signal.err().unwrap_or_else(|| failure("shutdown", "signal", "forced stop"))); + } + () = after_response.wait_idle() => {} + () = sleep_until(deadline) => { + outcome = Err(failure("shutdown", "budget", "grace exhausted")); + } + } + } outcome } diff --git a/crates/edgezero-adapter-axum/src/lib.rs b/crates/edgezero-adapter-axum/src/lib.rs index 04b5d25a..8c13246f 100644 --- a/crates/edgezero-adapter-axum/src/lib.rs +++ b/crates/edgezero-adapter-axum/src/lib.rs @@ -1,5 +1,7 @@ //! Axum adapter for `EdgeZero` routers and applications. +#[cfg(feature = "axum")] +mod after_response; #[cfg(feature = "axum")] mod config_snapshot; #[cfg(feature = "axum")] diff --git a/crates/edgezero-adapter-axum/src/response.rs b/crates/edgezero-adapter-axum/src/response.rs index e0c2f5bc..0fad1667 100644 --- a/crates/edgezero-adapter-axum/src/response.rs +++ b/crates/edgezero-adapter-axum/src/response.rs @@ -29,6 +29,8 @@ use tokio::task::yield_now; use tokio::time::Instant as TokioInstant; use tokio::time::timeout_at; +use crate::after_response::AfterResponseRelease; + const INTERNAL_FALLBACK_BODY: &[u8] = b"internal server error"; const TIMEOUT_FALLBACK_BODY: &[u8] = b"response write deadline exceeded"; @@ -212,6 +214,8 @@ struct ActiveEgress { /// Private Hyper body that owns one response-egress attempt through source completion or drop. pub(crate) struct AxumEgressBody { active: Option, + /// Dropped with the body, after the attempt terminates in `Drop`, to start after-response tasks. + after_response: Option, remaining_hint: Option, source: ResponseSource, staged_final: Option, @@ -242,9 +246,15 @@ impl AxumEgressBody { ) } + /// Holds `release` until Hyper drops this body, which starts the request's after-response tasks. + pub(crate) fn release_after_response_on_drop(&mut self, release: AfterResponseRelease) { + self.after_response = Some(release); + } + fn detached(bytes: Bytes) -> Self { Self { active: None, + after_response: None, remaining_hint: u64::try_from(bytes.len()).ok(), source: if bytes.is_empty() { ResponseSource::Done @@ -298,6 +308,7 @@ impl AxumEgressBody { kind, state, }), + after_response: None, remaining_hint: declared_length.or(inferred_length), source, staged_final: None, diff --git a/crates/edgezero-adapter-axum/src/service.rs b/crates/edgezero-adapter-axum/src/service.rs index a84cdb49..4ece1db2 100644 --- a/crates/edgezero-adapter-axum/src/service.rs +++ b/crates/edgezero-adapter-axum/src/service.rs @@ -33,6 +33,7 @@ use tokio::sync::watch; #[cfg(test)] use tower::Service as TowerService; +use crate::after_response::{AfterResponseTracker, request_tasks}; #[cfg(test)] use crate::connection::{ConnectionExit, serve_http1}; #[cfg(test)] @@ -50,6 +51,7 @@ pub(crate) struct AxumIngressAbort; /// Shared application state used to construct one private Hyper service per HTTP/1 connection. #[derive(Clone)] pub(crate) struct AxumServiceState { + after_response: AfterResponseTracker, app: Arc, config_registry: Option, config_store_handle: Option, @@ -89,6 +91,7 @@ impl AxumServiceState { .cloned() .or_else(|| initialize(app.monotonic_clock())); Self { + after_response: AfterResponseTracker::default(), app: Arc::new(app), executor, config_registry: None, @@ -204,6 +207,12 @@ impl AxumServiceState { self } + /// Shares the lifecycle owner's tracker so shutdown can drain after-response tasks. + pub(crate) fn with_after_response_tracker(mut self, tracker: AfterResponseTracker) -> Self { + self.after_response = tracker; + self + } + pub(crate) fn with_phase(mut self, phase: watch::Receiver) -> Self { self.phase = Some(phase); self @@ -371,8 +380,15 @@ impl AxumServiceState { core_request.extensions_mut().insert(registry); } + // Dropping `release` starts this request's after-response tasks: normally when Hyper + // drops the response body, or here if this future is cancelled before a body exists. + let (after_response, release) = + request_tasks(self.executor.clone(), self.after_response.clone()); + core_request.extensions_mut().insert(after_response); let egress = app.dispatch_admitted(prepared, core_request).await; - Ok(prepare_egress_response(egress, connection).await) + let mut response = prepare_egress_response(egress, connection).await; + response.body_mut().release_after_response_on_drop(release); + Ok(response) } } diff --git a/crates/edgezero-adapter-axum/tests/after_response.rs b/crates/edgezero-adapter-axum/tests/after_response.rs new file mode 100644 index 00000000..6aacffa2 --- /dev/null +++ b/crates/edgezero-adapter-axum/tests/after_response.rs @@ -0,0 +1,263 @@ +//! After-response tasks through a real Axum listener and Hyper connection. +#![cfg(all(test, feature = "axum"))] +#![expect( + clippy::expect_used, + clippy::tests_outside_test_module, + reason = "this integration-test crate uses explicit fixture diagnostics" +)] + +use std::net::SocketAddr; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; + +use bytes::Bytes; +use edgezero_adapter_axum::dev_server::{AxumDevServer, AxumDevServerConfig}; +use edgezero_adapter_axum::ingress_config::AxumIngressConfig; +use edgezero_core::app::App; +use edgezero_core::body::Body; +use edgezero_core::context::RequestContext; +use edgezero_core::error::EdgeError; +use edgezero_core::http::response_builder; +use edgezero_core::outbound::OutboundRequest; +use edgezero_core::router::RouterService; +use futures::future::pending; +use futures::stream::{self, StreamExt as _}; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; +use tokio::net::{TcpListener, TcpStream}; +use tokio::sync::Notify; +use tokio::task::JoinHandle; +use tokio::time::{sleep, timeout}; + +const WAIT: Duration = Duration::from_secs(5); + +async fn serve(router: RouterService) -> (SocketAddr, JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("listener"); + let address = listener.local_addr().expect("address"); + let server = AxumDevServer::with_config( + App::new(router), + AxumDevServerConfig { + addr: address, + enable_ctrl_c: false, + ingress: AxumIngressConfig::new(8, 8192, 100, Duration::from_secs(10)) + .expect("ingress"), + }, + ); + (address, tokio::spawn(server.serve(listener, pending()))) +} + +async fn eventually(flag: &AtomicBool) -> bool { + timeout(WAIT, async { + while !flag.load(Ordering::SeqCst) { + sleep(Duration::from_millis(5)).await; + } + }) + .await + .is_ok() +} + +async fn read_until(stream: &mut TcpStream, buffer: &mut Vec, marker: &[u8]) { + timeout(WAIT, async { + while !buffer.windows(marker.len()).any(|window| window == marker) { + let mut chunk = [0_u8; 1024]; + let read = stream.read(&mut chunk).await.expect("read"); + assert_ne!(read, 0, "connection closed before {marker:?}"); + buffer.extend_from_slice(chunk.get(..read).expect("read length")); + } + }) + .await + .expect("expected response bytes"); +} + +#[tokio::test] +async fn task_starts_only_after_the_final_body_frame_on_a_kept_alive_connection() { + let ran = Arc::new(AtomicBool::new(false)); + let release = Arc::new(Notify::new()); + let shared_ran = Arc::clone(&ran); + let shared_release = Arc::clone(&release); + let router = RouterService::builder() + .get("/gated", move |ctx: RequestContext| { + let task_ran = Arc::clone(&shared_ran); + let body_release = Arc::clone(&shared_release); + async move { + ctx.after_response(Duration::from_secs(5), async move { + task_ran.store(true, Ordering::SeqCst); + })?; + let chunks = stream::unfold(0_u8, move |step| { + let gate = Arc::clone(&body_release); + async move { + match step { + 0 => Some((Ok::<_, EdgeError>(Bytes::from_static(b"first")), 1)), + 1 => { + gate.notified().await; + Some((Ok(Bytes::from_static(b"final")), 2)) + } + _ => None, + } + } + }); + Ok::<_, EdgeError>( + response_builder() + .body(Body::from_stream(chunks)) + .expect("response"), + ) + } + }) + .build(); + let (address, server) = serve(router).await; + let mut client = TcpStream::connect(address).await.expect("connect"); + client + .write_all(b"GET /gated HTTP/1.1\r\nHost: localhost\r\n\r\n") + .await + .expect("request"); + let mut received = Vec::new(); + read_until(&mut client, &mut received, b"first").await; + sleep(Duration::from_millis(100)).await; + assert!( + !ran.load(Ordering::SeqCst), + "the task must not start while the response body is still streaming" + ); + release.notify_one(); + read_until(&mut client, &mut received, b"0\r\n\r\n").await; + assert!( + eventually(&ran).await, + "the task runs once the final frame is handed to Hyper, without closing the connection" + ); + // The keep-alive connection still serves requests after the task ran. + client + .write_all(b"GET /missing HTTP/1.1\r\nHost: localhost\r\n\r\n") + .await + .expect("second request"); + read_until(&mut client, &mut received, b"404").await; + server.abort(); +} + +#[tokio::test] +async fn client_disconnect_mid_stream_still_releases_the_task() { + let ran = Arc::new(AtomicBool::new(false)); + let shared_ran = Arc::clone(&ran); + let router = RouterService::builder() + .get("/stalled", move |ctx: RequestContext| { + let task_ran = Arc::clone(&shared_ran); + async move { + ctx.after_response(Duration::from_secs(5), async move { + task_ran.store(true, Ordering::SeqCst); + })?; + let chunks = + stream::once(async { Ok::<_, EdgeError>(Bytes::from_static(b"first")) }) + .chain(stream::pending()); + Ok::<_, EdgeError>( + response_builder() + .body(Body::from_stream(chunks)) + .expect("response"), + ) + } + }) + .build(); + let (address, server) = serve(router).await; + let mut client = TcpStream::connect(address).await.expect("connect"); + client + .write_all(b"GET /stalled HTTP/1.1\r\nHost: localhost\r\n\r\n") + .await + .expect("request"); + let mut received = Vec::new(); + read_until(&mut client, &mut received, b"first").await; + sleep(Duration::from_millis(50)).await; + assert!(!ran.load(Ordering::SeqCst), "the body is still open"); + drop(client); + assert!( + eventually(&ran).await, + "dropping the connection drops the body and releases the task" + ); + server.abort(); +} + +#[tokio::test] +async fn task_registered_before_an_error_response_still_runs() { + let ran = Arc::new(AtomicBool::new(false)); + let shared_ran = Arc::clone(&ran); + let router = RouterService::builder() + .get("/fails", move |ctx: RequestContext| { + let task_ran = Arc::clone(&shared_ran); + async move { + ctx.after_response(Duration::from_secs(5), async move { + task_ran.store(true, Ordering::SeqCst); + })?; + Err::<&'static str, _>(EdgeError::service_unavailable("handler failed")) + } + }) + .build(); + let (address, server) = serve(router).await; + let mut client = TcpStream::connect(address).await.expect("connect"); + client + .write_all(b"GET /fails HTTP/1.1\r\nHost: localhost\r\n\r\n") + .await + .expect("request"); + let mut received = Vec::new(); + read_until(&mut client, &mut received, b"503").await; + assert!( + eventually(&ran).await, + "an error response still releases the task" + ); + server.abort(); +} + +#[tokio::test] +async fn outbound_request_started_inside_a_task_reaches_the_upstream() { + let upstream = TcpListener::bind("127.0.0.1:0").await.expect("upstream"); + let target = format!( + "http://{}/sync", + upstream.local_addr().expect("upstream address") + ); + let reached = Arc::new(AtomicBool::new(false)); + let succeeded = Arc::new(AtomicBool::new(false)); + let upstream_reached = Arc::clone(&reached); + let accept = tokio::spawn(async move { + let (mut socket, _peer) = upstream.accept().await.expect("accept"); + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + head.push(socket.read_u8().await.expect("upstream read")); + } + assert!(head.starts_with(b"GET /sync ")); + upstream_reached.store(true, Ordering::SeqCst); + socket + .write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n") + .await + .expect("upstream write"); + }); + let shared_succeeded = Arc::clone(&succeeded); + let router = RouterService::builder() + .get("/sync", move |ctx: RequestContext| { + let destination = target.clone(); + let task_succeeded = Arc::clone(&shared_succeeded); + async move { + let client = ctx.http_client().expect("outbound client"); + ctx.after_response(Duration::from_secs(5), async move { + let request = OutboundRequest::get(&destination).expect("outbound request"); + if client.send(request).await.is_ok() { + task_succeeded.store(true, Ordering::SeqCst); + } + })?; + Ok::<_, EdgeError>("queued") + } + }) + .build(); + let (address, server) = serve(router).await; + let mut client = TcpStream::connect(address).await.expect("connect"); + client + .write_all(b"GET /sync HTTP/1.1\r\nHost: localhost\r\n\r\n") + .await + .expect("request"); + let mut received = Vec::new(); + read_until(&mut client, &mut received, b"queued").await; + assert!( + eventually(&reached).await, + "the task's request reached the upstream" + ); + assert!( + eventually(&succeeded).await, + "the task's outbound request completed" + ); + accept.await.expect("upstream task"); + server.abort(); +} diff --git a/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs b/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs index ebb2de55..e74620cd 100644 --- a/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs +++ b/crates/edgezero-adapter-axum/tests/fixtures/production_host.rs @@ -61,6 +61,7 @@ impl Hooks for FixtureApp { .get("/stream", endless_stream) .get("/flood", flood) .get("/after", after) + .get("/after-task", after_task) .get("/write", write) .get("/read", read) .get("/alias", alias) @@ -130,6 +131,20 @@ async fn after() -> Result<&'static str, EdgeError> { Ok("forbidden dispatch") } +#[action] +async fn after_task(ctx: RequestContext) -> Result<&'static str, EdgeError> { + let millis = env::var("FIXTURE_TASK_MILLIS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(500); + ctx.after_response(Duration::from_secs(30), async move { + let _guard = WorkGuard("AFTER_TASK_DROPPED"); + sleep(Duration::from_millis(millis)).await; + println!("AFTER_TASK_DONE"); + })?; + Ok("after-task-queued") +} + #[action] async fn finite(ctx: RequestContext) -> Result<&'static str, EdgeError> { let observed = reader(&ctx)?; @@ -139,7 +154,11 @@ async fn finite(ctx: RequestContext) -> Result<&'static str, EdgeError> { sleep(Duration::from_millis(5)).await; } println!("DRAINING"); - sleep(Duration::from_millis(80)).await; + let millis = env::var("FIXTURE_FINITE_MILLIS") + .ok() + .and_then(|value| value.parse().ok()) + .unwrap_or(80); + sleep(Duration::from_millis(millis)).await; Ok("finite-complete") } diff --git a/crates/edgezero-adapter-axum/tests/production_hosting.rs b/crates/edgezero-adapter-axum/tests/production_hosting.rs index 8d24aa39..120fc788 100644 --- a/crates/edgezero-adapter-axum/tests/production_hosting.rs +++ b/crates/edgezero-adapter-axum/tests/production_hosting.rs @@ -315,6 +315,119 @@ mod tests { } } + #[test] + fn sigterm_waits_for_after_response_task_within_the_grace() { + let roots = Roots::new(); + let addr = address(); + let mut cmd = command("bare", &roots, addr); + cmd.env("EDGEZERO__ADAPTER__SHUTDOWN_GRACE_SECONDS", "3") + .env("FIXTURE_TASK_MILLIS", "1500"); + let mut server = Server::spawn(cmd, addr); + server.ready(); + let response = server.request("/after-task").expect("after-task response"); + assert!(response.contains("after-task-queued"), "{response}"); + assert!( + !server.log().contains("AFTER_TASK_DONE"), + "the task must still be running when SIGTERM arrives" + ); + let signalled = Instant::now(); + server.signal("TERM"); + server.wait_line("AFTER_TASK_DONE"); + assert!(server.wait().success(), "{}", server.log()); + assert!(signalled.elapsed() >= Duration::from_millis(1000)); + assert!( + !server.log().contains("grace exhausted"), + "{}", + server.log() + ); + } + + #[test] + fn grace_exhaustion_cancels_after_response_task() { + let roots = Roots::new(); + let addr = address(); + let mut cmd = command("bare", &roots, addr); + cmd.env("FIXTURE_TASK_MILLIS", "10000"); + let mut server = Server::spawn(cmd, addr); + server.ready(); + let response = server.request("/after-task").expect("after-task response"); + assert!(response.contains("after-task-queued"), "{response}"); + let signalled = Instant::now(); + server.signal("TERM"); + assert!(!server.wait().success(), "{}", server.log()); + assert!(signalled.elapsed() >= Duration::from_millis(900)); + assert!(signalled.elapsed() < Duration::from_secs(4)); + let log = server.log(); + assert!(log.contains("grace exhausted"), "{log}"); + assert!(!log.contains("AFTER_TASK_DONE"), "{log}"); + assert_eq!(log.matches("AFTER_TASK_DROPPED").count(), 1, "{log}"); + } + + #[test] + fn connections_and_after_response_tasks_share_one_grace() { + let roots = Roots::new(); + let addr = address(); + let mut cmd = command("bare", &roots, addr); + // Draining the connection uses most of the 1 s grace; a restarted deadline would not. + cmd.env("FIXTURE_TASK_MILLIS", "10000") + .env("FIXTURE_FINITE_MILLIS", "800"); + let mut server = Server::spawn(cmd, addr); + server.ready(); + let queued = server.request("/after-task").expect("after-task response"); + assert!(queued.contains("after-task-queued"), "{queued}"); + let mut client = server.socket().expect("finite socket"); + client + .write_all(b"GET /finite HTTP/1.1\r\nHost: fixture\r\nConnection: close\r\n\r\n") + .expect("finite request"); + server.wait_line("DISPATCH"); + let signalled = Instant::now(); + server.signal("TERM"); + let mut response = String::new(); + client + .read_to_string(&mut response) + .expect("finite response"); + assert!(response.contains("finite-complete"), "{response}"); + assert!(!server.wait().success(), "{}", server.log()); + let elapsed = signalled.elapsed(); + assert!(elapsed >= Duration::from_millis(900), "{elapsed:?}"); + assert!( + elapsed < Duration::from_millis(1500), + "tasks must not get a fresh grace after connections drain: {elapsed:?}" + ); + let log = server.log(); + assert!(log.contains("grace exhausted"), "{log}"); + assert!(!log.contains("AFTER_TASK_DONE"), "{log}"); + } + + #[test] + fn second_signal_during_after_response_wait_forces_stop() { + let roots = Roots::new(); + let addr = address(); + let mut cmd = command("bare", &roots, addr); + cmd.env("EDGEZERO__ADAPTER__SHUTDOWN_GRACE_SECONDS", "5") + .env("FIXTURE_TASK_MILLIS", "10000"); + let mut server = Server::spawn(cmd, addr); + server.ready(); + let queued = server.request("/after-task").expect("after-task response"); + assert!(queued.contains("after-task-queued"), "{queued}"); + server.signal("TERM"); + // No connections remain, so shutdown is now waiting for the task. + thread::sleep(Duration::from_millis(300)); + assert!( + server.child.try_wait().expect("process status").is_none(), + "{}", + server.log() + ); + let forced = Instant::now(); + server.signal("INT"); + assert!(!server.wait().success(), "{}", server.log()); + assert!(forced.elapsed() < Duration::from_millis(700)); + let log = server.log(); + assert!(log.contains("forced stop"), "{log}"); + assert!(!log.contains("AFTER_TASK_DONE"), "{log}"); + assert_eq!(log.matches("AFTER_TASK_DROPPED").count(), 1, "{log}"); + } + #[test] fn second_signal_forces_before_budget_exhaustion() { let roots = Roots::new(); diff --git a/crates/edgezero-adapter-cloudflare/src/cli.rs b/crates/edgezero-adapter-cloudflare/src/cli.rs index f8126552..a4cc2a10 100644 --- a/crates/edgezero-adapter-cloudflare/src/cli.rs +++ b/crates/edgezero-adapter-cloudflare/src/cli.rs @@ -160,7 +160,8 @@ impl Adapter for CloudflareCliAdapter { | Capability::OutboundFlexiblePhaseBudget | Capability::OutboundHttp | Capability::OutboundRedirectFollowing => CapabilitySupport::Native, - Capability::ConfigReadAllocationBounds + Capability::AfterResponseTasks + | Capability::ConfigReadAllocationBounds | Capability::OutboundCompleteResourceAccounting | Capability::RawIngressFramingValidation | Capability::RawIngressHeadLimits @@ -1525,6 +1526,10 @@ mod tests { )] fn adapter_capability_matrix_matches_contracts() { for (capability, support) in [ + ( + Capability::AfterResponseTasks, + CapabilitySupport::Unsupported, + ), ( Capability::ConfigReadAllocationBounds, CapabilitySupport::Unsupported, diff --git a/crates/edgezero-adapter-fastly/src/after_response.rs b/crates/edgezero-adapter-fastly/src/after_response.rs new file mode 100644 index 00000000..9629c67e --- /dev/null +++ b/crates/edgezero-adapter-fastly/src/after_response.rs @@ -0,0 +1,216 @@ +//! Fastly after-response tasks: queued during one request lifecycle, then driven after the +//! response is sent, inside the same callback scope, each bounded by its budget. + +use std::cell::{Cell, RefCell}; +use std::time::Duration; + +use edgezero_core::after_response::{AfterResponseHandle, AfterResponseScheduler}; +use edgezero_core::error::EdgeError; +#[cfg(feature = "fastly")] +use edgezero_core::time::MonotonicClock; +use futures_util::future::LocalBoxFuture; + +type Queued = (Duration, LocalBoxFuture<'static, ()>); + +/// One armed request lifecycle: its unique generation and its queued tasks. +struct Lifecycle { + generation: u64, + tasks: Vec, +} + +thread_local! { + /// `Some` while a request lifecycle accepts tasks; `None` otherwise. + static QUEUE: RefCell> = const { RefCell::new(None) }; + /// Source of lifecycle generations; a handle only queues into the lifecycle that issued it. + static NEXT_GENERATION: Cell = const { Cell::new(0) }; +} + +/// Request-lifecycle guard. Restores the previously armed lifecycle when dropped. +pub(crate) struct ArmedQueue { + previous: Option, +} + +struct FastlyAfterResponse { + generation: u64, +} + +impl ArmedQueue { + /// Closes registration and returns the queued tasks; the guard still restores on drop. + #[expect( + clippy::unused_self, + reason = "taking the guard by reference ties draining to an armed lifecycle" + )] + pub(crate) fn take(&self) -> Vec { + QUEUE.with(|queue| match queue.try_borrow_mut() { + Ok(mut queued) => queued + .take() + .map(|lifecycle| lifecycle.tasks) + .unwrap_or_default(), + Err(_borrowed) => { + log::warn!("after-response queue was busy; queued tasks were dropped"); + Vec::new() + } + }) + } +} + +#[expect( + clippy::missing_trait_methods, + reason = "`Drop::pin_drop` is unstable (`pin_ergonomics`) and conflicts with `drop`" +)] +impl Drop for ArmedQueue { + fn drop(&mut self) { + let previous = self.previous.take(); + QUEUE.with(|queue| { + if let Ok(mut queued) = queue.try_borrow_mut() { + *queued = previous; + } + }); + } +} + +impl AfterResponseScheduler for FastlyAfterResponse { + fn schedule( + &self, + budget: Duration, + task: LocalBoxFuture<'static, ()>, + ) -> Result<(), EdgeError> { + QUEUE.with(|queue| match queue.try_borrow_mut().as_deref_mut() { + Ok(Some(lifecycle)) if lifecycle.generation == self.generation => { + lifecycle.tasks.push((budget, task)); + Ok(()) + } + Ok(_) | Err(_) => Err(EdgeError::internal(anyhow::anyhow!( + "after-response tasks can only be registered while the request is handled" + ))), + }) + } +} + +/// Opens a fresh queue for one request lifecycle. +pub(crate) fn arm() -> ArmedQueue { + let generation = NEXT_GENERATION.with(|next| { + let issued = next.get(); + next.set(issued.wrapping_add(1)); + issued + }); + let previous = QUEUE.with(|queue| { + queue.replace(Some(Lifecycle { + generation, + tasks: Vec::new(), + })) + }); + ArmedQueue { previous } +} + +/// Returns the current lifecycle's handle while one is armed. +pub(crate) fn handle_if_armed() -> Option { + QUEUE + .with(|queue| -> Option { Some(queue.try_borrow().ok()?.as_ref()?.generation) }) + .map(|generation| AfterResponseHandle::new(FastlyAfterResponse { generation })) +} + +/// Drives `tasks` to completion or their budgets inside the current callback scope. +#[cfg(feature = "fastly")] +pub(crate) fn drain(tasks: Vec, clock: MonotonicClock) { + use edgezero_core::time::{DEADLINE_FAR_FUTURE, Deadline}; + use futures::executor; + use futures_util::future::{Either, join_all, select}; + + use crate::outbound::{drive_callback, native_time}; + + if tasks.is_empty() { + return; + } + let start = clock.now(); + let time = native_time(clock.clone()); + let bounded = tasks.into_iter().map(|(budget, task)| { + let deadline = Deadline::at_instant( + start + .checked_add(budget.min(DEADLINE_FAR_FUTURE)) + .unwrap_or(start), + ); + let timer = time.wait_until(deadline); + async move { + match select(task, timer).await { + Either::Left(((), _timer)) => {} + Either::Right((Ok(()), _task)) => log::warn!( + "after-response task exceeded its {budget:?} budget and was cancelled" + ), + Either::Right((Err(_error), _task)) => { + log::warn!("after-response task timer failed; the task was cancelled"); + } + } + } + }); + if executor::block_on(drive_callback(join_all(bounded), clock, start)).is_err() { + log::warn!("after-response tasks could not be driven and were cancelled"); + } +} + +#[cfg(test)] +mod tests { + use super::{arm, handle_if_armed}; + use std::cell::Cell; + use std::rc::Rc; + use std::time::Duration; + + #[test] + fn unarmed_requests_get_no_handle() { + assert!(handle_if_armed().is_none()); + } + + #[test] + fn armed_queue_collects_tasks_then_closes_registration() { + let armed = arm(); + let handle = handle_if_armed().expect("armed lifecycle installs a handle"); + let polled = Rc::new(Cell::new(false)); + let observed = Rc::clone(&polled); + handle + .schedule(Duration::from_millis(5), async move { observed.set(true) }) + .unwrap(); + let tasks = armed.take(); + assert_eq!(tasks.len(), 1); + assert_eq!( + tasks.first().map(|(budget, _task)| *budget), + Some(Duration::from_millis(5)) + ); + assert!( + handle.schedule(Duration::from_secs(1), async {}).is_err(), + "a task cannot register more tasks once the queue is taken" + ); + assert!(!polled.get()); + drop(armed); + assert!(handle_if_armed().is_none()); + } + + #[test] + fn a_retained_handle_cannot_queue_into_a_later_lifecycle() { + let first = arm(); + let retained = handle_if_armed().expect("first handle"); + drop(first); + let second = arm(); + assert!( + retained.schedule(Duration::from_secs(1), async {}).is_err(), + "a handle from an earlier request is rejected" + ); + assert!(second.take().is_empty()); + } + + #[test] + fn nested_lifecycle_restores_the_outer_queue() { + let outer = arm(); + let outer_handle = handle_if_armed().expect("outer handle"); + outer_handle + .schedule(Duration::from_secs(1), async {}) + .unwrap(); + let inner = arm(); + assert!(inner.take().is_empty(), "the inner lifecycle starts empty"); + drop(inner); + assert_eq!( + outer.take().len(), + 1, + "the outer queue survives the inner lifecycle" + ); + } +} diff --git a/crates/edgezero-adapter-fastly/src/cli.rs b/crates/edgezero-adapter-fastly/src/cli.rs index 6815312a..ac3c1304 100644 --- a/crates/edgezero-adapter-fastly/src/cli.rs +++ b/crates/edgezero-adapter-fastly/src/cli.rs @@ -2038,6 +2038,7 @@ fn owns_managed_deploy(context: &AdapterDeployContext) -> bool { impl Adapter for FastlyCliAdapter { fn capability(&self, capability: Capability) -> CapabilitySupport { match capability { + Capability::AfterResponseTasks => CapabilitySupport::BoundedCooperative, Capability::IngressAdmission | Capability::OutboundAuthorityOverride | Capability::OutboundCacheBypass @@ -6991,6 +6992,10 @@ mod tests { )] fn adapter_capability_matrix_matches_outbound_spec() { for (capability, support) in [ + ( + Capability::AfterResponseTasks, + CapabilitySupport::BoundedCooperative, + ), ( Capability::ConfigReadAllocationBounds, CapabilitySupport::Unsupported, diff --git a/crates/edgezero-adapter-fastly/src/lib.rs b/crates/edgezero-adapter-fastly/src/lib.rs index 94bc30c5..4ad4e20e 100644 --- a/crates/edgezero-adapter-fastly/src/lib.rs +++ b/crates/edgezero-adapter-fastly/src/lib.rs @@ -9,6 +9,8 @@ ) )] +#[cfg(any(feature = "fastly", test))] +mod after_response; // Compile storage helpers only for their runtime, CLI or colocated tests. #[cfg(any( feature = "fastly", diff --git a/crates/edgezero-adapter-fastly/src/request.rs b/crates/edgezero-adapter-fastly/src/request.rs index 214d85e9..9d901cf5 100644 --- a/crates/edgezero-adapter-fastly/src/request.rs +++ b/crates/edgezero-adapter-fastly/src/request.rs @@ -34,6 +34,7 @@ use futures::executor; use futures_util::stream; use std::collections::BTreeMap; +use crate::after_response; use crate::config_store::FastlyConfigStore; use crate::context::{FastlyOutboundObservation, FastlyRequestContext}; use crate::delivery::{ @@ -251,8 +252,11 @@ impl<'app> FastlyService<'app> { { with_callback(|| { init_fastly_abi(); - let outcome = self.dispatch_with(req, prepare); - finish_request(outcome, finalize, deliver) + let clock = self.app.monotonic_clock(); + with_after_response(clock, || { + let outcome = self.dispatch_with(req, prepare); + finish_request(outcome, finalize, deliver) + }) }) } @@ -453,6 +457,17 @@ where scratch } +/// Runs one request lifecycle, then its after-response tasks inside the same callback scope. +/// +/// Registration is open while `run` executes. The tasks start after `run` returns, which is +/// after the response attempt ended, and each is bounded by its budget. +fn with_after_response(clock: MonotonicClock, run: impl FnOnce() -> T) -> T { + let armed = after_response::arm(); + let result = run(); + after_response::drain(armed.take(), clock); + result +} + fn dispatch_with_handles( app: &App, mut req: FastlyRequest, @@ -475,6 +490,10 @@ where ) })?; head_request.extensions_mut().extend(scratch); + // Inserted after the hook's extensions so preparation cannot replace the adapter's handle. + if let Some(handle) = after_response::handle_if_armed() { + head_request.extensions_mut().insert(handle); + } let outcome = dispatch_ingress_reader(app, head_request, stores, request_start, move || { Ok(req.try_take_body()) }); @@ -601,6 +620,42 @@ pub fn send_request_with_registries_and_hooks( where Prepare: FnOnce(&mut FastlyRequest, &mut Extensions), Finalize: FnOnce(&mut Response) -> State, +{ + deliver_with_registries(app, stores, req, prepare, finalize, send_egress_response) +} + +/// Exercises the registry-backed lifecycle used by `run_app` with an injected delivery boundary. +#[cfg(any(test, feature = "test-utils"))] +#[doc(hidden)] +#[inline] +pub fn send_request_with_registries_and_hooks_for_test( + app: &App, + stores: StoresMetadata, + req: FastlyRequest, + prepare: Prepare, + finalize: Finalize, + deliver: Deliver, +) -> FastlyRequestDelivery +where + Prepare: FnOnce(&mut FastlyRequest, &mut Extensions), + Finalize: FnOnce(&mut Response) -> State, + Deliver: FnOnce(ResponseEgressEnvelope) -> FastlyResponseDelivery, +{ + deliver_with_registries(app, stores, req, prepare, finalize, deliver) +} + +fn deliver_with_registries( + app: &App, + stores: StoresMetadata, + req: FastlyRequest, + prepare: Prepare, + finalize: Finalize, + deliver: Deliver, +) -> FastlyRequestDelivery +where + Prepare: FnOnce(&mut FastlyRequest, &mut Extensions), + Finalize: FnOnce(&mut Response) -> State, + Deliver: FnOnce(ResponseEgressEnvelope) -> FastlyResponseDelivery, { with_callback(|| { init_fastly_abi(); @@ -608,19 +663,21 @@ where let kv_registry = build_kv_registry(stores.kv); let config_registry = build_config_registry(stores.config); let secret_registry = build_secret_registry(stores.secrets); - let outcome = dispatch_with_handles( - app, - req, - Stores { - config_registry, - kv_registry, - secret_registry, - ..Default::default() - }, - request_start, - prepare, - ); - finish_request(outcome, finalize, send_egress_response) + with_after_response(app.monotonic_clock(), || { + let outcome = dispatch_with_handles( + app, + req, + Stores { + config_registry, + kv_registry, + secret_registry, + ..Default::default() + }, + request_start, + prepare, + ); + finish_request(outcome, finalize, deliver) + }) }) } diff --git a/crates/edgezero-adapter-fastly/tests/contract.rs b/crates/edgezero-adapter-fastly/tests/contract.rs index 0314ece2..b522c166 100644 --- a/crates/edgezero-adapter-fastly/tests/contract.rs +++ b/crates/edgezero-adapter-fastly/tests/contract.rs @@ -33,7 +33,11 @@ mod tests { use edgezero_adapter_fastly::delivery::{ FastlyRequestDelivery, FastlyResponseCommitState, testing::response_delivery, }; - use edgezero_adapter_fastly::request::{FastlyService, into_core_request}; + use edgezero_adapter_fastly::request::{ + FastlyService, into_core_request, send_request_with_registries_and_hooks_for_test, + }; + use edgezero_core::after_response::{AfterResponseHandle, AfterResponseScheduler}; + use edgezero_core::app::StoresMetadata; use edgezero_core::app::{App, Hooks}; use edgezero_core::body::Body; use edgezero_core::config_store::{ @@ -42,6 +46,7 @@ mod tests { }; use edgezero_core::context::{BodyKind, RequestContext}; use edgezero_core::error::EdgeError; + use edgezero_core::error_detail::ProviderFailureAdapter; use edgezero_core::http::{HeaderMap, Method, Response, StatusCode, response_builder}; use edgezero_core::ingress::{AdmissionDecision, IngressGrant}; use edgezero_core::outbound::{ @@ -54,6 +59,7 @@ mod tests { use fastly::Request as FastlyRequest; use fastly::http::Method as FastlyMethod; use futures::{executor::block_on, stream}; + use futures_util::future::LocalBoxFuture; fn complete_envelope(envelope: ResponseEgressEnvelope) -> Response { complete_envelope_with_report(envelope).0 @@ -843,6 +849,216 @@ mod tests { } #[cfg(feature = "test-utils")] + fn after_response_router( + events: &Arc>>, + budget: Duration, + started: &Arc, + dropped: &Arc, + ) -> RouterService { + struct TaskDropProbe(Arc); + + #[expect( + clippy::missing_trait_methods, + reason = "`Drop::pin_drop` is unstable (`pin_ergonomics`) and conflicts with `drop`" + )] + impl Drop for TaskDropProbe { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + let handler_events = Arc::clone(events); + let handler_started = Arc::clone(started); + let handler_dropped = Arc::clone(dropped); + RouterService::builder() + .get("/after", move |ctx: RequestContext| { + let task_events = Arc::clone(&handler_events); + let task_started = Arc::clone(&handler_started); + let probe = TaskDropProbe(Arc::clone(&handler_dropped)); + async move { + let handle = ctx + .extensions() + .get::() + .cloned() + .expect("an armed Fastly lifecycle installs the handle"); + let time = ctx.http_client().expect("outbound client").time(); + ctx.after_response(budget, async move { + let _probe = probe; + task_started.fetch_add(1, Ordering::SeqCst); + time.wait_until(Deadline::after(Duration::from_millis(10))) + .await + .expect("timers work inside the callback scope"); + let nested = handle.schedule(Duration::from_secs(1), async {}); + task_events + .lock() + .expect("events") + .push(if nested.is_err() { + "task" + } else { + "nested-accepted" + }); + })?; + Ok::<_, EdgeError>("queued") + } + }) + .build() + } + + struct HijackingScheduler; + + impl AfterResponseScheduler for HijackingScheduler { + fn schedule( + &self, + _budget: Duration, + _task: LocalBoxFuture<'static, ()>, + ) -> Result<(), EdgeError> { + Err(EdgeError::internal(anyhow::anyhow!( + "a prepare hook replaced the adapter scheduler" + ))) + } + } + + fn run_after_response( + router: RouterService, + events: &Arc>>, + commitment: FastlyResponseCommitState, + ) -> FastlyRequestDelivery<()> { + let app = App::new(router); + let delivery_events = Arc::clone(events); + let delivered = FastlyService::new(&app).run_request_with_hooks_for_test( + fastly_request(FastlyMethod::GET, "/after", None), + |_raw, extensions| { + // The adapter installs its own handle after the hook's extensions. + extensions.insert(AfterResponseHandle::new(HijackingScheduler)); + }, + |_response| (), + move |envelope| { + let (response, report) = complete_envelope_with_report(envelope); + assert_eq!(response.status(), StatusCode::OK); + delivery_events.lock().expect("events").push("deliver"); + response_delivery(commitment, report, Ok(())) + }, + ); + assert!(matches!(delivered, FastlyRequestDelivery::Routed { .. })); + delivered + } + + #[test] + fn after_response_task_runs_after_delivery_with_timers_and_closed_registration() { + let events = Arc::new(Mutex::new(Vec::new())); + let started = Arc::new(AtomicUsize::new(0)); + let dropped = Arc::new(AtomicUsize::new(0)); + let router = after_response_router(&events, Duration::from_secs(2), &started, &dropped); + let _delivered = run_after_response(router, &events, FastlyResponseCommitState::Committed); + assert_eq!(*events.lock().expect("events"), ["deliver", "task"]); + assert_eq!(dropped.load(Ordering::SeqCst), 1); + } + + #[test] + fn after_response_task_runs_when_delivery_did_not_commit() { + let events = Arc::new(Mutex::new(Vec::new())); + let started = Arc::new(AtomicUsize::new(0)); + let dropped = Arc::new(AtomicUsize::new(0)); + let router = after_response_router(&events, Duration::from_secs(2), &started, &dropped); + let delivered = + run_after_response(router, &events, FastlyResponseCommitState::NotCommitted); + // The caller may still send a native replacement, after the tasks already ran. + assert!(delivered.may_send_native_response()); + assert_eq!(*events.lock().expect("events"), ["deliver", "task"]); + } + + #[test] + fn after_response_task_runs_through_the_registry_lifecycle_used_by_run_app() { + let events = Arc::new(Mutex::new(Vec::new())); + let started = Arc::new(AtomicUsize::new(0)); + let dropped = Arc::new(AtomicUsize::new(0)); + let app = App::new(after_response_router( + &events, + Duration::from_secs(2), + &started, + &dropped, + )); + let delivery_events = Arc::clone(&events); + let delivered = send_request_with_registries_and_hooks_for_test( + &app, + StoresMetadata::default(), + fastly_request(FastlyMethod::GET, "/after", None), + |_raw, extensions| { + extensions.insert(AfterResponseHandle::new(HijackingScheduler)); + }, + |_response| (), + move |envelope| { + let (response, report) = complete_envelope_with_report(envelope); + assert_eq!(response.status(), StatusCode::OK); + delivery_events.lock().expect("events").push("deliver"); + response_delivery(FastlyResponseCommitState::Committed, report, Ok(())) + }, + ); + assert!(matches!(delivered, FastlyRequestDelivery::Routed { .. })); + assert_eq!(*events.lock().expect("events"), ["deliver", "task"]); + } + + #[test] + fn after_response_task_can_start_outbound_requests_after_delivery() { + let outcome = Arc::new(Mutex::new(None)); + let handler_outcome = Arc::clone(&outcome); + let router = RouterService::builder() + .get("/after", move |ctx: RequestContext| { + let task_outcome = Arc::clone(&handler_outcome); + async move { + let client = ctx.http_client().expect("outbound client"); + ctx.after_response(Duration::from_secs(2), async move { + let request = OutboundRequest::get("http://127.0.0.1:9/after-response") + .expect("request") + .timeout(Duration::from_secs(1)); + // A closed outbound scope fails before any exchange and carries no + // provider detail; reaching Fastly's transport (a refused backend + // here) yields a Fastly provider failure. + let reached_transport = match client.send(request).await { + Ok(_response) => true, + Err(error) => error.provider_failure_detail().is_some_and(|detail| { + detail.adapter() == ProviderFailureAdapter::Fastly + }), + }; + *task_outcome.lock().expect("outcome") = Some(reached_transport); + })?; + Ok::<_, EdgeError>("queued") + } + }) + .build(); + let events = Arc::new(Mutex::new(Vec::new())); + let _delivered = run_after_response(router, &events, FastlyResponseCommitState::Committed); + let reached_transport = outcome + .lock() + .expect("outcome") + .expect("the task's outbound request completed"); + assert!( + reached_transport, + "the task's request reached Fastly's transport after delivery" + ); + } + + #[test] + fn after_response_budget_cancels_a_started_task() { + let events = Arc::new(Mutex::new(Vec::new())); + let started = Arc::new(AtomicUsize::new(0)); + let dropped = Arc::new(AtomicUsize::new(0)); + // The task needs 10 ms for its timer; a 1 ms budget cancels it after it started. + let router = after_response_router(&events, Duration::from_millis(1), &started, &dropped); + let _delivered = run_after_response(router, &events, FastlyResponseCommitState::Committed); + assert_eq!(started.load(Ordering::SeqCst), 1, "the task started"); + assert_eq!( + dropped.load(Ordering::SeqCst), + 1, + "the budget dropped the task" + ); + assert_eq!( + *events.lock().expect("events"), + ["deliver"], + "the task never finished" + ); + } + mod ingress_contract { use std::collections::VecDeque; use std::io::{self, Read}; diff --git a/crates/edgezero-adapter-spin/src/cli.rs b/crates/edgezero-adapter-spin/src/cli.rs index ff00d80f..bed2a1e5 100644 --- a/crates/edgezero-adapter-spin/src/cli.rs +++ b/crates/edgezero-adapter-spin/src/cli.rs @@ -161,7 +161,8 @@ impl Adapter for SpinCliAdapter { | Capability::ResponseEgressCompletion | Capability::ResponseWriteDeadlines | Capability::StreamedUploadDeadlines => CapabilitySupport::BestEffort, - Capability::ConfigReadAllocationBounds + Capability::AfterResponseTasks + | Capability::ConfigReadAllocationBounds | Capability::OutboundAuthorityOverride | Capability::OutboundCompleteResourceAccounting | Capability::RawIngressFramingValidation @@ -1410,6 +1411,10 @@ mod tests { )] fn adapter_capability_matrix_matches_contracts() { for (capability, support) in [ + ( + Capability::AfterResponseTasks, + CapabilitySupport::Unsupported, + ), ( Capability::ConfigReadAllocationBounds, CapabilitySupport::Unsupported, diff --git a/crates/edgezero-core/src/after_response.rs b/crates/edgezero-core/src/after_response.rs new file mode 100644 index 00000000..78c3201d --- /dev/null +++ b/crates/edgezero-core/src/after_response.rs @@ -0,0 +1,207 @@ +//! Portable after-response tasks. +//! +//! A handler registers a task through [`RequestContext::after_response`](crate::context::RequestContext::after_response). +//! The adapter starts it once the request's response attempt has ended at the adapter's terminal +//! egress boundary, and cancels it when its budget runs out. Adapters without support install no +//! handle, so registration fails with [`EdgeError::NotImplemented`]. + +use std::fmt; +use std::sync::Arc; +use std::time::Duration; + +use futures_util::future::LocalBoxFuture; + +use crate::error::EdgeError; + +/// Message returned when the adapter installed no after-response scheduler for the request. +pub const AFTER_RESPONSE_UNSUPPORTED: &str = + "after-response tasks are not supported by this adapter"; + +/// Adapter-owned scheduler for one request's after-response tasks. +/// +/// The scheduler is shared through request extensions, so it is `Send + Sync`. The tasks are local +/// futures; each adapter keeps them on the thread that owns the request. +pub trait AfterResponseScheduler: Send + Sync { + /// Queues `task` to start after the response attempt ends, cancelled once `budget` elapses. + /// + /// # Errors + /// Returns an error when the adapter cannot accept the task. The task is dropped unpolled. + fn schedule( + &self, + budget: Duration, + task: LocalBoxFuture<'static, ()>, + ) -> Result<(), EdgeError>; +} + +/// Cloneable handle that adapters insert into request extensions. +#[derive(Clone)] +pub struct AfterResponseHandle(Arc); + +impl AfterResponseHandle { + /// Wraps an adapter scheduler. + #[must_use] + #[inline] + pub fn new(scheduler: S) -> Self + where + S: AfterResponseScheduler + 'static, + { + Self(Arc::new(scheduler)) + } + + /// Registers `task` with a positive `budget`. + /// + /// # Errors + /// Returns an internal error for a zero budget, or the adapter's scheduling error. + #[inline] + pub fn schedule(&self, budget: Duration, task: F) -> Result<(), EdgeError> + where + F: Future + 'static, + { + if budget.is_zero() { + return Err(EdgeError::internal(anyhow::anyhow!( + "after-response task budget must be positive" + ))); + } + self.0.schedule(budget, Box::pin(task)) + } +} + +impl fmt::Debug for AfterResponseHandle { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AfterResponseHandle") + .finish_non_exhaustive() + } +} + +/// Registers `task` through `handle`, or reports the portable capability error. +pub(crate) fn schedule_with( + handle: Option<&AfterResponseHandle>, + budget: Duration, + task: F, +) -> Result<(), EdgeError> +where + F: Future + 'static, +{ + handle.map_or_else( + || Err(EdgeError::not_implemented(AFTER_RESPONSE_UNSUPPORTED)), + |scheduler| scheduler.schedule(budget, task), + ) +} + +#[cfg(test)] +mod tests { + use super::{AFTER_RESPONSE_UNSUPPORTED, AfterResponseHandle, AfterResponseScheduler}; + use crate::context::RequestContext; + use crate::error::EdgeError; + use crate::http::{Method, StatusCode, request_builder}; + use crate::{Body, params::PathParams}; + use futures::executor::block_on; + use futures_util::future::LocalBoxFuture; + use std::cell::{Cell, RefCell}; + use std::rc::Rc; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + struct DropProbe(Rc>); + + #[expect( + clippy::missing_trait_methods, + reason = "`Drop::pin_drop` is unstable (`pin_ergonomics`) and conflicts with `drop`" + )] + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.set(true); + } + } + + #[derive(Default)] + struct Recording { + budgets: Mutex>, + } + + thread_local! { + static RECORDED: RefCell>> = + const { RefCell::new(Vec::new()) }; + } + + impl AfterResponseScheduler for Arc { + #[inline] + fn schedule( + &self, + budget: Duration, + task: LocalBoxFuture<'static, ()>, + ) -> Result<(), EdgeError> { + if let Ok(mut budgets) = self.budgets.lock() { + budgets.push(budget); + } + RECORDED.with(|recorded| recorded.borrow_mut().push(task)); + Ok(()) + } + } + + fn context_with(scheduler: Option) -> RequestContext { + let mut request = request_builder() + .method(Method::GET) + .uri("/") + .body(Body::empty()) + .unwrap(); + if let Some(handle) = scheduler { + request.extensions_mut().insert(handle); + } + RequestContext::new(request, PathParams::default()) + } + + #[test] + fn missing_scheduler_reports_not_implemented_and_never_polls() { + let ctx = context_with(None); + let polled = Rc::new(Cell::new(false)); + let dropped = Rc::new(Cell::new(false)); + let probe = DropProbe(Rc::clone(&dropped)); + let observed = Rc::clone(&polled); + let error = ctx + .after_response(Duration::from_secs(1), async move { + let _probe = probe; + observed.set(true); + }) + .unwrap_err(); + assert_eq!(error.status(), StatusCode::NOT_IMPLEMENTED); + assert_eq!(error.message(), AFTER_RESPONSE_UNSUPPORTED); + assert!(!polled.get(), "an unsupported task is never polled"); + assert!( + dropped.get(), + "an unsupported task is dropped at registration" + ); + } + + #[test] + fn zero_budget_is_rejected_before_the_scheduler() { + let recording = Arc::new(Recording::default()); + let ctx = context_with(Some(AfterResponseHandle::new(Arc::clone(&recording)))); + let error = ctx.after_response(Duration::ZERO, async {}).unwrap_err(); + assert_eq!(error.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!(recording.budgets.lock().unwrap().is_empty()); + } + + #[test] + fn scheduler_receives_the_exact_budget_and_a_runnable_task() { + let recording = Arc::new(Recording::default()); + let ctx = context_with(Some(AfterResponseHandle::new(Arc::clone(&recording)))); + let ran = Rc::new(Cell::new(false)); + let observed = Rc::clone(&ran); + ctx.after_response(Duration::from_millis(250), async move { + observed.set(true); + }) + .unwrap(); + assert_eq!( + *recording.budgets.lock().unwrap(), + [Duration::from_millis(250)] + ); + assert!(!ran.get(), "registration does not run the task"); + let task = RECORDED + .with(|recorded| recorded.borrow_mut().pop()) + .unwrap(); + block_on(task); + assert!(ran.get()); + } +} diff --git a/crates/edgezero-core/src/context.rs b/crates/edgezero-core/src/context.rs index dce1c1bd..6bcc2c9f 100644 --- a/crates/edgezero-core/src/context.rs +++ b/crates/edgezero-core/src/context.rs @@ -4,8 +4,10 @@ use std::{ env, mem, rc::Rc, task::Poll, + time::Duration, }; +use crate::after_response::{AfterResponseHandle, schedule_with}; use crate::body::{Body, BodyStream}; use crate::config_store::ConfigExtractionLimits; use crate::error::{ @@ -326,6 +328,28 @@ impl Drop for DrainGuard<'_> { } impl RequestContext { + /// Registers `task` to run after this request's response leaves the adapter. + /// + /// The task starts once the response attempt ends at the adapter's terminal egress boundary, + /// whatever its outcome, and is cancelled when `budget` elapses. Start outbound requests + /// inside `task`: a send future created beforehand belongs to the request's scope. + /// + /// # Errors + /// Returns [`EdgeError::NotImplemented`] when the adapter does not support after-response + /// tasks, an internal error for a zero `budget`, or the adapter's scheduling error. The task + /// is dropped unpolled on error. + #[inline] + pub fn after_response(&self, budget: Duration, task: F) -> Result<(), EdgeError> + where + F: Future + 'static, + { + schedule_with( + self.parts.extensions.get::(), + budget, + task, + ) + } + /// Drains and caches the inbound body under the caller's byte cap. /// /// # Errors diff --git a/crates/edgezero-core/src/lib.rs b/crates/edgezero-core/src/lib.rs index 80fe7a46..3b380610 100644 --- a/crates/edgezero-core/src/lib.rs +++ b/crates/edgezero-core/src/lib.rs @@ -14,6 +14,7 @@ extern crate self as edgezero_core; pub mod addr; +pub mod after_response; pub mod app; pub mod app_config; pub mod blob_envelope; diff --git a/crates/edgezero-core/src/manifest.rs b/crates/edgezero-core/src/manifest.rs index cd1f4964..29f4e8c9 100644 --- a/crates/edgezero-core/src/manifest.rs +++ b/crates/edgezero-core/src/manifest.rs @@ -84,6 +84,8 @@ impl BakedManifest { #[serde(rename_all = "kebab-case")] #[non_exhaustive] pub enum Capability { + /// Bounded tasks that start after the response leaves the adapter. + AfterResponseTasks, BufferedResponseContentLength, ConfigReadAllocationBounds, ConfigReadDeadlines, @@ -130,6 +132,7 @@ impl Capability { #[must_use] pub fn as_str(&self) -> &'static str { match self { + Self::AfterResponseTasks => "after-response-tasks", Self::BufferedResponseContentLength => "buffered-response-content-length", Self::ConfigReadAllocationBounds => "config-read-allocation-bounds", Self::ConfigReadDeadlines => "config-read-deadlines", @@ -1832,6 +1835,7 @@ env = "APP_TOKEN" let source = r#" [capabilities] required = [ + "after-response-tasks", "buffered-response-content-length", "config-read-allocation-bounds", "config-read-deadlines", @@ -1868,6 +1872,7 @@ hosts = ["*", "HTTPS://Example.COM", "api.example.com:8443"] "#; let loader = ManifestLoader::try_load_from_str(source).expect("capability manifest"); let expected = [ + Capability::AfterResponseTasks, Capability::BufferedResponseContentLength, Capability::ConfigReadAllocationBounds, Capability::ConfigReadDeadlines, @@ -1908,6 +1913,7 @@ hosts = ["*", "HTTPS://Example.COM", "api.example.com:8443"] ) ); for (capability, expected_name) in expected.iter().zip([ + "after-response-tasks", "buffered-response-content-length", "config-read-allocation-bounds", "config-read-deadlines", diff --git a/docs/guide/adapters/axum.md b/docs/guide/adapters/axum.md index 1c01b4c0..b126f2cd 100644 --- a/docs/guide/adapters/axum.md +++ b/docs/guide/adapters/axum.md @@ -580,6 +580,19 @@ extensions, and body. Restarting the server creates a fresh app. Native Axum measurements are a useful ownership reference but do not establish WASM runtime performance or resource lifetimes. +## After-response tasks + +`ctx.after_response(budget, task)` spawns the task on the server's executor. It starts when Hyper +drops the response body, after the final frame is handed over or when the connection ends early, and +is cancelled at its budget. On SIGTERM or SIGINT, the production entrypoints, `run_app` and +`AxumDevServer::run` first drain connections, then wait for registered tasks, all within one grace +(`EDGEZERO__ADAPTER__SHUTDOWN_GRACE_SECONDS`, or a fixed 10 s for `AxumDevServer::run`). Tasks still pending at the grace are cancelled +and the process exits non-zero with `grace exhausted`; a second signal still stops immediately. +`AxumDevServer::serve` cancels tasks at shutdown, like its connections. +Tasks share the server's single-threaded executor with every connection. The budget cancels a +task only at an await point, so a CPU-bound or blocking task overruns its budget and delays all +request serving. + ## Ingress provenance Conversion captures runtime URI parts and retains the original HeaderMap and diff --git a/docs/guide/adapters/fastly.md b/docs/guide/adapters/fastly.md index 03be0ae7..43b6cb8c 100644 --- a/docs/guide/adapters/fastly.md +++ b/docs/guide/adapters/fastly.md @@ -481,6 +481,17 @@ response-egress capabilities at BestEffort. See the [Streaming guide](/guide/streaming) and [capability matrix](/guide/capabilities#outbound-matrix) for the exact boundary. +## After-Response Tasks + +`ctx.after_response(budget, task)` queues the task for the current request. After the response is +sent, and before the request callback returns, the adapter drives queued tasks in the same outbound +scope, so outbound requests and timers work. Each budget is checked when the task yields to the +adapter's progress driver; a blocked hostcall cannot be preempted. Tasks delay the next request on +a reused `serve_app` instance. When delivery did not commit and the caller may send its own native +response, the tasks have already run by the time the lifecycle returns. Requests converted with the +public `into_core_request` outside a lifecycle get no scheduler, so registration returns `501`. +A handle retained from an earlier request is rejected rather than queued into a later one. + ## Testing Run contract tests for the Fastly adapter: diff --git a/docs/guide/capabilities.md b/docs/guide/capabilities.md index 7f99f6de..daa814ff 100644 --- a/docs/guide/capabilities.md +++ b/docs/guide/capabilities.md @@ -328,6 +328,36 @@ terminal report. The boundaries remain `BestEffort` because none proves end-clie Provider-owned buffering, SDK allocations, and bytes beyond each stated acceptance boundary are not included. These limitations block whole-response-lifetime certification. +## After-Response Task Matrix + +| Capability | Axum | Cloudflare | Fastly | Spin | +| ---------------------- | ------------------ | ----------- | ------------------ | ----------- | +| `after-response-tasks` | BoundedCooperative | Unsupported | BoundedCooperative | Unsupported | + +`ctx.after_response(budget, task)` registers a task while a request is handled. The adapter starts +the task once the response attempt ends at its terminal egress boundary, whatever the outcome. That +boundary is the one the response-egress cells describe, so it does not prove the client received the +response. The task is dropped, and so cancelled, when its budget elapses. Registration closes when +the request's tasks are released, so a task cannot register more tasks. Start outbound requests +inside the task. + +Both supporting adapters are `BoundedCooperative`: the budget cancels a task only when it yields, +so CPU-bound or blocking work overruns it. + +- Axum spawns each task on the server's executor and releases it when Hyper drops the response body. + Shutdown through the production entrypoints, `run_app` and `AxumDevServer::run` (fixed 10 s grace) + waits for registered tasks after connections drain, within the same grace + (`EDGEZERO__ADAPTER__SHUTDOWN_GRACE_SECONDS` where configurable). Tasks still pending at the grace + are cancelled, and the process exits with the existing `grace exhausted` failure; a second signal + still stops immediately. The prebound `AxumDevServer::serve` embedding cancels them, consistent + with its cancel-not-drain contract. Blocking work in a task also delays the shared executor. +- Fastly drives the tasks after the response is sent, before the request callback returns, inside + the same outbound scope. The budget is checked whenever a task yields to the adapter's progress + driver; a blocked hostcall cannot be preempted. The tasks delay the next request on a reused + `serve_app` instance, and a caller-sent native replacement after an uncommitted delivery. +- Cloudflare and Spin install no scheduler, so registration returns `501 Not Implemented`. Cloudflare + `waitUntil` support is tracked in [#478](https://github.com/IABTechLab/edgezero/issues/478). + ## Outbound Matrix | Capability | Axum | Cloudflare | Fastly | Spin | diff --git a/docs/guide/handlers.md b/docs/guide/handlers.md index 3029c103..92aacd6a 100644 --- a/docs/guide/handlers.md +++ b/docs/guide/handlers.md @@ -209,20 +209,58 @@ async fn inspect(ctx: RequestContext) -> Result, EdgeError> { explicit validated-extractor limits. For unvalidated data, use `ctx.json_within::(max).await` or `ctx.form_within::(max).await`. -| Method | Returns | -| ----------------------------- | ------------------------------------------------------------ | -| `method()` | `&Method` - request method | -| `uri()` | `&Uri` - request URI | -| `headers()` | `&HeaderMap` - request headers | -| `path_params()` | `&PathParams` - raw path parameters | -| `path::()` | Deserialize path params to `T` | -| `query::()` | Deserialize query string to `T` | -| `json_within::(max).await` | Bounded JSON deserialization as `Result` | -| `form_within::(max).await` | Bounded form deserialization as `Result` | -| `body_bytes(max).await` | `Result` - bounded, cached body read | -| `take_body()` | `Result` - transfer body ownership | -| `into_request()` | `Result` - consume context, take request | -| `http_client()` | `Option` - adapter outbound HTTP client | +| Method | Returns | +| ------------------------------ | ------------------------------------------------------------ | +| `method()` | `&Method` - request method | +| `uri()` | `&Uri` - request URI | +| `headers()` | `&HeaderMap` - request headers | +| `path_params()` | `&PathParams` - raw path parameters | +| `path::()` | Deserialize path params to `T` | +| `query::()` | Deserialize query string to `T` | +| `json_within::(max).await` | Bounded JSON deserialization as `Result` | +| `form_within::(max).await` | Bounded form deserialization as `Result` | +| `body_bytes(max).await` | `Result` - bounded, cached body read | +| `take_body()` | `Result` - transfer body ownership | +| `into_request()` | `Result` - consume context, take request | +| `http_client()` | `Option` - adapter outbound HTTP client | +| `after_response(budget, task)` | `Result<(), EdgeError>` - run `task` after the response | + +### After-Response Tasks + +Work that must not delay the response, such as a follow-up store write or a sync request, can run +after it. The task starts once the response attempt ends and is cancelled when its budget elapses: + +```rust +use std::time::Duration; +use edgezero_core::context::RequestContext; +use edgezero_core::outbound::OutboundRequest; +use edgezero_core::EdgeError; + +async fn page(ctx: RequestContext) -> Result<&'static str, EdgeError> { + if let Some(client) = ctx.http_client() { + let registered = ctx.after_response(Duration::from_secs(2), async move { + // Build and send the request inside the task, not before it. + let sent = match OutboundRequest::get("https://sync.example.com/ping") { + Ok(request) => client.send(request).await.map(|_response| ()), + Err(error) => Err(error), + }; + if let Err(error) = sent { + log::warn!("after-response sync failed: {error}"); + } + }); + // Optional work must not fail the page on an adapter without support. + if let Err(error) = registered { + log::warn!("after-response sync not scheduled: {error}"); + } + } + Ok("ok") +} +``` + +Registration returns `501 Not Implemented` on adapters without support; see the +[after-response task matrix](./capabilities.md#after-response-task-matrix). Declare +`after-response-tasks` as a required capability when the work is not optional. Tasks return `()`, so +handle and log their own failures. A task cannot register further tasks. ### Store Extractors diff --git a/docs/superpowers/plans/2026-11-17-after-response-tasks.md b/docs/superpowers/plans/2026-11-17-after-response-tasks.md new file mode 100644 index 00000000..c4b63fe6 --- /dev/null +++ b/docs/superpowers/plans/2026-11-17-after-response-tasks.md @@ -0,0 +1,115 @@ +# After-Response Tasks Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox syntax for tracking. + +**Goal:** Issue #476. A task registered during a request runs after the response leaves the adapter, bounded by its budget, on Fastly and Axum. Axum SIGTERM waits for registered tasks within the grace. Other adapters return a capability error. + +**Architecture:** A portable `AfterResponseHandle` sits in request extensions, and `RequestContext::after_response` reads it. Axum spawns each task on its `LocalSet`, gated on response-body drop, counted by a tracker and drained by `drain_tasks`. Fastly queues tasks in a request-local, thread-local queue and drives them after `finish_request`, inside the callback scope. + +**Tech stack:** Rust 1.98.1 (`cargo +1.98.1`), tokio (existing features), futures. No new dependencies. + +**Spec:** [After-response tasks design](../specs/2026-11-17-after-response-tasks-design.md). + +**Base:** stacked on PR #406 (`issue-392-396-production-hosting-spec`, `e37e0df2`). + +## Task 1: Core contract + +**Files:** `crates/edgezero-core/src/after_response.rs` (new), `lib.rs`, `context.rs`, `manifest.rs`. + +- [x] RED: tests in `after_response.rs` for three cases. + - No scheduler: registration returns `NotImplemented` with status 501, and the task is never polled. A drop probe shows the future was dropped. + - Zero budget: registration returns an internal (500) error, and the scheduler is never called. + - Recording scheduler: it receives the exact budget, and the stored future runs to its side effect when polled. +- [x] GREEN: add the `AfterResponseScheduler` trait, the `AfterResponseHandle` handle (`new`, `schedule`) and `RequestContext::after_response`. Export the module from `lib.rs`. +- [x] Add `Capability::AfterResponseTasks` → `"after-response-tasks"` and extend the manifest capability list test. +- [x] Verify: `cargo +1.98.1 test -p edgezero-core` and `cargo +1.98.1 check -p edgezero-core --target wasm32-unknown-unknown`. + +## Task 2: Axum scheduler, delivery gate and budget + +**Files:** `crates/edgezero-adapter-axum/src/after_response.rs` (new), `lib.rs`, `service.rs`, `response.rs`. + +- [x] Add `AfterResponseTracker`, an `Arc>`. + - `register()` returns a guard that decrements the count on drop. Both updates use `send_modify`. + - `wait_idle()` waits until the count is zero. + - `pending()` returns the current count. +- [x] Add `AxumAfterResponse { executor, tracker, delivered: watch::Receiver<()> }`, which implements `AfterResponseScheduler`. Its spawned task holds the guard, awaits `delivered.changed()` (an error means the sender was dropped, i.e. the request was released), then binds `let scope = OutboundScopeOwner::callback();` and runs `timeout(budget, scope.run(task))`, warning on timeout. `schedule` returns an internal error once the release has been dropped (`delivered.has_changed()` is `Err`); a spawn error is returned. The guard is created before `spawn` and moved into the future. +- [x] `AxumServiceState` gains a `tracker` field, set through `with_after_response_tracker`. Constructors default to a fresh tracker. +- [x] `dispatch_scoped_request`: + - Creates `watch::channel(())` before `dispatch_admitted`. + - Inserts `AfterResponseHandle::new(AxumAfterResponse{..})` into the core request extensions. + - Attaches the sender to the returned response through `AxumEgressBody::release_after_response_on_drop`. +- [x] `AxumEgressBody` gains `after_response: Option`. Plain drop order releases it after the attempt terminates in `Drop`. `finish()` runs on EOF, and Hyper drops the body after the final frame. +- [x] RED/GREEN server tests in `tests/after_response.rs`, through the public `AxumDevServer::serve` and a real listener: + - **Delivery gate:** the response body is a stream gated by a oneshot. The handler registers a task that sets a flag. + 1. Read the headers and assert the flag is false. + 2. Release the final chunk, read to the end over a keep-alive connection that stays open, then assert the flag becomes true within 1 s. + - **Budget (executor-level unit test):** the task sets `started`, then awaits `pending()` with a drop probe and a 50 ms budget. Assert it started, the probe is alive at 49 ms, then dropped, and the tracker is zero. + - **Registration after release** returns an error, and the task is never polled. + - **Outbound inside a task:** the task sends a request to a local upstream listener after delivery. Assert the upstream received it. + - **Error response:** a handler that registers a task and then returns `Err` still runs the task. +- [x] Verify: `cargo +1.98.1 test -p edgezero-adapter-axum`. + +## Task 3: Axum SIGTERM drain + +**Files:** `crates/edgezero-adapter-axum/src/dev_server.rs`, `tests/fixtures/production_host.rs`, `tests/production_hosting.rs`. + +- [x] `run_owned` creates one tracker, passes it to the service, and hands it to `drain_tasks`. +- [x] `drain_tasks`: after the connection loop, while the outcome is `Ok`, select on a second signal, `sleep_until(deadline)` and `tracker.wait_idle()`. + - On timeout, log `"{n} after-response task(s) abandoned at shutdown grace"` and return `grace exhausted`. + - The `cancel_connections` (prebound) path does not wait. +- [x] Fixture route `/after-task`: it registers a task that sleeps for `FIXTURE_TASK_MILLIS` (default 500), then prints `AFTER_TASK_DONE`, and responds `200 after-task-queued`. +- [x] Subprocess tests: + - **`sigterm_waits_for_after_response_task`:** a 1.5 s task. Request `/after-task`, read the response, send SIGTERM immediately, and assert `AFTER_TASK_DONE` has not been printed yet. Then expect `AFTER_TASK_DONE`, followed by a clean exit (status 0). + - **`grace_exhaustion_cancels_after_response_task`:** grace 1 s, task 10 s, SIGTERM. Expect a non-zero exit, `grace exhausted` in the log, no `AFTER_TASK_DONE`, and elapsed time under 4 s. +- [x] Subprocess tests after review: `connections_and_after_response_tasks_share_one_grace` (800 ms drain, 1 s grace, must exit well before 1.8 s) and `second_signal_during_after_response_wait_forces_stop`. The task wait selects idle ahead of the deadline and logs abandoned tasks on every exit path. +- [x] Verify: `cargo +1.98.1 test -p edgezero-adapter-axum --test production_hosting`, plus the existing tests unchanged. + +## Task 4: Fastly queue and drain + +**Files:** `crates/edgezero-adapter-fastly/src/after_response.rs` (new), `lib.rs`, `request.rs`, `tests/contract.rs`. + +- [x] Add a thread-local `QUEUE: RefCell)>>>`. + - `arm()` returns a guard that restores the previous queue. + - `FastlyAfterResponse` schedules into the queue, or errors when it is not armed. + - One helper, `with_after_response(clock, dispatch_and_finish)`, arms, runs the closure, then `drain(clock)`. `drain` disarms and takes the tasks (no borrow is held while polling), then joins them, each raced against `native_time(clock).wait_until(start + budget)`, under `block_on(drive_callback(..))`. It warns on timeout or on a driver error. Each deadline is `Deadline::at_instant(start + budget)`, saturating. +- [x] `dispatch_with_handles` inserts `AfterResponseHandle::new(FastlyAfterResponse)` only while a queue is armed, after the `prepare` scratch extensions are copied. `into_core_request` and `capture_egress_for_test` get no handle. +- [x] `run_request_with_delivery` and `send_request_with_registries_and_hooks` both call `with_after_response` inside `with_callback`, with the clock read from the app before dispatch. The registries funnel gains `send_request_with_registries_and_hooks_for_test`. Each lifecycle has a generation, so a retained handle is rejected. +- [x] Host unit tests (feature off): + - The handle is inserted only while armed. + - An armed queue collects tasks, and the guard restores the previous queue. +- [x] Viceroy contract tests through `run_request_with_hooks_for_test`: + - The `deliver` closure pushes `deliver`. The task first awaits a 10 ms timer, then pushes `task`. Expected order: `[deliver, task]`, through both funnels, with a `prepare` hook inserting a rejecting handle. + - A task that sets `started` and then needs a 10 ms timer, with a 1 ms budget and a drop probe: `started` is set, the probe has dropped, and the task never finished. + - A task's outbound request to a refused backend reports a Fastly provider failure (the exchange ran after delivery). + - With no commitment, `may_send_native_response()` is true and the task already ran. + - A task that tries to register a second task gets an error. + - A `deliver` closure reporting no commitment: tasks still run before the funnel returns. +- [x] Verify: `cargo +1.98.1 test -p edgezero-adapter-fastly`, and the wasm contract suite under Viceroy (see the tool note below). + +## Task 5: Capability matrices and docs + +- [x] Adapter `capability()` entries: + - Axum: `BoundedCooperative`. + - Fastly: `BoundedCooperative`. + - Cloudflare and Spin: `Unsupported`, falling through to the wildcard, with an explicit arm for readability. +- [x] Update each `adapter_capability_matrix_matches_contracts` test. +- [x] `docs/guide/capabilities.md`: a new `## After-Response Tasks` section with the matrix and its semantics. +- [x] `docs/guide/handlers.md`: a usage example. +- [x] `docs/guide/adapters/axum.md`: the SIGTERM drain includes tasks. +- [x] `docs/guide/adapters/fastly.md`: tasks run after send and before the callback returns, including the native-replacement note. +- [x] `CHANGELOG.md` under `## Unreleased`: the new API, and the fact that Axum SIGTERM can now exit non-zero when tasks outlive the grace. Keep it out of the migration table that the checker compares. +- [x] Verify the docs: `node scripts/check_outbound_docs_contract.mjs`, then in `docs/`: `npm ci && npx prettier --check . && npm run lint`. + +## Task 6: Gates + +- [x] `cargo +1.98.1 fmt --all -- --check` +- [x] `cargo +1.98.1 clippy --workspace --all-targets --all-features -- -D warnings` +- [x] `cargo +1.98.1 test --workspace --all-targets` +- [x] `cargo +1.98.1 check --workspace --all-targets --features "fastly cloudflare spin"` +- [x] `cargo +1.98.1 check -p edgezero-adapter-spin --target wasm32-wasip2 --features spin` +- [x] `cargo +1.98.1 check -p edgezero-adapter-fastly --target wasm32-wasip1 --features fastly` +- [x] `cargo +1.98.1 check -p edgezero-core --target wasm32-unknown-unknown` + +## Tool note + +`.tool-versions` pins Viceroy 0.21.0, but the machine's default binary is 0.17.0. The Fastly contract and wasm library suites ran under a stock Viceroy 0.21.0, built with `cargo install viceroy --version =0.21.0 --locked --root `, the same way `scripts/ci_viceroy.sh stock` builds it. CI's patched readiness build was not used locally. Docs checks ran under Node 24.20.0. diff --git a/docs/superpowers/specs/2026-11-17-after-response-tasks-design.md b/docs/superpowers/specs/2026-11-17-after-response-tasks-design.md new file mode 100644 index 00000000..b8524fc4 --- /dev/null +++ b/docs/superpowers/specs/2026-11-17-after-response-tasks-design.md @@ -0,0 +1,100 @@ +# After-response tasks on Fastly and Axum + +Date: 2026-11-17 +Status: implemented. Two independent review rounds resolved every blocking and optional finding before implementation. + +Issue: [#476](https://github.com/IABTechLab/edgezero/issues/476). Follow-ups: [#478](https://github.com/IABTechLab/edgezero/issues/478) (Cloudflare `waitUntil`) and [#483](https://github.com/IABTechLab/edgezero/issues/483) (Spin capability error). + +Base: [PR #406](https://github.com/IABTechLab/edgezero/pull/406) at `e37e0df2`, which carries #275, #499 and #460. This work needs #406's SIGTERM drain and #275's Fastly callback scope. + +## Goal + +A handler or middleware registers a task while it handles a request. The adapter runs the task after the response leaves the adapter, and stops it when its time budget runs out. On Axum, SIGTERM waits for registered tasks within the existing shutdown grace. Adapters without support return a capability error when the task is registered. They never drop it silently. + +Trusted Server needs this for pull sync and guarded follow-up record writes (its ID-5 and AX-1). The work must not delay the document response. + +## Contract + +```rust +impl RequestContext { + pub fn after_response(&self, budget: Duration, task: F) -> Result<(), EdgeError> + where + F: Future + 'static; +} +``` + +- **Registration** succeeds only when the adapter installed a scheduler for this request. Otherwise it returns `EdgeError::NotImplemented` (HTTP 501), with the message `after-response tasks are not supported by this adapter`. Cloudflare and Spin behave this way until #478 and #483 land. +- **A zero budget** is a programming error. Registration returns `EdgeError::internal` (500), not a client-blaming 422, and the task is never polled. +- **Start.** The task is not polled before the request's response attempt ends at the adapter's terminal egress boundary. It starts whatever the outcome: completed, client disconnected, deadline exceeded, error response or fallback. This boundary is the same one `ResponseEgressOutcome::HostHandoff` describes. It is not proof that the client received the bytes. The work exists for the server's own state, so client receipt does not gate it. Cloudflare's `waitUntil` starts its promise immediately, so #478 must add its own end-of-response gate to meet this rule. +- **Budget.** Each task gets its own budget, measured from the moment the task may start. A task that is still pending when the budget runs out is dropped, which cancels it, and the adapter logs a warning. The task's output is `()`. The application handles and logs its own failures. Retries belong to the application, as Trusted Server's "retried on the next navigation" does. +- **Ordering.** Tasks from one request run concurrently, with no ordering guarantee among them or across requests. +- **Futures need not be `Send`.** They follow the portable rule that core futures are `?Send`. They may capture request handles such as `HttpClient` or `KvHandle` clones, and they may use outbound HTTP and timers. +- **Start outbound work inside the task.** Outbound futures capture the ambient callback scope when they are created. A send future created in the handler belongs to the request's scope, which closes when the request ends on Axum, so it is revoked. Write `ctx.after_response(budget, async move { client.send(request).await; })`, not a future built beforehand. +- **Registration window.** Registration is open while the request is being handled, from dispatch until the response attempt ends. Once the request's tasks are released, a retained `AfterResponseHandle` returns an internal error. A task therefore cannot register more tasks, and the total work per request is bounded by the budgets registered during the request. On Fastly, each armed lifecycle has a generation number, so a handle retained from an earlier request on `serve_app`'s reused instance is rejected rather than queued into a later request. + +Core types (in `edgezero_core::after_response`): + +- `AfterResponseScheduler: Send + Sync` is an adapter trait with `fn schedule(&self, budget: Duration, task: LocalBoxFuture<'static, ()>) -> Result<(), EdgeError>`. +- `AfterResponseHandle` is a cloneable `Arc` handle. Adapters insert it into request extensions. `RequestContext::after_response` reads it. + +The handle is `Send + Sync`, as request extensions require. The futures are local, so each adapter keeps them on its own thread (see below). This follows the pattern already used by `AxumOutboundExecutorHandle`. + +## Manifest capability + +New `Capability::AfterResponseTasks` (`after-response-tasks`). Applications can list it under `[capabilities] required`, so `edgezero build`, `serve` and `deploy` refuse an adapter that cannot run the tasks. + +| Adapter | Support | Reason | +| ---------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | +| Axum | `BoundedCooperative` | Tokio timer cancels at the budget, at await points only; drained on SIGTERM within the grace. The prebound `serve()` embedding cancels, by its contract. | +| Fastly | `BoundedCooperative` | The budget is checked whenever the task yields to the adapter's progress driver; Fastly cannot preempt a blocked source or hostcall. | +| Cloudflare | `Unsupported` | #478 adds `waitUntil`. | +| Spin | `Unsupported` | #483 keeps the capability error. | + +## Axum + +- **Scheduler.** The scheduler holds the request's `AxumOutboundExecutorHandle`, the server's task tracker and a delivery receiver. `schedule` fails with an internal error once the request's release has been dropped. Otherwise it registers the task with the tracker and spawns it at once with `spawn_local` on the existing `LocalSet`; a spawn error is returned to the caller. Before polling the application future, the spawned task waits for the delivery signal. It then runs the future under `tokio::time::timeout(budget, …)`, inside a fresh `OutboundScopeOwner::callback().run(..)`, whose owner is bound to a local for the whole await. That scope gives outbound work started in the task a managed owner, which is revoked when the task finishes, times out or is cancelled. +- **Delivery signal.** `dispatch_scoped_request` creates a `watch` channel before dispatch and moves its sender into the `AxumEgressBody` that it returns. Hyper drops the body after its final frame, or when the connection ends early. Dropping the body drops the sender, which releases every task the request registered. If the service future is cancelled before a body exists, the sender is dropped with that future, and the tasks still run. Requests rejected before dispatch never reach a handler, so they register nothing. +- **Tracker.** An `Arc` holds a `watch::Sender` (updated with `send_modify`) that counts tasks registered but not yet finished. A guard, created before `spawn` and moved into the spawned future, decrements the count; if `spawn` fails, dropping the future drops the guard. It also decrements on completion, timeout, panic or cancellation. The tracker is created by the lifecycle owner (`run_owned`) and shared by every connection. `AxumServiceState` constructors used outside `run_owned` create their own tracker, which nothing drains. +- **SIGTERM drain.** `drain_tasks` keeps its single deadline (`now + grace`). It first reaps connections as it does today, then waits for the tracker to reach zero, still under that deadline; an already-idle tracker wins over a deadline that passed meanwhile. A second signal still forces an immediate stop. If the grace runs out with tasks pending, it returns the existing `grace exhausted` failure, so the process exits non-zero. On every exit path it logs how many tasks were abandoned. This applies to the production entrypoints, `run_app` and `AxumDevServer::run`. Leaving `run_local` drops the `LocalSet`, which cancels the remaining tasks. As with handlers today, runtime drop still waits for in-flight blocking-pool work such as DNS resolution. +- **Unchanged paths.** The prebound `serve()` embedding path keeps its documented cancel-not-drain semantics: tasks are cancelled with the runtime. Neither the development `run_app` path nor the production path changes its signals or grace configuration (`EDGEZERO__ADAPTER__SHUTDOWN_GRACE_SECONDS`, default 10 s). + +## Fastly + +- **Scheduler.** A `Send + Sync` scheduler, stamped with its lifecycle's generation, pushes the task and its budget into a thread-local queue when that lifecycle is still the armed one. Each request lifecycle arms the queue and takes it back afterwards. `dispatch_with_handles` inserts the handle only while a queue is armed. It inserts the handle after copying the `prepare` hook's scratch extensions, so a hook cannot replace it. Requests converted by the public `into_core_request` (for example under `serve_custom` or `run_custom` with manual dispatch) and `capture_egress_for_test` therefore get no handle, and registration returns the portable 501. +- **Arming and draining.** Both request funnels, `FastlyService::run_request_with_delivery` and `send_request_with_registries_and_hooks`, wrap dispatch and `finish_request` in `with_after_response`, which arms the queue first and drains afterwards. Each funnel has a test-utils entry with an injected delivery boundary (`run_request_with_hooks_for_test`, `send_request_with_registries_and_hooks_for_test`), so CI exercises both. The drain clock is read from the app before dispatch. After `finish_request` returns, still inside the same `with_callback` scope, they take the queued tasks out of the queue (so no `RefCell` borrow is held while polling) and drive them with `executor::block_on(drive_callback(...))`. The queue is disarmed before the tasks are polled, so registration from inside a task fails, as on Axum. Running inside that scope keeps outbound requests and timers working. Each task races `native_time(clock).wait_until(Deadline::at_instant(start + budget))`, saturating to `DEADLINE_FAR_FUTURE`. The arm guard restores the previous queue even if a nested lifecycle runs. +- **Covered entry points.** Every application entry point already funnels through these two functions: `run_app`, `run_app_with_hooks`, `serve_app*`, `run_app_with_config`, `FastlyService::send*` and the test-utils `run_request_with_hooks_for_test`. The generated `main` and the templates do not change. `send_assembly_error` has no application and registers nothing. +- **Instance lifetime.** The response has already been sent, so the tasks delay only the return from the request callback. On `serve_app`'s reused instance, that delays the next request on that instance. This is documented. +- **Native replacement.** When delivery did not commit and `FastlyRequestDelivery::may_send_native_response()` allows the caller to send its own replacement response, the tasks have already run by the time the funnel returns. They delay that replacement by at most the longest registered budget. This rare failure path is documented and tested. Skipping the tasks would drop work silently, and moving the drain outside the funnel would lose the callback scope. + +## Cloudflare and Spin + +No scheduler is installed, so registration returns the 501 capability error. Their private egress `spawn` hooks stay unchanged. #478 and #483 own the change. + +## Preservation and acceptance + +| Scenario | Current | Intended | Evidence | +| ------------------------------------------------- | ------------------------------------ | -------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | +| Axum handler registers a task, streaming response | n/a | Task not polled until the final body frame is handed to Hyper | In-crate server test: gated streaming body, keep-alive connection; task flag false before release, true after the client reads the full body | +| Axum task exceeds its budget | n/a | Task started, then dropped at the budget; tracker returns to zero | Executor-level unit test: started flag, probe alive before the budget, dropped after, tracker zero | +| Registration after release | n/a | Internal error, task never polled | Axum unit test and Fastly host test | +| Axum SIGTERM with a pending task | Connections drained only | Process waits for the task within the grace, exits 0 | Subprocess test: a 1.5 s task, SIGTERM sent right after the response; the marker is absent at SIGTERM, then printed, then a clean exit | +| Axum SIGTERM, task longer than the grace | n/a | `grace exhausted`, non-zero exit, task cancelled | Subprocess test with a 1 s grace and a long task; no completion marker | +| Fastly handler registers a task | n/a | Runs after `deliver` returns, inside the callback scope | Viceroy contract test: recorded order is `delivered` then `task`; a timer inside the task works; registration from inside a task fails | +| Fastly delivery not committed | n/a | Tasks still run before the funnel returns | Viceroy contract test: no commitment, `may_send_native_response()` is true, and the task already ran | +| Fastly task uses outbound HTTP | n/a | The request reaches Fastly's transport after delivery | Viceroy contract test: a refused backend yields a Fastly provider failure, not a closed-scope error | +| Axum connections and tasks share the grace | n/a | No fresh deadline for tasks | Subprocess test: an 800 ms drain plus a pending task exit near the 1 s grace, not ~1.8 s | +| Axum second signal during the task wait | n/a | Immediate forced stop | Subprocess test | +| Fastly manual `into_core_request` | n/a | No handle; 501 at registration | Host unit test: no handle while unarmed; a retained handle is rejected by a later lifecycle | +| Fastly `prepare` hook inserts its own handle | n/a | The adapter handle wins | Viceroy contract tests insert a rejecting handle in `prepare`; tasks still run | +| Fastly `run_app` / `serve_app` funnel | n/a | Same behavior as the service funnel | Viceroy contract test through `send_request_with_registries_and_hooks_for_test` | +| Axum task uses outbound HTTP | n/a | Request started inside the task succeeds after delivery | In-crate test against a local upstream listener | +| Fastly task exceeds its budget | n/a | Dropped at the budget | Viceroy contract test with a pending task and a short budget | +| No scheduler (Cloudflare, Spin, bare core) | n/a | 501 `not_implemented` at registration | Core unit test | +| Existing Axum shutdown and egress behavior | Drain, grace, second signal, cancel | Unchanged when no tasks are registered | Existing `production_hosting` and egress tests | +| Capability gate | n/a | `required = ["after-response-tasks"]` passes on Axum and Fastly, fails on Cloudflare and Spin | Adapter matrix tests and the manifest list test | + +## Out of scope + +- Cloudflare `waitUntil` (#478) and Spin's adapter-named error (#483). +- Retry policy, persistence of failed tasks, or a global concurrency limit. +- Proof of client receipt. This uses the same egress boundary the response-egress matrix already documents. diff --git a/scripts/check_outbound_docs_contract.mjs b/scripts/check_outbound_docs_contract.mjs index a5dbd13c..ed100ece 100644 --- a/scripts/check_outbound_docs_contract.mjs +++ b/scripts/check_outbound_docs_contract.mjs @@ -232,6 +232,15 @@ const expectedResponseEgressRows = [ 'BestEffort', ], ] +const expectedAfterResponseRows = [ + [ + 'after-response-tasks', + 'BoundedCooperative', + 'Unsupported', + 'BoundedCooperative', + 'Unsupported', + ], +] const expectedLimitHeader = ['Control', 'Scope', 'Default'] const expectedLimitRows = [ [ @@ -1904,6 +1913,14 @@ const matrices = [ 'response egress' ), ], + [ + 'after-response task', + expectedAfterResponseRows, + readMatrix( + findMatrixHeader('After-Response Task Matrix', 'after-response task'), + 'after-response task' + ), + ], [ 'outbound', expectedOutboundRows,