From 4ebd8b24263db07ffa147529e0d1a3cdf95faa89 Mon Sep 17 00:00:00 2001 From: matrixarkai Date: Wed, 18 Nov 2026 00:08:01 -0800 Subject: [PATCH 1/2] Retry a busy replica for a few tens of milliseconds before skipping a heartbeat The heartbeat takes the replica lock with try_lock, which a request thread holds while it applies a write. When it was busy the report failed and the next try was a whole heartbeat interval away (3 s by default), so under write load a node could skip report after report and, with a TTL of a few intervals, drop out of every group. The report now waits in 30 ms steps for up to five tries (stopping at once on shutdown) before it gives up on the interval. Co-Authored-By: Claude Sonnet 5.5 --- src/cluster/server_heartbeat.rs | 87 ++++++++++++++++++++++++++++++--- 1 file changed, 79 insertions(+), 8 deletions(-) diff --git a/src/cluster/server_heartbeat.rs b/src/cluster/server_heartbeat.rs index 39101e17..ec148dcf 100644 --- a/src/cluster/server_heartbeat.rs +++ b/src/cluster/server_heartbeat.rs @@ -78,6 +78,36 @@ fn local_membership( Ok(status.clone()) } +/// How long the heartbeat waits for a busy replica between tries, and how many times it tries +/// before it gives up on this interval. A request thread holds the replica for as long as one +/// write takes to apply; the report needs the replica for a moment only, so a short wait gets it +/// without a full heartbeat interval (3 s by default) being lost to one contended try. +const BUSY_RETRY_STEP: Duration = Duration::from_millis(30); +const BUSY_TRIES: usize = 5; + +/// The replica for a health report: waits in short steps while a request thread holds it, and +/// gives up (reporting that it was busy) only after [`BUSY_TRIES`] tries, or at once when the +/// server is stopping. +fn lock_for_report<'a>( + replica: &'a Mutex, + stop: &ServerShutdown, +) -> Result, crate::ObjectError> { + for attempt in 0..BUSY_TRIES { + match replica.try_lock() { + Ok(state) => return Ok(state), + Err(std::sync::TryLockError::Poisoned(_)) => { + return Err(crate::ObjectError::Corruption("replica health lock poisoned".into())) + } + Err(std::sync::TryLockError::WouldBlock) => { + if attempt + 1 == BUSY_TRIES || stop.wait_for(BUSY_RETRY_STEP) { + break; + } + } + } + } + Err(crate::ObjectError::PreconditionFailed("replica health is busy applying state".into())) +} + pub(super) fn run( options: ReplicaHeartbeatOptions, replica: Arc>, @@ -116,14 +146,7 @@ pub(super) fn run( #[cfg(feature = "chunk-consensus")] let dynamic = consensus.as_ref().filter(|c| c.identity().version == 2); let (used, capacity, online, membership) = { - let mut state = replica.try_lock().map_err(|error| match error { - std::sync::TryLockError::WouldBlock => crate::ObjectError::PreconditionFailed( - "replica health is busy applying state".into(), - ), - std::sync::TryLockError::Poisoned(_) => { - crate::ObjectError::Corruption("replica health lock poisoned".into()) - } - })?; + let mut state = lock_for_report(&replica, &stop)?; if let Some(journal) = state.journal.as_mut() { if journal.check_health().is_err() { state.failed = true; @@ -449,6 +472,54 @@ mod tests { assert!(local_membership(&state).is_err()); } + #[test] + fn a_busy_replica_delays_the_report_by_the_wait_and_not_by_a_whole_interval() { + let meta = MetaNodeServer::bind("127.0.0.1:0", MetaNodeOptions::default()).unwrap(); + let mut chunk = ReplicaNodeServer::bind("127.0.0.1:0", ReplicaNodeOptions::default()).unwrap(); + let endpoint = chunk.local_addr().unwrap(); + let metadata = meta.local_addr().unwrap(); + meta.seed_data_roster(&DataNodeSeed::parse_roster(&format!("chunk-1@{endpoint}@rack")).unwrap(), 1 << 28) + .unwrap(); + chunk + .enable_heartbeats(ReplicaHeartbeatOptions { + placement_id: None, + tls: None, + expected_metadata_identity: None, + node_id: "chunk-1".into(), + advertised_endpoint: endpoint, + metadata_endpoints: vec![metadata], + // a whole interval is far longer than the test waits + interval: Duration::from_secs(30), + }) + .unwrap(); + let state = chunk.replica.clone(); + let chunk_stop = chunk.shutdown_handle(); + let meta_stop = meta.shutdown_handle(); + let meta_worker = std::thread::spawn(move || meta.serve().unwrap()); + // the replica is busy when the first report is due, and for less than the tries cover + let guard = state.lock().unwrap(); + let chunk_worker = std::thread::spawn(move || chunk.serve().unwrap()); + let mut client = RemoteMetaClient::connect(&metadata.to_string(), &MetaClientOptions::default()).unwrap(); + assert_eq!(client.list_placement_nodes().unwrap()[0].last_heartbeat_ms, 0); + std::thread::sleep(Duration::from_millis(60)); + drop(guard); + let until = std::time::Instant::now() + Duration::from_secs(3); + loop { + if client.list_placement_nodes().unwrap()[0].last_heartbeat_ms != 0 { + break; + } + assert!( + std::time::Instant::now() < until, + "the report waited a whole interval for a replica that was busy for 60 ms" + ); + std::thread::sleep(Duration::from_millis(20)); + } + chunk_stop.stop(); + meta_stop.stop(); + chunk_worker.join().unwrap(); + meta_worker.join().unwrap(); + } + #[test] fn busy_replica_state_does_not_block_heartbeat_shutdown() { let meta = MetaNodeServer::bind("127.0.0.1:0", MetaNodeOptions::default()).unwrap(); From b1af16c145336e395e3870afad21dd40c9308216 Mon Sep 17 00:00:00 2001 From: matrixarkai Date: Wed, 18 Nov 2026 00:45:20 -0800 Subject: [PATCH 2/2] Pin the heartbeat guard at 8 tests The guard counts the tests of cluster::server::heartbeat::tests; the busy replica test added for the quick retry is one of them. Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8e5b222..82fbff53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -186,8 +186,8 @@ jobs: run: | set -euo pipefail cargo test --locked --all-features --lib cluster::server::heartbeat::tests -- --test-threads=1 | tee /tmp/fenced-health.log - grep -q '^running 7 tests$' /tmp/fenced-health.log - grep -q '^test result: ok. 7 passed' /tmp/fenced-health.log + grep -q '^running 8 tests$' /tmp/fenced-health.log + grep -q '^test result: ok. 8 passed' /tmp/fenced-health.log - name: catalog commands preserve durable state and quorum admission run: |