From c61e2afcfcacfaed5aa52651acdbfd12b297bd3d Mon Sep 17 00:00:00 2001 From: matrixarkai Date: Tue, 17 Nov 2026 21:31:33 -0800 Subject: [PATCH 1/2] Say why an append gave up after several attempts The 412 the gateway answers when every attempt of an append was refused as a lost race did not carry the refusal, so a failure that was not a race (for example a bound) looked like one. The message now ends with the last refusal. Co-Authored-By: Claude Sonnet 5.5 --- src/net/cluster_gateway.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/net/cluster_gateway.rs b/src/net/cluster_gateway.rs index 26b0709f..a46998ac 100644 --- a/src/net/cluster_gateway.rs +++ b/src/net/cluster_gateway.rs @@ -829,6 +829,8 @@ impl ClusterGateway { let resource = format!("/{bucket}/{key}"); const ATTEMPTS: usize = 4; let mut last_staged_mismatch: Option = None; + // The refusal that made the last attempt retry, so a gateway that gives up can say why. + let mut last_retried = String::new(); for attempt in 0..ATTEMPTS { // A retry exists because the end moved, so it never trusts the cache. let cached = if attempt == 0 { @@ -900,6 +902,7 @@ impl ClusterGateway { // A refusal decided from the manifest this call read answers the client as // it stands; only a lost publish race is worth another look. if !decided && lost_race(&failure.error) { + last_retried = failure.error.to_string(); if staged_state_mismatch(&failure.error) { last_staged_mismatch = Some(failure.error); } @@ -908,6 +911,7 @@ impl ClusterGateway { // A refusal decided from a REMEMBERED manifest may only mean the memory is // stale (another gateway appended), so it is checked once against a read. if decided && used_cache && lost_race(&failure.error) { + last_retried = failure.error.to_string(); continue; } return error_response(failure.error, &resource, false); @@ -921,7 +925,9 @@ impl ClusterGateway { } s3::error_response( s3::S3ErrorCode::PreconditionFailed, - "the object kept changing under this append; gave up after several attempts", + &format!( + "the object kept changing under this append; gave up after several attempts (the last refusal: {last_retried})" + ), &resource, ) } From 981456a5fd9226bfc75f5dcc96ab89476f697f12 Mon Sep 17 00:00:00 2001 From: matrixarkai Date: Tue, 17 Nov 2026 21:48:56 -0800 Subject: [PATCH 2/2] Answer a placement with no healthy group as 503, and keep the chunk heartbeat TTL above one interval When every chunk server has lapsed (one late heartbeat expires a node when the TTL is under two heartbeat intervals) the metadata plane refuses a write with root placement could not satisfy replica count. The gateway took that PreconditionFailed for a lost race, retried it at once and answered 412 the object kept changing under this append, which told the client its offsets were wrong. It is the cluster being briefly unable: a retryable 503 with the reason, and no immediate retry. The bring-up script set the heartbeat TTL to 5000 ms against the default 3000 ms interval; the default is now 15000 ms, five intervals. Co-Authored-By: Claude Sonnet 5.5 --- scripts/cluster_gateway_bringup.sh | 7 ++++-- src/net/cluster_gateway.rs | 40 +++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/scripts/cluster_gateway_bringup.sh b/scripts/cluster_gateway_bringup.sh index a3fbe4da..81e76dcd 100755 --- a/scripts/cluster_gateway_bringup.sh +++ b/scripts/cluster_gateway_bringup.sh @@ -16,6 +16,9 @@ # GATEWAY_ENV extra NAME=VALUE pairs for the gateway, space separated, e.g. # "MATRIXOBJECT_CLUSTER_COMPACT_APPEND=1" # CHECKPOINT_BYTES MATRIXOBJECT_METASERVER_RAFT_CHECKPOINT_BYTES for raft3 voters (default unset) +# HEARTBEAT_TTL_MS how long a chunk server stays healthy after its last heartbeat (default 15000, five +# of the default 3000 ms intervals: a TTL under two intervals expires a node on one +# late heartbeat and placement then refuses every write) # # Only processes this script started are signalled: their pids are recorded in $RUN/pids and # `down` reads that file. Nothing is matched by name. @@ -100,7 +103,7 @@ up() { MATRIXOBJECT_METASERVER_ADDR="127.0.0.1:$((BASE))" \ MATRIXOBJECT_METASERVER_CHUNK_GROUPS_FILE="$RUN/catalog.bin" \ MATRIXOBJECT_METASERVER_STATE_DIR="$RUN/meta1" \ - MATRIXOBJECT_METASERVER_HEARTBEAT_TTL_MS=5000 \ + MATRIXOBJECT_METASERVER_HEARTBEAT_TTL_MS="${HEARTBEAT_TTL_MS:-15000}" \ "$BIN/metaserver" wait_line "$RUN/meta1.log" 20 || fail "metaserver did not report listening; see $RUN/meta1.log" META_EPS="127.0.0.1:$BASE" @@ -118,7 +121,7 @@ up() { MATRIXOBJECT_METASERVER_RAFT_STATE_DIR="$RUN/meta$i" \ MATRIXOBJECT_METASERVER_RAFT_TIMEOUT_MS=3000 "${CKPT_ENV[@]}" \ MATRIXOBJECT_METASERVER_CHUNK_GROUPS_FILE="$RUN/catalog.bin" \ - MATRIXOBJECT_METASERVER_HEARTBEAT_TTL_MS=5000 \ + MATRIXOBJECT_METASERVER_HEARTBEAT_TTL_MS="${HEARTBEAT_TTL_MS:-15000}" \ "$BIN/metaserver" META_EPS="$META_EPS${META_EPS:+,}127.0.0.1:$((BASE + 3 + i))" done diff --git a/src/net/cluster_gateway.rs b/src/net/cluster_gateway.rs index a46998ac..255f51cc 100644 --- a/src/net/cluster_gateway.rs +++ b/src/net/cluster_gateway.rs @@ -1163,10 +1163,21 @@ fn closed_range(header: &str) -> Option<(u64, u64)> { /// Whether a refusal says another writer published first, as opposed to a refusal of the /// request itself. fn lost_race(error: &ObjectError) -> bool { - (matches!(error, ObjectError::PreconditionFailed(_)) && !plane_unreachable(error)) + (matches!(error, ObjectError::PreconditionFailed(_)) + && !plane_unreachable(error) + && !placement_unavailable(error)) || staged_state_mismatch(error) } +/// The metadata plane has no healthy data group to place the write on (the chunk servers' +/// heartbeats have lapsed, for example when one is missed under load). Nothing is wrong with the +/// client's request and nothing changed under it: it is the cluster being briefly unable, which +/// clears when the next heartbeat lands, so it is a retryable 503, not a lost race to retry at +/// once (which would only be refused again) and not a 412. +fn placement_unavailable(error: &ObjectError) -> bool { + matches!(error, ObjectError::PreconditionFailed(message) if message.contains("could not satisfy replica count")) +} + /// The SDK refuses a staged write as an invalid argument when the allocation it holds no longer /// matches the metadata record (a reservation that went stale, a generation that moved, a lease /// that was renewed or swept). Nothing is wrong with the client's request, and a fresh @@ -1287,6 +1298,7 @@ fn error_response(error: ObjectError, resource: &str, missing_is_bucket: bool) - s3::S3ErrorCode::InvalidRequest } _ if plane_unreachable(&error) + || placement_unavailable(&error) || staged_state_mismatch(&error) || consensus_unavailable(&error) => { @@ -1876,6 +1888,32 @@ mod tests { assert_eq!(append(&gateway, "wal", 3, b"de").status, 200); } + const NO_HEALTHY_GROUP: &str = "precondition failed: root placement could not satisfy replica count"; + + #[test] + fn a_placement_with_no_healthy_group_is_a_503_and_not_a_lost_race() { + let error = ObjectError::PreconditionFailed(NO_HEALTHY_GROUP.into()); + assert!(!lost_race(&error), "it must not be retried at once as a race"); + let (gateway, armed) = timeout_gateway(error, false); + assert_eq!(append(&gateway, "wal", 0, b"abc").status, 200); + *armed.lock().unwrap() = true; + let response = append(&gateway, "wal", 3, b"de"); + assert_eq!(response.status, 503, "{}", String::from_utf8_lossy(&response.body)); + assert!(String::from_utf8_lossy(&response.body).contains("could not satisfy replica count")); + assert_eq!(gateway.handle(&request("GET", "/b/wal", &[], b"")).body, b"abc", "nothing was written"); + // the client's retry at the same offset lands once the groups are healthy again + assert_eq!(append(&gateway, "wal", 3, b"de").status, 200); + } + + #[test] + fn a_real_lost_race_is_still_a_412() { + // another writer moved the object: a precondition refusal that is not the placement's + let error = ObjectError::PreconditionFailed("object manifest changed before write".into()); + assert!(lost_race(&error)); + let response = error_response(error, "/b/k", false); + assert_eq!(response.status, 412); + } + #[test] fn a_full_consensus_queue_is_a_503_and_never_a_400() { let (gateway, armed) = timeout_gateway(