diff --git a/scripts/cluster_gateway_bringup.sh b/scripts/cluster_gateway_bringup.sh index a3fbe4da..81e76dcd 100755 --- a/scripts/cluster_gateway_bringup.sh +++ b/scripts/cluster_gateway_bringup.sh @@ -16,6 +16,9 @@ # GATEWAY_ENV extra NAME=VALUE pairs for the gateway, space separated, e.g. # "MATRIXOBJECT_CLUSTER_COMPACT_APPEND=1" # CHECKPOINT_BYTES MATRIXOBJECT_METASERVER_RAFT_CHECKPOINT_BYTES for raft3 voters (default unset) +# HEARTBEAT_TTL_MS how long a chunk server stays healthy after its last heartbeat (default 15000, five +# of the default 3000 ms intervals: a TTL under two intervals expires a node on one +# late heartbeat and placement then refuses every write) # # Only processes this script started are signalled: their pids are recorded in $RUN/pids and # `down` reads that file. Nothing is matched by name. @@ -100,7 +103,7 @@ up() { MATRIXOBJECT_METASERVER_ADDR="127.0.0.1:$((BASE))" \ MATRIXOBJECT_METASERVER_CHUNK_GROUPS_FILE="$RUN/catalog.bin" \ MATRIXOBJECT_METASERVER_STATE_DIR="$RUN/meta1" \ - MATRIXOBJECT_METASERVER_HEARTBEAT_TTL_MS=5000 \ + MATRIXOBJECT_METASERVER_HEARTBEAT_TTL_MS="${HEARTBEAT_TTL_MS:-15000}" \ "$BIN/metaserver" wait_line "$RUN/meta1.log" 20 || fail "metaserver did not report listening; see $RUN/meta1.log" META_EPS="127.0.0.1:$BASE" @@ -118,7 +121,7 @@ up() { MATRIXOBJECT_METASERVER_RAFT_STATE_DIR="$RUN/meta$i" \ MATRIXOBJECT_METASERVER_RAFT_TIMEOUT_MS=3000 "${CKPT_ENV[@]}" \ MATRIXOBJECT_METASERVER_CHUNK_GROUPS_FILE="$RUN/catalog.bin" \ - MATRIXOBJECT_METASERVER_HEARTBEAT_TTL_MS=5000 \ + MATRIXOBJECT_METASERVER_HEARTBEAT_TTL_MS="${HEARTBEAT_TTL_MS:-15000}" \ "$BIN/metaserver" META_EPS="$META_EPS${META_EPS:+,}127.0.0.1:$((BASE + 3 + i))" done diff --git a/src/net/cluster_gateway.rs b/src/net/cluster_gateway.rs index 26b0709f..255f51cc 100644 --- a/src/net/cluster_gateway.rs +++ b/src/net/cluster_gateway.rs @@ -829,6 +829,8 @@ impl ClusterGateway { let resource = format!("/{bucket}/{key}"); const ATTEMPTS: usize = 4; let mut last_staged_mismatch: Option = None; + // The refusal that made the last attempt retry, so a gateway that gives up can say why. + let mut last_retried = String::new(); for attempt in 0..ATTEMPTS { // A retry exists because the end moved, so it never trusts the cache. let cached = if attempt == 0 { @@ -900,6 +902,7 @@ impl ClusterGateway { // A refusal decided from the manifest this call read answers the client as // it stands; only a lost publish race is worth another look. if !decided && lost_race(&failure.error) { + last_retried = failure.error.to_string(); if staged_state_mismatch(&failure.error) { last_staged_mismatch = Some(failure.error); } @@ -908,6 +911,7 @@ impl ClusterGateway { // A refusal decided from a REMEMBERED manifest may only mean the memory is // stale (another gateway appended), so it is checked once against a read. if decided && used_cache && lost_race(&failure.error) { + last_retried = failure.error.to_string(); continue; } return error_response(failure.error, &resource, false); @@ -921,7 +925,9 @@ impl ClusterGateway { } s3::error_response( s3::S3ErrorCode::PreconditionFailed, - "the object kept changing under this append; gave up after several attempts", + &format!( + "the object kept changing under this append; gave up after several attempts (the last refusal: {last_retried})" + ), &resource, ) } @@ -1157,10 +1163,21 @@ fn closed_range(header: &str) -> Option<(u64, u64)> { /// Whether a refusal says another writer published first, as opposed to a refusal of the /// request itself. fn lost_race(error: &ObjectError) -> bool { - (matches!(error, ObjectError::PreconditionFailed(_)) && !plane_unreachable(error)) + (matches!(error, ObjectError::PreconditionFailed(_)) + && !plane_unreachable(error) + && !placement_unavailable(error)) || staged_state_mismatch(error) } +/// The metadata plane has no healthy data group to place the write on (the chunk servers' +/// heartbeats have lapsed, for example when one is missed under load). Nothing is wrong with the +/// client's request and nothing changed under it: it is the cluster being briefly unable, which +/// clears when the next heartbeat lands, so it is a retryable 503, not a lost race to retry at +/// once (which would only be refused again) and not a 412. +fn placement_unavailable(error: &ObjectError) -> bool { + matches!(error, ObjectError::PreconditionFailed(message) if message.contains("could not satisfy replica count")) +} + /// The SDK refuses a staged write as an invalid argument when the allocation it holds no longer /// matches the metadata record (a reservation that went stale, a generation that moved, a lease /// that was renewed or swept). Nothing is wrong with the client's request, and a fresh @@ -1281,6 +1298,7 @@ fn error_response(error: ObjectError, resource: &str, missing_is_bucket: bool) - s3::S3ErrorCode::InvalidRequest } _ if plane_unreachable(&error) + || placement_unavailable(&error) || staged_state_mismatch(&error) || consensus_unavailable(&error) => { @@ -1870,6 +1888,32 @@ mod tests { assert_eq!(append(&gateway, "wal", 3, b"de").status, 200); } + const NO_HEALTHY_GROUP: &str = "precondition failed: root placement could not satisfy replica count"; + + #[test] + fn a_placement_with_no_healthy_group_is_a_503_and_not_a_lost_race() { + let error = ObjectError::PreconditionFailed(NO_HEALTHY_GROUP.into()); + assert!(!lost_race(&error), "it must not be retried at once as a race"); + let (gateway, armed) = timeout_gateway(error, false); + assert_eq!(append(&gateway, "wal", 0, b"abc").status, 200); + *armed.lock().unwrap() = true; + let response = append(&gateway, "wal", 3, b"de"); + assert_eq!(response.status, 503, "{}", String::from_utf8_lossy(&response.body)); + assert!(String::from_utf8_lossy(&response.body).contains("could not satisfy replica count")); + assert_eq!(gateway.handle(&request("GET", "/b/wal", &[], b"")).body, b"abc", "nothing was written"); + // the client's retry at the same offset lands once the groups are healthy again + assert_eq!(append(&gateway, "wal", 3, b"de").status, 200); + } + + #[test] + fn a_real_lost_race_is_still_a_412() { + // another writer moved the object: a precondition refusal that is not the placement's + let error = ObjectError::PreconditionFailed("object manifest changed before write".into()); + assert!(lost_race(&error)); + let response = error_response(error, "/b/k", false); + assert_eq!(response.status, 412); + } + #[test] fn a_full_consensus_queue_is_a_503_and_never_a_400() { let (gateway, armed) = timeout_gateway(