diff --git a/scripts/cluster_gateway_bringup.sh b/scripts/cluster_gateway_bringup.sh index 81e76dcd..b40a5227 100755 --- a/scripts/cluster_gateway_bringup.sh +++ b/scripts/cluster_gateway_bringup.sh @@ -16,6 +16,7 @@ # GATEWAY_ENV extra NAME=VALUE pairs for the gateway, space separated, e.g. # "MATRIXOBJECT_CLUSTER_COMPACT_APPEND=1" # CHECKPOINT_BYTES MATRIXOBJECT_METASERVER_RAFT_CHECKPOINT_BYTES for raft3 voters (default unset) +# CHUNK_HEARTBEAT_MS chunk server heartbeat interval (default 200 here, 3000 in the product) # HEARTBEAT_TTL_MS how long a chunk server stays healthy after its last heartbeat (default 15000, five # of the default 3000 ms intervals: a TTL under two intervals expires a node on one # late heartbeat and placement then refuses every write) @@ -36,6 +37,12 @@ PIDS="$RUN/pids" fail() { echo "STOP: $*" >&2; exit 1; } +# a chunk server that is not heard from for HEARTBEAT_TTL_MS is placed out of every group: with a TTL under +# three heartbeat intervals one heartbeat skipped under load takes a node out until the next one lands +HB_INTERVAL_MS=${CHUNK_HEARTBEAT_MS:-200} +TTL_MS=${HEARTBEAT_TTL_MS:-15000} +[ "$TTL_MS" -ge $((3 * HB_INTERVAL_MS)) ] || { echo "STOP: HEARTBEAT_TTL_MS=$TTL_MS is under three chunk heartbeat intervals ($HB_INTERVAL_MS ms)" >&2; exit 1; } + port_free() { ! ss -lnt 2>/dev/null | awk '{print $4}' | grep -q ":$1\$"; } # wait for "listening on" in a log, up to N seconds @@ -141,7 +148,7 @@ up() { MATRIXOBJECT_REPLICA_MAX_PAGES="$MAX_PAGES" \ MATRIXOBJECT_CHUNK_NODE_ID="chunk-$i" \ MATRIXOBJECT_CHUNK_METADATA_ENDPOINTS="$META_EPS" \ - MATRIXOBJECT_CHUNK_HEARTBEAT_MS=200 \ + MATRIXOBJECT_CHUNK_HEARTBEAT_MS="$HB_INTERVAL_MS" \ "$BIN/chunkserver" done for i in 1 2 3; do wait_line "$RUN/chunk$i.log" 20 || fail "chunkserver $i did not report listening; see $RUN/chunk$i.log"; done diff --git a/src/net/cluster_gateway.rs b/src/net/cluster_gateway.rs index 255f51cc..bc035c44 100644 --- a/src/net/cluster_gateway.rs +++ b/src/net/cluster_gateway.rs @@ -1165,7 +1165,8 @@ fn closed_range(header: &str) -> Option<(u64, u64)> { fn lost_race(error: &ObjectError) -> bool { (matches!(error, ObjectError::PreconditionFailed(_)) && !plane_unreachable(error) - && !placement_unavailable(error)) + && !placement_unavailable(error) + && !leader_unavailable(error)) || staged_state_mismatch(error) } @@ -1174,6 +1175,13 @@ fn lost_race(error: &ObjectError) -> bool { /// client's request and nothing changed under it: it is the cluster being briefly unable, which /// clears when the next heartbeat lands, so it is a retryable 503, not a lost race to retry at /// once (which would only be refused again) and not a 412. +/// No metadata leader admitted the request (an election, or a quorum that has not settled, as in +/// the first seconds of a cluster): nothing changed under the client's request, and retrying it at +/// once only meets the same answer. A retryable 503, like [`placement_unavailable`]. +fn leader_unavailable(error: &ObjectError) -> bool { + matches!(error, ObjectError::PreconditionFailed(message) if message.contains("was not admitted by a leader")) +} + fn placement_unavailable(error: &ObjectError) -> bool { matches!(error, ObjectError::PreconditionFailed(message) if message.contains("could not satisfy replica count")) } @@ -1299,6 +1307,7 @@ fn error_response(error: ObjectError, resource: &str, missing_is_bucket: bool) - } _ if plane_unreachable(&error) || placement_unavailable(&error) + || leader_unavailable(&error) || staged_state_mismatch(&error) || consensus_unavailable(&error) => { @@ -1905,6 +1914,19 @@ mod tests { assert_eq!(append(&gateway, "wal", 3, b"de").status, 200); } + #[test] + fn a_request_no_leader_admitted_is_a_503_and_not_a_lost_race() { + let error = ObjectError::PreconditionFailed("precondition failed: metadata request was not admitted by a leader".into()); + assert!(!lost_race(&error)); + let (gateway, armed) = timeout_gateway(error, false); + assert_eq!(append(&gateway, "wal", 0, b"abc").status, 200); + *armed.lock().unwrap() = true; + let response = append(&gateway, "wal", 3, b"de"); + assert_eq!(response.status, 503, "{}", String::from_utf8_lossy(&response.body)); + assert!(String::from_utf8_lossy(&response.body).contains("not admitted by a leader")); + assert_eq!(append(&gateway, "wal", 3, b"de").status, 200); + } + #[test] fn a_real_lost_race_is_still_a_412() { // another writer moved the object: a precondition refusal that is not the placement's