Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion scripts/cluster_gateway_bringup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
# GATEWAY_ENV extra NAME=VALUE pairs for the gateway, space separated, e.g.
# "MATRIXOBJECT_CLUSTER_COMPACT_APPEND=1"
# CHECKPOINT_BYTES MATRIXOBJECT_METASERVER_RAFT_CHECKPOINT_BYTES for raft3 voters (default unset)
# CHUNK_HEARTBEAT_MS chunk server heartbeat interval (default 200 here, 3000 in the product)
# HEARTBEAT_TTL_MS how long a chunk server stays healthy after its last heartbeat (default 15000, five
# of the default 3000 ms intervals: a TTL under two intervals expires a node on one
# late heartbeat and placement then refuses every write)
Expand All @@ -36,6 +37,12 @@ PIDS="$RUN/pids"

fail() { echo "STOP: $*" >&2; exit 1; }

# a chunk server that is not heard from for HEARTBEAT_TTL_MS is placed out of every group: with a TTL under
# three heartbeat intervals one heartbeat skipped under load takes a node out until the next one lands
HB_INTERVAL_MS=${CHUNK_HEARTBEAT_MS:-200}
TTL_MS=${HEARTBEAT_TTL_MS:-15000}
[ "$TTL_MS" -ge $((3 * HB_INTERVAL_MS)) ] || { echo "STOP: HEARTBEAT_TTL_MS=$TTL_MS is under three chunk heartbeat intervals ($HB_INTERVAL_MS ms)" >&2; exit 1; }

port_free() { ! ss -lnt 2>/dev/null | awk '{print $4}' | grep -q ":$1\$"; }

# wait for "listening on" in a log, up to N seconds
Expand Down Expand Up @@ -141,7 +148,7 @@ up() {
MATRIXOBJECT_REPLICA_MAX_PAGES="$MAX_PAGES" \
MATRIXOBJECT_CHUNK_NODE_ID="chunk-$i" \
MATRIXOBJECT_CHUNK_METADATA_ENDPOINTS="$META_EPS" \
MATRIXOBJECT_CHUNK_HEARTBEAT_MS=200 \
MATRIXOBJECT_CHUNK_HEARTBEAT_MS="$HB_INTERVAL_MS" \
"$BIN/chunkserver"
done
for i in 1 2 3; do wait_line "$RUN/chunk$i.log" 20 || fail "chunkserver $i did not report listening; see $RUN/chunk$i.log"; done
Expand Down
24 changes: 23 additions & 1 deletion src/net/cluster_gateway.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1165,7 +1165,8 @@ fn closed_range(header: &str) -> Option<(u64, u64)> {
fn lost_race(error: &ObjectError) -> bool {
(matches!(error, ObjectError::PreconditionFailed(_))
&& !plane_unreachable(error)
&& !placement_unavailable(error))
&& !placement_unavailable(error)
&& !leader_unavailable(error))
|| staged_state_mismatch(error)
}

Expand All @@ -1174,6 +1175,13 @@ fn lost_race(error: &ObjectError) -> bool {
/// client's request and nothing changed under it: it is the cluster being briefly unable, which
/// clears when the next heartbeat lands, so it is a retryable 503, not a lost race to retry at
/// once (which would only be refused again) and not a 412.
/// No metadata leader admitted the request (an election, or a quorum that has not settled, as in
/// the first seconds of a cluster): nothing changed under the client's request, and retrying it at
/// once only meets the same answer. A retryable 503, like [`placement_unavailable`].
fn leader_unavailable(error: &ObjectError) -> bool {
matches!(error, ObjectError::PreconditionFailed(message) if message.contains("was not admitted by a leader"))
}

fn placement_unavailable(error: &ObjectError) -> bool {
matches!(error, ObjectError::PreconditionFailed(message) if message.contains("could not satisfy replica count"))
}
Expand Down Expand Up @@ -1299,6 +1307,7 @@ fn error_response(error: ObjectError, resource: &str, missing_is_bucket: bool) -
}
_ if plane_unreachable(&error)
|| placement_unavailable(&error)
|| leader_unavailable(&error)
|| staged_state_mismatch(&error)
|| consensus_unavailable(&error) =>
{
Expand Down Expand Up @@ -1905,6 +1914,19 @@ mod tests {
assert_eq!(append(&gateway, "wal", 3, b"de").status, 200);
}

#[test]
fn a_request_no_leader_admitted_is_a_503_and_not_a_lost_race() {
let error = ObjectError::PreconditionFailed("precondition failed: metadata request was not admitted by a leader".into());
assert!(!lost_race(&error));
let (gateway, armed) = timeout_gateway(error, false);
assert_eq!(append(&gateway, "wal", 0, b"abc").status, 200);
*armed.lock().unwrap() = true;
let response = append(&gateway, "wal", 3, b"de");
assert_eq!(response.status, 503, "{}", String::from_utf8_lossy(&response.body));
assert!(String::from_utf8_lossy(&response.body).contains("not admitted by a leader"));
assert_eq!(append(&gateway, "wal", 3, b"de").status, 200);
}

#[test]
fn a_real_lost_race_is_still_a_412() {
// another writer moved the object: a precondition refusal that is not the placement's
Expand Down
Loading