From 449a7360f1db67d95b4b1a8f9a52ae39418659f6 Mon Sep 17 00:00:00 2001 From: matrixarkai Date: Tue, 17 Nov 2026 17:58:21 -0800 Subject: [PATCH] Declare and raise cluster features through the log Co-Authored-By: Claude Sonnet 5.5 --- src/cluster/features_commands.rs | 351 ++++++++++++++++++++++ src/cluster/meta_catalog_command_tests.rs | 1 + src/cluster/meta_client.rs | 19 ++ src/cluster/meta_consensus.rs | 13 + src/cluster/meta_message.rs | 14 + src/cluster/meta_server.rs | 24 ++ src/cluster/meta_wire.rs | 4 + src/cluster/mod.rs | 4 + src/cluster/placement_journal.rs | 1 + src/placement_outcome.rs | 118 ++++++++ 10 files changed, 549 insertions(+) create mode 100644 src/cluster/features_commands.rs diff --git a/src/cluster/features_commands.rs b/src/cluster/features_commands.rs new file mode 100644 index 00000000..82c2f2d7 --- /dev/null +++ b/src/cluster/features_commands.rs @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 MatrixArkAI +//! The commands that change and read the cluster feature set. +//! +//! * declare (action 1): a metadata voter records the feature bits it reads. Written by the voter +//! itself, so a late joiner and a replay see the same evidence as everyone else. +//! * raise (action 2): the group agrees to emit `bits`. Refused unless every voter the caller +//! names has declared support for every one of them, and unless this build reads them itself. +//! A bit is never lowered. +//! * read (action 3): the current set, from the committed state. +//! +//! The check is made here, on the leader, against the state the outcome is bound to: the outcome +//! carries the whole set it was decided from and applies only to exactly that set, so a leader +//! change between the decision and the commit cannot slip a bit past the check. An entry that is +//! already committed is applied unconditionally; a node that cannot read it fails, it does not skip. +use super::meta_message::MetaReply; +use crate::cluster_features::{ClusterFeatures, SUPPORTED_FEATURES}; +use crate::placement_outcome::{FeatureTable, PlacementOutcome}; +use crate::{MatrixMetaService, ObjectError}; +use prost::Message; + +pub const ACTION_DECLARE: u32 = 1; +pub const ACTION_RAISE: u32 = 2; +pub const ACTION_READ: u32 = 3; +const MAX_VOTERS: usize = crate::cluster_features::MAX_DECLARED_VOTERS; + +#[derive(Clone, PartialEq, Eq, Message)] +pub struct FeaturesRequest { + #[prost(uint32, tag = "1")] + pub version: u32, + #[prost(uint32, tag = "2")] + pub action: u32, + /// Declare: the voter making the declaration. Zero otherwise. + #[prost(uint64, tag = "3")] + pub voter: u64, + /// Declare: the bits the voter reads. Raise: the bits to raise. Zero for a read. + #[prost(uint64, tag = "4")] + pub bits: u64, + /// Raise: every voter of the group, as the caller read them from the membership. Each must + /// have declared the bits. + #[prost(uint64, repeated, tag = "5")] + pub voters: Vec, +} + +impl FeaturesRequest { + pub fn declare(voter: u64, bits: u64) -> Self { + Self { version: 1, action: ACTION_DECLARE, voter, bits, voters: Vec::new() } + } + pub fn raise(bits: u64, voters: Vec) -> Self { + Self { version: 1, action: ACTION_RAISE, voter: 0, bits, voters } + } + pub fn read() -> Self { + Self { version: 1, action: ACTION_READ, ..Default::default() } + } + pub fn is_read(&self) -> bool { + self.action == ACTION_READ + } + pub fn validate(&self) -> Result<(), ObjectError> { + let shaped = self.version == 1 + && match self.action { + ACTION_DECLARE => self.voter != 0 && self.voters.is_empty(), + ACTION_RAISE => { + self.voter == 0 + && self.bits != 0 + && !self.voters.is_empty() + && self.voters.len() <= MAX_VOTERS + && self.voters.windows(2).all(|pair| pair[0] < pair[1]) + && !self.voters.contains(&0) + } + ACTION_READ => self.voter == 0 && self.bits == 0 && self.voters.is_empty(), + _ => false, + }; + if !shaped { + return Err(ObjectError::InvalidArgument("invalid cluster feature request".into())); + } + Ok(()) + } + pub fn from_payload(bytes: &[u8]) -> Result { + if bytes.len() > 16 * 1024 { + return Err(ObjectError::Corruption("cluster feature request exceeds its bound".into())); + } + let request = Self::decode(bytes).map_err(|_| ObjectError::Corruption("invalid cluster feature request".into()))?; + if request.encode_to_vec() != bytes { + return Err(ObjectError::Corruption("noncanonical cluster feature request".into())); + } + request.validate().map_err(|_| ObjectError::Corruption("invalid cluster feature request".into()))?; + Ok(request) + } +} + +/// Decode a reply payload, refusing anything that does not re-encode to the same bytes. +pub fn reply_from_payload(bytes: &[u8]) -> Result { + if bytes.len() > 64 * 1024 { + return Err(ObjectError::Corruption("cluster feature reply exceeds its bound".into())); + } + let table = FeatureTable::decode(bytes).map_err(|_| ObjectError::Corruption("invalid cluster feature reply".into()))?; + if table.encode_to_vec() != bytes { + return Err(ObjectError::Corruption("noncanonical cluster feature reply".into())); + } + table.to_features().map_err(|_| ObjectError::Corruption("invalid cluster feature reply".into()))?; + Ok(table) +} + +/// The reply of every action: the set as it stands after the command. +pub fn reply(features: &ClusterFeatures) -> MetaReply { + MetaReply::Features(FeatureTable::from_features(features)) +} + +/// Decide a declare or a raise against `service`. `Ok(None)` means the command changes nothing (the +/// voter already declared those bits, or they are already raised): the caller answers with the +/// current set and proposes nothing. +pub(super) fn prepare( + service: &MatrixMetaService, + request: &FeaturesRequest, +) -> Result, ObjectError> { + request.validate()?; + let mut after = service.cluster_features.clone(); + match request.action { + ACTION_DECLARE => { + if after.declared.get(&request.voter) == Some(&request.bits) { + return Ok(None); + } + after.declared.insert(request.voter, request.bits); + } + ACTION_RAISE => { + if request.bits & !SUPPORTED_FEATURES != 0 { + return Err(ObjectError::PreconditionFailed("a requested feature is not one this build reads".into())); + } + let common = service.cluster_features.common_support(&request.voters); + if request.bits & !common != 0 { + return Err(ObjectError::PreconditionFailed( + "a voter has not declared support for every requested feature".into(), + )); + } + if after.raised & request.bits == request.bits { + return Ok(None); + } + after.raised |= request.bits; + } + _ => return Err(ObjectError::InvalidArgument("a read is not decided".into())), + } + PlacementOutcome::features_change(service, &after).map(Some) +} + +pub(super) fn read(service: &MatrixMetaService, request: &FeaturesRequest) -> Result { + request.validate()?; + Ok(reply(&service.cluster_features)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cluster::meta_wire::MetaRequestCode; + use crate::cluster_features::*; + + fn declare(service: &mut MatrixMetaService, voter: u64, bits: u64) -> bool { + match prepare(service, &FeaturesRequest::declare(voter, bits)).unwrap() { + Some(outcome) => { + // every committed entry goes through the payload, as it does in the log + let payload = outcome.to_payload().unwrap(); + PlacementOutcome::from_payload(&payload).unwrap().apply(service).unwrap(); + true + } + None => false, + } + } + fn raise(service: &mut MatrixMetaService, bits: u64, voters: &[u64]) -> Result { + Ok(match prepare(service, &FeaturesRequest::raise(bits, voters.to_vec()))? { + Some(outcome) => { + let payload = outcome.to_payload().unwrap(); + PlacementOutcome::from_payload(&payload).unwrap().apply(service).unwrap(); + true + } + None => false, + }) + } + const BOTH: u64 = FEATURE_COMPACT_APPEND | FEATURE_RESERVE_NEXT; + + #[test] + fn a_declaration_is_recorded_once_and_a_repeat_changes_nothing() { + let mut service = MatrixMetaService::new(); + let index = service.placement_commit_index; + assert!(declare(&mut service, 1, BOTH)); + assert_eq!(service.cluster_features.declared.get(&1), Some(&BOTH)); + assert_eq!(service.placement_commit_index, index + 1); + assert!(!declare(&mut service, 1, BOTH), "a restart repeating its declaration proposes nothing"); + assert_eq!(service.placement_commit_index, index + 1); + // a voter whose build changed declares again and replaces its entry + assert!(declare(&mut service, 1, FEATURE_COMPACT_APPEND)); + assert_eq!(service.cluster_features.declared.get(&1), Some(&FEATURE_COMPACT_APPEND)); + } + + #[test] + fn a_mixed_set_keeps_the_bit_low_until_every_voter_declares_it() { + let mut service = MatrixMetaService::new(); + declare(&mut service, 1, BOTH); + declare(&mut service, 2, BOTH); + // voter 3 runs an older build: it declared less, or nothing at all + for refused in [BOTH, FEATURE_RESERVE_NEXT] { + let index = service.placement_commit_index; + assert!(raise(&mut service, refused, &[1, 2, 3]).is_err(), "voter 3 has not declared anything"); + assert_eq!(service.cluster_features.raised, 0); + assert_eq!(service.placement_commit_index, index, "a refused raise writes nothing"); + } + declare(&mut service, 3, FEATURE_COMPACT_APPEND); + assert!(raise(&mut service, BOTH, &[1, 2, 3]).is_err(), "voter 3 reads only one of the two"); + assert!(raise(&mut service, FEATURE_COMPACT_APPEND, &[1, 2, 3]).unwrap()); + assert!(service.cluster_features.has(FEATURE_COMPACT_APPEND)); + assert!(!service.cluster_features.has(FEATURE_RESERVE_NEXT)); + // voter 3 upgrades and declares; now the second bit may be raised + declare(&mut service, 3, BOTH); + assert!(raise(&mut service, FEATURE_RESERVE_NEXT, &[1, 2, 3]).unwrap()); + assert!(service.cluster_features.has(BOTH)); + // raising what is raised is a no-op + assert!(!raise(&mut service, BOTH, &[1, 2, 3]).unwrap()); + } + + #[test] + fn a_voter_that_is_not_named_cannot_be_forgotten_and_an_unknown_bit_is_refused() { + let mut service = MatrixMetaService::new(); + declare(&mut service, 1, u64::MAX); + // the caller names voter 2, which never declared + assert!(raise(&mut service, FEATURE_COMPACT_APPEND, &[1, 2]).is_err()); + // a bit this build does not read is refused even when the voter declares it + assert!(raise(&mut service, 1 << 40, &[1]).is_err()); + assert_eq!(service.cluster_features.raised, 0); + } + + #[test] + fn an_outcome_applies_only_to_the_set_it_was_decided_from() { + let mut service = MatrixMetaService::new(); + declare(&mut service, 1, BOTH); + // a leader decides a raise; before it commits, another declaration commits + let stale = prepare(&service, &FeaturesRequest::raise(FEATURE_COMPACT_APPEND, vec![1])).unwrap().unwrap(); + declare(&mut service, 2, BOTH); + assert!(stale.apply(&mut service).is_err(), "decided against a set that no longer exists"); + assert_eq!(service.cluster_features.raised, 0); + // and the same outcome cannot be applied twice + let fresh = prepare(&service, &FeaturesRequest::raise(FEATURE_COMPACT_APPEND, vec![1, 2])).unwrap().unwrap(); + fresh.apply(&mut service).unwrap(); + assert!(fresh.apply(&mut service).is_err()); + } + + #[test] + fn the_outcome_shape_is_strict() { + let mut service = MatrixMetaService::new(); + declare(&mut service, 1, BOTH); + declare(&mut service, 2, BOTH); + raise(&mut service, FEATURE_COMPACT_APPEND, &[1, 2]).unwrap(); + let base = service.cluster_features.clone(); + let decide = |after: ClusterFeatures| PlacementOutcome::features_change(&service, &after); + let mut lowered = base.clone(); + lowered.raised = 0; + assert!(decide(lowered).is_err(), "a bit never goes down"); + let mut two = base.clone(); + two.declared.insert(1, 0); + two.declared.insert(2, 0); + assert!(decide(two).is_err(), "one voter at a time"); + let mut removed = base.clone(); + removed.declared.remove(&1); + assert!(decide(removed).is_err(), "a declaration is replaced, not removed"); + let mut both = base.clone(); + both.raised |= FEATURE_RESERVE_NEXT; + both.declared.insert(1, 0); + assert!(decide(both).is_err(), "a raise and a declaration are separate commands"); + assert!(decide(base.clone()).is_err(), "no change is not an outcome"); + let mut unknown = base.clone(); + unknown.raised |= 1 << 40; + assert!(decide(unknown).is_err()); + // the version and the field go together + let good = prepare(&service, &FeaturesRequest::declare(3, 1)).unwrap().unwrap(); + let mut wrong_version = good.clone(); + wrong_version.version = 12; + assert!(wrong_version.to_payload().is_err()); + let mut no_change = good.clone(); + no_change.features = None; + assert!(no_change.to_payload().is_err()); + let mut extra = good.clone(); + extra.after_index += 1; + assert!(extra.to_payload().is_err()); + } + + #[test] + fn a_replay_of_the_log_and_a_late_joiner_reach_the_same_set() { + let mut live = MatrixMetaService::new(); + let mut log: Vec> = Vec::new(); + for request in [ + FeaturesRequest::declare(1, BOTH), + FeaturesRequest::declare(2, BOTH), + FeaturesRequest::declare(3, FEATURE_COMPACT_APPEND), + FeaturesRequest::raise(FEATURE_COMPACT_APPEND, vec![1, 2, 3]), + ] { + let outcome = prepare(&live, &request).unwrap().unwrap(); + let payload = outcome.to_payload().unwrap(); + PlacementOutcome::from_payload(&payload).unwrap().apply(&mut live).unwrap(); + log.push(payload); + } + assert_eq!(log.len(), 4); + // restart: a fresh node replays the log from the start + let mut replayed = MatrixMetaService::new(); + for payload in &log { + PlacementOutcome::from_payload(payload).unwrap().apply(&mut replayed).unwrap(); + } + assert_eq!(replayed.cluster_features, live.cluster_features); + assert_eq!(replayed.placement_commit_index, live.placement_commit_index); + // late joiner: installs a snapshot taken after the second entry, then replays the rest + let mut early = MatrixMetaService::new(); + for payload in &log[..2] { + PlacementOutcome::from_payload(payload).unwrap().apply(&mut early).unwrap(); + } + let snapshot = crate::MetaSnapshot::from_bytes(&early.export_snapshot().to_bytes().unwrap()).unwrap(); + let mut joiner = MatrixMetaService::new(); + joiner.install_snapshot(snapshot.clone()).unwrap(); + joiner.install_placement_roster(&snapshot).unwrap(); + for payload in &log[2..] { + PlacementOutcome::from_payload(payload).unwrap().apply(&mut joiner).unwrap(); + } + assert_eq!(joiner.cluster_features, live.cluster_features); + assert!(joiner.cluster_features.has(FEATURE_COMPACT_APPEND)); + } + + #[test] + fn the_requests_have_one_canonical_shape() { + for good in [FeaturesRequest::declare(7, 3), FeaturesRequest::raise(1, vec![1, 2, 3]), FeaturesRequest::read()] { + let bytes = good.encode_to_vec(); + assert_eq!(FeaturesRequest::from_payload(&bytes).unwrap(), good); + assert!(FeaturesRequest::from_payload(&[bytes.clone(), vec![0]].concat()).is_err(), "trailing byte"); + } + let bad = [ + FeaturesRequest { version: 2, ..FeaturesRequest::read() }, + FeaturesRequest { action: 9, ..FeaturesRequest::read() }, + FeaturesRequest::declare(0, 1), + FeaturesRequest { voters: vec![1], ..FeaturesRequest::declare(1, 1) }, + FeaturesRequest::raise(0, vec![1]), + FeaturesRequest::raise(1, vec![]), + FeaturesRequest::raise(1, vec![2, 1]), + FeaturesRequest::raise(1, vec![1, 1]), + FeaturesRequest::raise(1, vec![0, 1]), + FeaturesRequest { bits: 1, ..FeaturesRequest::read() }, + ]; + for request in bad { + assert!(request.validate().is_err(), "{request:?}"); + assert!(FeaturesRequest::from_payload(&request.encode_to_vec()).is_err()); + } + assert_eq!(MetaRequestCode::from_u8(96), Some(MetaRequestCode::Features)); + assert!(MetaRequestCode::all().contains(&MetaRequestCode::Features)); + let reply_table = FeatureTable::from_features(&ClusterFeatures { raised: 1, declared: [(1, 3)].into() }); + assert!(reply_from_payload(&reply_table.encode_to_vec()).is_ok()); + assert!(reply_from_payload(&[reply_table.encode_to_vec(), vec![0]].concat()).is_err()); + } +} diff --git a/src/cluster/meta_catalog_command_tests.rs b/src/cluster/meta_catalog_command_tests.rs index 0a1bab6e..24835528 100755 --- a/src/cluster/meta_catalog_command_tests.rs +++ b/src/cluster/meta_catalog_command_tests.rs @@ -154,6 +154,7 @@ fn catalog_journal_recovers_mixed_records_and_every_torn_tail() { read_session_removal: None, manifest_append: None, reserved: None, + features: None, }; expiry.apply(&mut service).unwrap(); let (second, _) = prepare_at(&mut service, &groups, &report(2), 65536, 110).unwrap(); diff --git a/src/cluster/meta_client.rs b/src/cluster/meta_client.rs index d1a063df..3539c9ab 100644 --- a/src/cluster/meta_client.rs +++ b/src/cluster/meta_client.rs @@ -396,6 +396,25 @@ impl RemoteMetaClient { reply => Err(unexpected_reply("metadata membership", &reply)), } } + /// The cluster feature set as committed. + pub fn read_cluster_features(&mut self) -> Result { + self.feature_call(super::FeaturesRequest::read()) + } + /// Record the feature bits `voter` reads. A repeat of what is already recorded changes nothing. + pub fn declare_voter_features(&mut self, voter: u64, bits: u64) -> Result { + self.feature_call(super::FeaturesRequest::declare(voter, bits)) + } + /// Raise `bits`. Refused unless every voter in `voters` (the whole membership, ascending) has + /// declared every one of them. + pub fn raise_cluster_features(&mut self, bits: u64, voters: Vec) -> Result { + self.feature_call(super::FeaturesRequest::raise(bits, voters)) + } + fn feature_call(&mut self, request: super::FeaturesRequest) -> Result { + match self.call(MetaRequest::Features(request))? { + MetaReply::Features(table) => table.to_features(), + other => Err(unexpected_reply("a cluster feature set", &other)), + } + } /// Describe this voter without leader routing or a read barrier. pub fn metadata_group_identity(&mut self) -> Result { match self.call_one(&MetaRequest::GroupIdentity).into_result()? { diff --git a/src/cluster/meta_consensus.rs b/src/cluster/meta_consensus.rs index 60d6e66e..b268f1be 100644 --- a/src/cluster/meta_consensus.rs +++ b/src/cluster/meta_consensus.rs @@ -744,6 +744,7 @@ fn read_only(request: &MetaRequest) -> bool { MetaRequest::StagedAllocation(command) => command.is_read(), MetaRequest::ManifestRetention(command) => command.is_read(), MetaRequest::Membership(command) => command.is_read(), + MetaRequest::Features(command) => command.is_read(), MetaRequest::ReadManifest { .. } | MetaRequest::ReadManifestRoutes { .. } | MetaRequest::ReadManifestRoutesWide { .. } @@ -1147,6 +1148,18 @@ fn prepare( let (outcome, reply) = super::manifest_copy_commands::prepare(service, groups, command, limit)?; return Ok((outcome.to_payload()?, Some(reply))); } + if let MetaRequest::Features(command) = request { + if !command.is_read() { + return Ok(match super::features_commands::prepare(service, command)? { + Some(outcome) => { + let after = outcome.features.as_ref().and_then(|change| change.after.clone()) + .ok_or_else(|| ObjectError::Corruption("feature outcome lacks its result".into()))?; + (outcome.to_payload()?, Some(MetaReply::Features(after))) + } + None => (Vec::new(), Some(super::features_commands::reply(&service.cluster_features))), + }); + } + } if let MetaRequest::StagedAllocation(command) = request { if !command.is_read() { let (outcome, reply) = super::staged_allocation_commands::prepare(service, groups, command, limit)?; diff --git a/src/cluster/meta_message.rs b/src/cluster/meta_message.rs index dc05143b..73d4bd9f 100644 --- a/src/cluster/meta_message.rs +++ b/src/cluster/meta_message.rs @@ -208,6 +208,7 @@ pub fn object_id(bucket: &str, key: &str) -> ObjectId { #[derive(Clone, Debug, Eq, PartialEq)] pub enum MetaRequest { CopyManifest(crate::manifest_copy::ManifestCopyRequest), + Features(super::features_commands::FeaturesRequest), PlanAppendRoutes(super::append_plan::AppendPlanRequest), StagedAllocation(crate::staged_allocation::StagedAllocationRequest), ManifestRetention(crate::manifest_retention::ManifestRetentionRequest), @@ -331,6 +332,7 @@ impl MetaRequest { MetaRequest::ManifestRetention(_) => MetaRequestCode::ManifestRetention, MetaRequest::StagedAllocation(_) => MetaRequestCode::StagedAllocation, MetaRequest::CopyManifest(_) => MetaRequestCode::CopyManifest, + MetaRequest::Features(_) => MetaRequestCode::Features, MetaRequest::PlanAppendRoutes(_) => MetaRequestCode::PlanAppendRoutes, MetaRequest::GroupIdentity => MetaRequestCode::GroupIdentity, MetaRequest::HeartbeatRootNode { .. } => MetaRequestCode::HeartbeatRootNode, @@ -407,6 +409,7 @@ impl MetaRequest { MetaRequest::ManifestRetention(request) => return request.encode_to_vec(), MetaRequest::StagedAllocation(request) => return request.encode_to_vec(), MetaRequest::CopyManifest(request) => return request.encode_to_vec(), + MetaRequest::Features(request) => return request.encode_to_vec(), MetaRequest::PlanAppendRoutes(request) => return request.encode_to_vec(), MetaRequest::GroupIdentity => {} MetaRequest::CommitObjectPlacementIfCurrent { @@ -559,6 +562,9 @@ impl MetaRequest { if code == MetaRequestCode::CopyManifest { return crate::manifest_copy::ManifestCopyRequest::from_payload(payload).map(Self::CopyManifest); } + if code == MetaRequestCode::Features { + return super::features_commands::FeaturesRequest::from_payload(payload).map(Self::Features); + } if code == MetaRequestCode::ListChunkRoutes { let request = super::chunk_group_route::RoutePageRead::from_payload(payload)?; return Ok(Self::ListChunkRoutes { after: request.after, limit: request.limit }); @@ -692,6 +698,7 @@ impl MetaRequest { | MetaRequestCode::ManifestRetention | MetaRequestCode::StagedAllocation | MetaRequestCode::CopyManifest + | MetaRequestCode::Features | MetaRequestCode::PlanAppendRoutes | MetaRequestCode::ReportChunkNode | MetaRequestCode::AllocateObjectPages @@ -859,6 +866,7 @@ fn validate_manifest_route_payload( #[derive(Clone, Debug, Eq, PartialEq)] pub enum MetaReply { StagedAllocation(crate::staged_allocation::StagedAllocationReply), + Features(crate::placement_outcome::FeatureTable), AppendPlan(super::append_plan::AppendPlanReply), ManifestRetention(crate::manifest_retention::ManifestRetentionReply), Manifests(crate::object_manifest::ManifestListPage), @@ -923,6 +931,7 @@ impl MetaReply { Self::Manifests(page) => page.encoded_len(), Self::ManifestRetention(reply) => reply.encoded_len(), Self::StagedAllocation(reply) => reply.encoded_len(), + Self::Features(reply) => reply.encoded_len(), Self::AppendPlan(reply) => reply.encoded_len(), Self::PlacementNode(node) => node_payload_len( &node.node_id, @@ -987,6 +996,7 @@ impl MetaReply { MetaReply::Manifests(page) => return page.encode_to_vec(), MetaReply::ManifestRetention(reply) => return reply.encode_to_vec(), MetaReply::StagedAllocation(reply) => return reply.encode_to_vec(), + MetaReply::Features(reply) => return reply.encode_to_vec(), MetaReply::AppendPlan(reply) => return reply.encode_to_vec(), MetaReply::PlacementNode(node) => put_node(&mut out, node), MetaReply::RootNodes(nodes) => { @@ -1184,6 +1194,9 @@ impl MetaReply { if code == MetaRequestCode::StagedAllocation { return crate::staged_allocation::StagedAllocationReply::from_payload(payload).map(Self::StagedAllocation); } + if code == MetaRequestCode::Features { + return super::features_commands::reply_from_payload(payload).map(Self::Features); + } if code == MetaRequestCode::ListChunkRoutes { return super::chunk_group_route::RoutePage::from_payload(payload) .map(Self::ChunkRoutePage); @@ -1226,6 +1239,7 @@ impl MetaReply { | MetaRequestCode::ManifestRetention | MetaRequestCode::StagedAllocation | MetaRequestCode::CopyManifest + | MetaRequestCode::Features | MetaRequestCode::PlanAppendRoutes | MetaRequestCode::GroupIdentity => unreachable!(), MetaRequestCode::ChunkCatalog diff --git a/src/cluster/meta_server.rs b/src/cluster/meta_server.rs index 076db9ef..46e093b6 100644 --- a/src/cluster/meta_server.rs +++ b/src/cluster/meta_server.rs @@ -831,6 +831,14 @@ fn apply( let decision = match &request { MetaRequest::CopyManifest(command) => Some(super::manifest_copy_commands::prepare( &guard.service, &guard.groups, command, max_payload_bytes)?), + MetaRequest::Features(command) if !command.is_read() => match super::features_commands::prepare(&guard.service, command)? { + Some(outcome) => { + let after = outcome.features.as_ref().and_then(|change| change.after.clone()) + .ok_or_else(|| ObjectError::Corruption("feature outcome lacks its result".into()))?; + Some((outcome, MetaReply::Features(after))) + } + None => return Ok(super::features_commands::reply(&guard.service.cluster_features)), + }, MetaRequest::StagedAllocation(command) if !command.is_read() => { if guard.storage.is_none() { return Err(ObjectError::PreconditionFailed("staged allocation requires durable metadata storage".into())); @@ -1050,6 +1058,7 @@ pub(super) fn decide_allocation( read_session_removal: None, manifest_append: None, reserved: None, + features: None, }; outcome.to_payload()?; Ok((outcome, reply)) @@ -1143,6 +1152,19 @@ pub(super) fn apply_with_groups( outcome.apply(service)?; return Ok(reply); } + if let MetaRequest::Features(command) = &request { + if !command.is_read() { + return match super::features_commands::prepare(service, command)? { + Some(outcome) => { + let after = outcome.features.as_ref().and_then(|change| change.after.clone()) + .ok_or_else(|| ObjectError::Corruption("feature outcome lacks its result".into()))?; + outcome.apply(service)?; + Ok(MetaReply::Features(after)) + } + None => Ok(super::features_commands::reply(&service.cluster_features)), + }; + } + } if let MetaRequest::StagedAllocation(command) = &request { if !command.is_read() { let (outcome, reply) = super::staged_allocation_commands::prepare(service, groups, command, limit)?; @@ -1363,6 +1385,8 @@ pub(super) fn apply_to_meta( ) -> Result { match request { MetaRequest::CopyManifest(_) => Err(ObjectError::InvalidArgument("atomic copy requires catalog dispatch".into())), + MetaRequest::Features(command) if command.is_read() => super::features_commands::read(guard, &command), + MetaRequest::Features(_) => Err(ObjectError::InvalidArgument("feature changes require catalog dispatch".into())), MetaRequest::StagedAllocation(command) => super::staged_allocation_commands::read(guard, &command, max_payload_bytes), MetaRequest::ManifestRetention(command) => { command.validate()?; diff --git a/src/cluster/meta_wire.rs b/src/cluster/meta_wire.rs index d73bffc3..e9f04096 100644 --- a/src/cluster/meta_wire.rs +++ b/src/cluster/meta_wire.rs @@ -109,6 +109,8 @@ pub enum MetaRequestCode { StagedAllocation = 93, CopyManifest = 94, PlanAppendRoutes = 95, + /// Declare, raise or read the cluster feature set. + Features = 96, } impl MetaRequestCode { @@ -151,6 +153,7 @@ impl MetaRequestCode { 93 => Some(MetaRequestCode::StagedAllocation), 94 => Some(MetaRequestCode::CopyManifest), 95 => Some(MetaRequestCode::PlanAppendRoutes), + 96 => Some(MetaRequestCode::Features), _ => None, } } @@ -190,6 +193,7 @@ impl MetaRequestCode { MetaRequestCode::StagedAllocation, MetaRequestCode::CopyManifest, MetaRequestCode::PlanAppendRoutes, + MetaRequestCode::Features, ] } } diff --git a/src/cluster/mod.rs b/src/cluster/mod.rs index 6343ba04..a8816e2b 100644 --- a/src/cluster/mod.rs +++ b/src/cluster/mod.rs @@ -190,6 +190,10 @@ mod staged_allocation_commands; #[cfg(feature = "meta-server")] mod staged_lease_sweeper; #[cfg(feature = "meta-server")] +mod features_commands; +#[cfg(feature = "meta-server")] +pub use features_commands::FeaturesRequest; +#[cfg(feature = "meta-server")] pub use meta_identity::MetaGroupIdentity; pub use message::{ChunkGroupIdentity, ReplicaReply, ReplicaRequest}; diff --git a/src/cluster/placement_journal.rs b/src/cluster/placement_journal.rs index bbdf4dce..25b38c6d 100644 --- a/src/cluster/placement_journal.rs +++ b/src/cluster/placement_journal.rs @@ -152,6 +152,7 @@ mod tests { read_session_removal: None, manifest_append: None, reserved: None, + features: None, }; let second = PlacementOutcome { before_index: base + 1, diff --git a/src/placement_outcome.rs b/src/placement_outcome.rs index 11e55d25..02742ff7 100644 --- a/src/placement_outcome.rs +++ b/src/placement_outcome.rs @@ -105,6 +105,74 @@ pub struct PlacementChange { #[prost(message, optional, tag = "2")] pub after: Option, } +/// One voter's declared feature bits. +#[derive(Clone, PartialEq, Eq, Message)] +pub struct FeatureDeclaration { + #[prost(uint64, tag = "1")] + pub voter: u64, + #[prost(uint64, tag = "2")] + pub bits: u64, +} +/// The cluster feature set as it is carried by an outcome. +#[derive(Clone, PartialEq, Eq, Message)] +pub struct FeatureTable { + #[prost(uint64, tag = "1")] + pub raised: u64, + /// Ordered by voter id, one entry per voter. + #[prost(message, repeated, tag = "2")] + pub declared: Vec, +} +impl FeatureTable { + pub(crate) fn from_features(features: &crate::cluster_features::ClusterFeatures) -> Self { + Self { + raised: features.raised, + declared: features.declared.iter().map(|(voter, bits)| FeatureDeclaration { voter: *voter, bits: *bits }).collect(), + } + } + pub(crate) fn to_features(&self) -> Result { + let mut declared = std::collections::BTreeMap::new(); + for entry in &self.declared { + if declared.insert(entry.voter, entry.bits).is_some() { + return Err(invalid("feature table repeats a voter")); + } + } + let features = crate::cluster_features::ClusterFeatures { raised: self.raised, declared }; + features.validate()?; + Ok(features) + } +} +/// A change of the cluster feature set: either one voter declares its bits, or the raised bits +/// grow. Applied only against exactly the set it was decided from. +#[derive(Clone, PartialEq, Message)] +pub struct FeaturesChange { + #[prost(message, optional, tag = "1")] + pub before: Option, + #[prost(message, optional, tag = "2")] + pub after: Option, +} +impl FeaturesChange { + fn validate(&self) -> Result<(), ObjectError> { + let before = self.before.as_ref().ok_or_else(|| invalid("feature change lacks its prior set"))?.to_features()?; + let after = self.after.as_ref().ok_or_else(|| invalid("feature change lacks its resulting set"))?.to_features()?; + let raised_changed = after.raised != before.raised; + let declared_changed = after.declared != before.declared; + // a bit never goes down, and this build refuses a bit it cannot read itself + if after.raised & before.raised != before.raised + || after.raised & !crate::cluster_features::SUPPORTED_FEATURES != 0 + || raised_changed == declared_changed + { + return Err(invalid("invalid cluster feature change")); + } + if declared_changed { + // exactly one voter is added or changed, none is removed + let changed = after.declared.iter().filter(|(voter, bits)| before.declared.get(voter) != Some(bits)).count(); + if changed != 1 || before.declared.keys().any(|voter| !after.declared.contains_key(voter)) { + return Err(invalid("a feature change declares one voter at a time")); + } + } + Ok(()) + } +} #[derive(Clone, PartialEq, Message)] pub struct PlacementOutcome { #[prost(uint32, tag = "1")] @@ -138,6 +206,9 @@ pub struct PlacementOutcome { /// The next append's allocation, created by the commit that publishes this one. #[prost(message, optional, tag = "15")] pub reserved: Option, + /// Version 14 only: a change of the cluster feature set and nothing else. + #[prost(message, optional, tag = "16")] + pub features: Option, } fn invalid(s: &str) -> ObjectError { ObjectError::Corruption(s.to_string()) @@ -270,6 +341,20 @@ impl PlacementOutcome { /// `manifest_validated` says the publication's manifests were validated by the caller /// (the expansion of a compact append), so they are not walked again. fn validate_manifest_binding_with(&self, manifest_validated: bool) -> Result<(), ObjectError> { + if self.features.is_some() || self.version == 14 { + let change = self.features.as_ref().filter(|_| self.version == 14) + .ok_or_else(|| invalid("a feature outcome is version 14 and carries its change"))?; + change.validate()?; + if self.before_index.checked_add(1) != Some(self.after_index) + || self.manifest.is_some() || self.retention.is_some() || self.staging.is_some() + || self.retention_deadline.is_some() || self.copy_source.is_some() + || self.publication_fence.is_some() || self.manifest_append.is_some() || self.reserved.is_some() + || self.read_session_open.is_some() || self.read_session_removal.is_some() + || !self.nodes.is_empty() || !self.placements.is_empty() { + return Err(invalid("a feature outcome changes unrelated state")); + } + return Ok(()); + } if self.reserved.is_some() && !(self.version == 13 || (self.version == 4 && self.manifest.is_some())) { return Err(invalid("a reservation belongs to a publication")); } @@ -513,6 +598,7 @@ impl PlacementOutcome { read_session_removal: None, manifest_append: Some(append), reserved: None, + features: None, }; outcome.validate_manifest_binding()?; outcome.validate_append_publication(service, true, before_manifest, after_manifest)?; @@ -604,6 +690,7 @@ impl PlacementOutcome { read_session_removal: None, manifest_append: None, reserved: None, + features: None, }; outcome.validate_manifest_binding()?; outcome.validate_publication(service, true)?; @@ -785,6 +872,7 @@ impl PlacementOutcome { read_session_removal: None, manifest_append: None, reserved: None, + features: None, }) } pub fn finish(mut self, service: &MatrixMetaService) -> Result { @@ -820,6 +908,25 @@ impl PlacementOutcome { self.to_payload()?; Ok(self) } + /// The outcome that records one voter's declaration or raises bits: `after` is the whole + /// set the service will hold. The caller has decided the change is allowed. + pub(crate) fn features_change( + service: &MatrixMetaService, + after: &crate::cluster_features::ClusterFeatures, + ) -> Result { + let outcome = Self { + version: 14, + before_index: service.placement_commit_index, + after_index: service.placement_commit_index + 1, + features: Some(FeaturesChange { + before: Some(FeatureTable::from_features(&service.cluster_features)), + after: Some(FeatureTable::from_features(after)), + }), + ..Default::default() + }; + outcome.to_payload()?; + Ok(outcome) + } pub fn to_payload(&self) -> Result, ObjectError> { self.validate_manifest_binding()?; if self.encoded_len() > LIMIT - 12 { @@ -853,6 +960,17 @@ impl PlacementOutcome { /// append against the stored manifest, which validated them. fn apply_with(&self, service: &mut MatrixMetaService, manifest_validated: bool) -> Result<(), ObjectError> { self.validate_manifest_binding_with(manifest_validated)?; + if let Some(change) = &self.features { + // checked against exactly the state it was decided from; never skipped + let before = change.before.as_ref().unwrap().to_features()?; + if self.before_index != service.placement_commit_index || before != service.cluster_features { + return Err(invalid("cluster feature outcome does not match the current set")); + } + service.cluster_features = change.after.as_ref().unwrap().to_features()?; + service.placement_commit_index = self.after_index; + service.invalidate_snapshot_export_cache(); + return Ok(()); + } if let Some(append) = &self.manifest_append { let (before, stored) = service.object_manifests.get_with_digest(&ObjectId::new(append.bucket.clone(), append.key.clone()))? .ok_or_else(|| invalid("compact append prior manifest missing"))?;