From 2e22abda3fc725655aaf7bb02f01b03ac1a804d5 Mon Sep 17 00:00:00 2001 From: baseRT sync Date: Fri, 30 Oct 2026 02:56:50 +1100 Subject: [PATCH] sync: open-core source for 0.4.3 Generated by tools/sync_public.py from the internal tree at 569482347a. Only allow-listed paths are included; tuned quant profiles and internal tooling are excluded by rule. --- base-convert/Cargo.lock | 17 +- base-convert/Cargo.toml | 2 +- base-convert/crates/base-arch/src/dflash.rs | 121 + base-convert/crates/base-convert/Cargo.toml | 2 + base-convert/crates/base-convert/src/embed.rs | 732 +++++ base-convert/crates/base-convert/src/hub.rs | 564 +++- base-convert/crates/base-convert/src/main.rs | 180 +- .../crates/base-convert/src/selfupdate.rs | 2685 +++++++++++++++++ base-convert/crates/base-convert/src/serve.rs | 178 +- .../crates/base-convert/src/superfluid.rs | 879 +++++- .../crates/base-convert/tests/self_update.rs | 1529 ++++++++++ base-convert/crates/base-hub/catalog.json | 42 +- base-convert/crates/base-hub/src/catalog.rs | 28 +- base-convert/crates/base-hub/src/gen.rs | 11 +- base-convert/crates/base-hub/src/registry.rs | 9 + base-convert/crates/base-hub/src/scan.rs | 6 + bindings/node/package.json | 2 +- bindings/python/baseRT/__init__.py | 2 +- bindings/python/setup.py | 2 +- bindings/rust/baseRT-sys/Cargo.toml | 2 +- bindings/rust/baseRT/Cargo.toml | 2 +- .../swift/Sources/CBaseRT/include/baseRT.h | 2 +- include/baseRT/baseRT.h | 2 +- include/baseRT/baseRT_tick.h | 8 +- packaging/npm/build-packages.mjs | 6 +- packaging/python/build-wheel.sh | 2 +- packaging/stage-engine.sh | 13 +- packaging/update-formula.sh | 2 + tests/scripts/test_packaging.py | 26 +- 29 files changed, 6874 insertions(+), 182 deletions(-) create mode 100644 base-convert/crates/base-convert/src/embed.rs create mode 100644 base-convert/crates/base-convert/src/selfupdate.rs create mode 100644 base-convert/crates/base-convert/tests/self_update.rs diff --git a/base-convert/Cargo.lock b/base-convert/Cargo.lock index bc9533e..6a9fd58 100644 --- a/base-convert/Cargo.lock +++ b/base-convert/Cargo.lock @@ -151,7 +151,7 @@ dependencies = [ [[package]] name = "base-arch" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "base-format", @@ -162,7 +162,7 @@ dependencies = [ [[package]] name = "base-awq" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "base-format", @@ -176,7 +176,7 @@ dependencies = [ [[package]] name = "base-convert" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "base-arch", @@ -191,6 +191,7 @@ dependencies = [ "ed25519-dalek", "half", "indicatif", + "libc", "memmap2", "rand_core 0.6.4", "serde_json", @@ -201,7 +202,7 @@ dependencies = [ [[package]] name = "base-format" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "bitflags", @@ -217,7 +218,7 @@ dependencies = [ [[package]] name = "base-hub" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "base-format", @@ -236,7 +237,7 @@ dependencies = [ [[package]] name = "base-quant" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "base-format", @@ -248,7 +249,7 @@ dependencies = [ [[package]] name = "base-readers" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "base-format", @@ -262,7 +263,7 @@ dependencies = [ [[package]] name = "base-sign" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "base-format", diff --git a/base-convert/Cargo.toml b/base-convert/Cargo.toml index c71f5c7..31a9f33 100644 --- a/base-convert/Cargo.toml +++ b/base-convert/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.4.2" +version = "0.4.3" edition = "2021" license = "Apache-2.0" repository = "https://github.com/basecompute/baseRT" diff --git a/base-convert/crates/base-arch/src/dflash.rs b/base-convert/crates/base-arch/src/dflash.rs index f3f0517..7b7bedc 100644 --- a/base-convert/crates/base-arch/src/dflash.rs +++ b/base-convert/crates/base-arch/src/dflash.rs @@ -50,12 +50,27 @@ fn fc_transform(canonical: &str) -> Option { pub struct DflashHfMapper; pub struct DsparkHfMapper; +/// Google's Gemma 4 MTP drafter (`Gemma4AssistantForCausalLM`, the +/// `*-assistant` repos): a few dense Gemma 4 layers with QUERY projections +/// only — they attend over the target's KV cache (`num_kv_shared_layers` = +/// every layer, `attention_k_eq_v`) — fed by `pre_projection` from +/// [target token embedding ; target post-norm hidden] (2 x backbone hidden) +/// and projected back to the backbone width by `post_projection` as the next +/// draft step's hidden. Its own `embed_tokens` is the DRAFT logit head +/// (hidden-width rows); the input embedding is the target's. +pub struct Gemma4MtpHfMapper; /// Canonical arch strings of the block-drafter family. pub fn is_block_drafter_arch(arch: &str) -> bool { arch == "dflash" || arch == "dspark" } +/// The Gemma 4 MTP head sidecar (not a block drafter: one draft per step, +/// recurrent through `post_projection`, no block / mask token). +pub fn is_mtp_sidecar_arch(arch: &str) -> bool { + arch == "gemma4-mtp" +} + /// Pick a drafter mapper from `config.json`'s `architectures`; None for an /// ordinary LM checkpoint (which routes by `model_type` as before). pub fn hf_mapper_for_architectures(config: &serde_json::Value) -> Option<&'static dyn HfMapper> { @@ -67,6 +82,9 @@ pub fn hf_mapper_for_architectures(config: &serde_json::Value) -> Option<&'stati if a.contains("DSpark") { return Some(&DsparkHfMapper); } + if a.starts_with("Gemma4Assistant") { + return Some(&Gemma4MtpHfMapper); + } } None } @@ -506,6 +524,57 @@ impl HfMapper for DflashHfMapper { } } +impl HfMapper for Gemma4MtpHfMapper { + fn canonical_arch(&self) -> &'static str { + "gemma4-mtp" + } + fn config_from_hf(&self, c: &serde_json::Value) -> Result { + // The head's layer config is a Gemma 4 text config (nested under + // text_config): per-layer SWA pattern, head dims, global KV heads, + // both rope thetas and the partial rotary factor come out of the + // Gemma 4 mapper unchanged. + let mut cfg = crate::gemma::Gemma4HfMapper.config_from_hf(c)?; + let text = c.get("text_config").unwrap_or(c); + let n_layers = cfg.num_hidden_layers; + let shared = text + .get("num_kv_shared_layers") + .and_then(|v| v.as_u64()) + .unwrap_or(0) as u32; + if shared != n_layers { + bail!( + "Gemma 4 assistant config.json: num_kv_shared_layers {shared} != num_hidden_layers {n_layers} — \ + the runtime runs this head with every layer reading the target's KV" + ); + } + // Whether the target's global layers keep one K=V bank (the 26B-A4B) + // or separate K and V (the E4B): recorded, not required — the head + // projects queries only and reads whichever banks the target wrote. + let k_eq_v = text + .get("attention_k_eq_v") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + let backbone = c + .get("backbone_hidden_size") + .and_then(|v| v.as_u64()) + .ok_or_else(|| { + anyhow!("Gemma 4 assistant config.json: backbone_hidden_size missing") + })?; + // No layer owns KV: the Gemma 4 mapper's rule (`shared < n_layers`) + // would have left every layer with its own. + cfg.n_layer_kv_from_start = 0; + // Its embed_tokens is the draft logit head, never tied to an input + // embedding of its own (the input embedding is the target's). + cfg.tie_word_embeddings = false; + let x = &mut cfg.extra_config; + x.insert("speculator_kind".into(), serde_json::json!("mtp-shared-kv")); + x.insert("num_layers".into(), serde_json::json!(n_layers)); + x.insert("backbone_hidden_size".into(), serde_json::json!(backbone)); + x.insert("num_kv_shared_layers".into(), serde_json::json!(shared)); + x.insert("attention_k_eq_v".into(), serde_json::json!(k_eq_v)); + Ok(cfg) + } +} + impl HfMapper for DsparkHfMapper { fn canonical_arch(&self) -> &'static str { "dspark" @@ -1181,3 +1250,55 @@ mod eagle3_tests { ); } } + +#[cfg(test)] +mod gemma4_mtp_tests { + use super::*; + use serde_json::json; + + /// google/gemma-4-E4B-it-assistant's config.json (HF bookkeeping fields + /// dropped). + fn e4b_assistant() -> serde_json::Value { + serde_json::from_str(r#"{"architectures": ["Gemma4AssistantForCausalLM"], "model_type": "gemma4_assistant", "backbone_hidden_size": 2560, "tie_word_embeddings": true, "num_centroids": 2048, "use_ordered_embeddings": true, "text_config": {"architectures": null, "attention_bias": false, "attention_k_eq_v": false, "bos_token_id": 2, "enable_moe_block": false, "eos_token_id": 1, "final_logit_softcapping": null, "global_head_dim": 512, "head_dim": 256, "hidden_activation": "gelu_pytorch_tanh", "hidden_size": 256, "hidden_size_per_layer_input": 0, "intermediate_size": 2048, "layer_types": ["sliding_attention", "sliding_attention", "sliding_attention", "full_attention"], "max_position_embeddings": 131072, "model_type": "gemma4_text", "moe_intermediate_size": null, "num_attention_heads": 4, "num_experts": null, "num_global_key_value_heads": null, "num_hidden_layers": 4, "num_key_value_heads": 2, "num_kv_shared_layers": 4, "pad_token_id": 0, "rms_norm_eps": 1e-06, "rope_parameters": {"full_attention": {"partial_rotary_factor": 0.25, "rope_theta": 1000000.0, "rope_type": "proportional"}, "sliding_attention": {"rope_theta": 10000.0, "rope_type": "default"}}, "sliding_window": 512, "tie_word_embeddings": true, "top_k_experts": null, "use_bidirectional_attention": null, "use_cache": true, "use_double_wide_mlp": false, "vocab_size": 262144, "vocab_size_per_layer_input": 0}}"#) + .unwrap() + } + + #[test] + fn the_assistant_architecture_picks_the_mtp_mapper() { + let c = e4b_assistant(); + let m = hf_mapper_for_architectures(&c).map(|m| m.canonical_arch()); + assert_eq!(m, Some("gemma4-mtp")); + assert!(is_mtp_sidecar_arch("gemma4-mtp")); + assert!(!is_block_drafter_arch("gemma4-mtp")); + } + + #[test] + fn the_head_owns_no_kv_and_records_its_backbone() { + let cfg = Gemma4MtpHfMapper.config_from_hf(&e4b_assistant()).unwrap(); + assert_eq!(cfg.num_hidden_layers, 4); + // Every layer reads the target's KV: none may own one (the Gemma 4 + // mapper's rule would have given all four their own). + assert_eq!(cfg.n_layer_kv_from_start, 0); + // Its embed_tokens is the draft logit head, not an input embedding. + assert!(!cfg.tie_word_embeddings); + assert_eq!(cfg.extra_config["backbone_hidden_size"], json!(2560)); + assert_eq!(cfg.extra_config["speculator_kind"], json!("mtp-shared-kv")); + assert_eq!(cfg.extra_config["attention_k_eq_v"], json!(false)); + } + + #[test] + fn a_head_with_its_own_kv_layers_or_no_backbone_width_is_refused() { + let mut c = e4b_assistant(); + c["text_config"]["num_kv_shared_layers"] = json!(2); + assert!( + Gemma4MtpHfMapper.config_from_hf(&c).is_err(), + "two layers would own KV" + ); + let mut c = e4b_assistant(); + c.as_object_mut().unwrap().remove("backbone_hidden_size"); + assert!( + Gemma4MtpHfMapper.config_from_hf(&c).is_err(), + "no backbone_hidden_size" + ); + } +} diff --git a/base-convert/crates/base-convert/Cargo.toml b/base-convert/crates/base-convert/Cargo.toml index 2a6a0f9..83a5472 100644 --- a/base-convert/crates/base-convert/Cargo.toml +++ b/base-convert/crates/base-convert/Cargo.toml @@ -33,6 +33,8 @@ tempfile.workspace = true dirs.workspace = true # `basert superfluid install` fetches superfluid's release tarball from GitHub. ureq = "3" +# `basert self`: whether a process is alive, and (macOS) what each runs. +libc = "0.2" [dev-dependencies] tempfile.workspace = true diff --git a/base-convert/crates/base-convert/src/embed.rs b/base-convert/crates/base-convert/src/embed.rs new file mode 100644 index 0000000..70ff056 --- /dev/null +++ b/base-convert/crates/base-convert/src/embed.rs @@ -0,0 +1,732 @@ +//! `basert add-speculator`: embed a converted drafter sidecar in a target +//! bundle as its built-in speculator head (`header.speculator`), the way +//! `convert` embeds a Qwen3.5 checkpoint's own MTP head. Google ships Gemma +//! 4's head as a separate checkpoint (`google/gemma-4--it-assistant`); +//! embedding it gives a Gemma 4 bundle the same `--speculate mtp-head` / +//! `auto` story as a Qwen one, with nothing to pair at serve time. +//! +//! Byte-level: the target's weights blob is copied verbatim, so every main +//! and mmproj tensor keeps its bytes, offsets and checksums; the head's +//! tensors are appended behind it, placed by the target's alignment rules +//! exactly as the writer would place them; the header gains the section and +//! `HAS_SPECULATOR`. Nothing is re-quantized, and a reconverted target is +//! not needed: the catalog's files are embedded as they are. +//! +//! The section is the sidecar's own header, not a Qwen-style one: `arch` is +//! the sidecar's (`gemma4-mtp`), `config` is the sidecar's whole config (the +//! head's own geometry — a 4-layer, narrow Gemma 4 decoder over the target's +//! KV, plus `backbone_hidden_size`) with `kind` added, and the tensors keep +//! the sidecar's names (`output.weight`, `layers.0.*`, `fc_embed.weight`, +//! …, no `mtp.` prefix). The runtime opens the section as if it were the +//! sidecar file. `kind` is the sidecar arch rather than `mtp` on purpose: a +//! runtime or superfluid from before this head existed reads `kind`, sees it +//! is not the one-layer MTP head it knows, and leaves the section alone +//! (`auto` decodes plainly, bare `mtp-head` refuses cleanly) instead of +//! loading the target a second time as a head it cannot run. +use anyhow::{bail, Context, Result}; +use base_format::{AlignmentConfig, ComputeRegion, HeaderFlags, BLOB_ALIGNMENT, PREFIX_LEN}; +use clap::Parser; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; + +#[derive(Parser, Debug)] +pub struct AddSpeculatorArgs { + /// Target `.base` bundle: the model the head drafts for. + pub target: PathBuf, + /// The head, converted on its own (`basert convert + /// google/gemma-4-E4B-it-assistant` writes a `gemma4-mtp` sidecar). + #[arg(long)] + pub head: PathBuf, + /// Where to write the embedded bundle (refused if it exists). + #[arg(short, long)] + pub output: PathBuf, + /// A version for the head, recorded as `head_version` in the section + /// (a catalog row carries it as `speculator_version`): a later head for + /// the same model gets a new one, and `basert serve --speculate` pulls a + /// file whose head is older than the catalog's. Any string; absent means + /// unversioned. + #[arg(long, value_name = "V")] + pub head_version: Option, +} + +pub fn cmd_add_speculator(args: AddSpeculatorArgs) -> Result<()> { + let s = add_speculator( + &args.target, + &args.head, + &args.output, + args.head_version.as_deref(), + )?; + eprintln!( + "{}: {} head ({} tensors, {} bytes) embedded as `{}`; {} bytes written", + args.output.display(), + s.head_arch, + s.n_head_tensors, + s.head_bytes, + s.kind, + s.out_bytes + ); + Ok(()) +} + +#[derive(Debug)] +pub struct Summary { + pub head_arch: String, + pub kind: String, + pub n_head_tensors: usize, + pub head_bytes: u64, + pub out_bytes: u64, +} + +/// A bundle's header as the open JSON it is on disk, plus where its blob +/// starts and how long the file is. +struct Opened { + path: PathBuf, + header: serde_json::Value, + blob_start: u64, + file_size: u64, +} + +fn open_header(path: &Path) -> Result { + let mut f = std::fs::File::open(path).with_context(|| format!("opening {}", path.display()))?; + let file_size = f.metadata()?.len(); + let mut prefix = [0u8; PREFIX_LEN as usize]; + f.read_exact(&mut prefix) + .with_context(|| format!("{}: not a .base file (truncated prefix)", path.display()))?; + if prefix[0..4] != base_format::MAGIC { + bail!("{}: not a .base file (bad magic)", path.display()); + } + let version = u32::from_le_bytes(prefix[4..8].try_into().unwrap()); + if version != base_format::FORMAT_VERSION { + bail!( + "{}: format version {version}, expected {}", + path.display(), + base_format::FORMAT_VERSION + ); + } + let header_len = u64::from_le_bytes(prefix[8..16].try_into().unwrap()); + if header_len == 0 || PREFIX_LEN + header_len > file_size { + bail!("{}: implausible header length {header_len}", path.display()); + } + let mut json = vec![0u8; header_len as usize]; + f.read_exact(&mut json)?; + let header: serde_json::Value = serde_json::from_slice(&json) + .with_context(|| format!("{}: header is not JSON", path.display()))?; + if !header.is_object() { + bail!("{}: header is not a JSON object", path.display()); + } + Ok(Opened { + path: path.to_path_buf(), + header, + blob_start: align_up(PREFIX_LEN + header_len, BLOB_ALIGNMENT), + file_size, + }) +} + +fn align_up(x: u64, a: u64) -> u64 { + x.div_ceil(a) * a +} + +fn tensors_of(section: &serde_json::Value) -> Vec<&serde_json::Value> { + section + .get("tensors") + .and_then(|t| t.as_array()) + .map(|a| a.iter().collect()) + .unwrap_or_default() +} + +fn u64_of(v: &serde_json::Value, key: &str) -> Option { + v.get(key).and_then(|x| x.as_u64()) +} + +/// The last blob byte any tensor of the header (main, mmproj, speculator) +/// or any of its sub-regions reaches. +fn blob_end(header: &serde_json::Value) -> u64 { + let mut end = 0u64; + let mut visit = |t: &serde_json::Value| { + for (off, len) in [ + ("offset", "length"), + ("scale_offset", "scale_length"), + ("bias_offset", "bias_length"), + ("awq_scale_offset", "awq_scale_length"), + ] { + if let (Some(o), Some(l)) = (u64_of(t, off), u64_of(t, len)) { + end = end.max(o + l); + } + } + }; + for t in tensors_of(header) { + visit(t); + } + for sec in ["mmproj", "speculator"] { + if let Some(s) = header.get(sec) { + for t in tensors_of(s) { + visit(t); + } + } + } + end +} + +fn parse_flags(header: &serde_json::Value) -> Result { + match header.get("flags") { + None => Ok(HeaderFlags::empty()), + Some(v) => serde_json::from_value(v.clone()).context("header flags"), + } +} + +fn has_sub_regions(t: &serde_json::Value) -> bool { + ["scale_offset", "bias_offset", "awq_scale_offset"] + .iter() + .any(|k| t.get(k).is_some_and(|v| !v.is_null())) +} + +/// Embed the head sidecar at `head` in the bundle at `target`, writing the +/// result to `out`; `head_version`, when given, is recorded in the section. +/// See the module doc for the layout. +pub fn add_speculator( + target: &Path, + head: &Path, + out: &Path, + head_version: Option<&str>, +) -> Result { + if out.exists() { + bail!("{}: exists; refusing to overwrite it", out.display()); + } + let t = open_header(target)?; + let h = open_header(head)?; + + // ── The target: a plain bundle with room behind its last tensor ── + if let Some(s) = t.header.get("speculator") { + bail!( + "{}: already carries a speculator head (arch {}); refusing to replace it", + target.display(), + s.get("arch").and_then(|a| a.as_str()).unwrap_or("?") + ); + } + let t_flags = parse_flags(&t.header)?; + let slot_flags = HeaderFlags::HAS_LORA + | HeaderFlags::HAS_COMPUTE_GRAPH + | HeaderFlags::HAS_KV_WARMUP + | HeaderFlags::HAS_TRACE_REF; + if t_flags.intersects(slot_flags) { + bail!( + "{}: carries extension slots ({:?}); embedding a head behind them is not supported", + target.display(), + t_flags + ); + } + if t_flags.contains(HeaderFlags::SIGNED) || t.header.get("sig").is_some() { + bail!("{}: is signed; the embedded bundle would need signing again (strip the signature first)", target.display()); + } + let t_blob_len = t.file_size - t.blob_start; + let t_end = blob_end(&t.header); + if t_end != t_blob_len { + bail!( + "{}: {} bytes follow the last tensor (extension slots?); refusing to append behind them", + target.display(), + t_blob_len as i128 - t_end as i128 + ); + } + + // ── The head: an MTP sidecar of this target's backbone, unquantized ── + let head_arch = h + .header + .get("arch") + .and_then(|a| a.as_str()) + .unwrap_or("") + .to_string(); + if !base_arch::dflash::is_mtp_sidecar_arch(&head_arch) { + bail!( + "{}: header arch `{head_arch}` is not an MTP head sidecar (expected gemma4-mtp)", + head.display() + ); + } + let backbone = head_arch.strip_suffix("-mtp").unwrap_or(&head_arch); + let t_arch = t.header.get("arch").and_then(|a| a.as_str()).unwrap_or(""); + if t_arch != backbone { + bail!( + "{}: arch `{t_arch}`, but the head drafts for `{backbone}`", + target.display() + ); + } + if h.header.get("mmproj").is_some() || h.header.get("speculator").is_some() { + bail!( + "{}: a head sidecar carries no sub-bundles of its own", + head.display() + ); + } + let h_cfg = h + .header + .get("config") + .filter(|c| c.is_object()) + .context("head config")?; + let t_cfg = t + .header + .get("config") + .filter(|c| c.is_object()) + .context("target config")?; + for (want, key_h, key_t) in [ + ("hidden width", "backbone_hidden_size", "hidden_size"), + ("vocabulary", "vocab_size", "vocab_size"), + ] { + let (a, b) = (u64_of(h_cfg, key_h), u64_of(t_cfg, key_t)); + if a.is_none() || a != b { + bail!( + "the head's {want} ({key_h} {}) does not match the target's ({key_t} {})", + a.map_or("missing".to_string(), |v| v.to_string()), + b.map_or("missing".to_string(), |v| v.to_string()) + ); + } + } + let h_tensors = tensors_of(&h.header); + if h_tensors.is_empty() { + bail!("{}: no tensors", head.display()); + } + if let Some(q) = h_tensors.iter().find(|t| has_sub_regions(t)) { + bail!( + "{}: tensor {} is quantized (has sub-regions); a head is embedded unquantized — convert it f16", + head.display(), + q.get("name").and_then(|n| n.as_str()).unwrap_or("?") + ); + } + let h_blob_len = h.file_size - h.blob_start; + if blob_end(&h.header) > h_blob_len { + bail!( + "{}: a tensor reaches past the end of the file", + head.display() + ); + } + + // ── Place the head's tensors behind the target's blob ── + let alignment: AlignmentConfig = match t.header.get("alignment") { + Some(a) => serde_json::from_value(a.clone()).context("target alignment")?, + None => AlignmentConfig::default(), + }; + let mut cursor = t_blob_len; + let mut placed: Vec<(serde_json::Value, u64, u64, u64)> = Vec::with_capacity(h_tensors.len()); // (entry, src offset, dst offset, length) + for src in &h_tensors { + let region: ComputeRegion = match src.get("compute_region") { + Some(r) => serde_json::from_value(r.clone()).context("head tensor compute_region")?, + None => ComputeRegion::default(), + }; + let (off, len) = ( + u64_of(src, "offset").context("head tensor offset")?, + u64_of(src, "length").context("head tensor length")?, + ); + let dst = align_up(cursor, alignment.align_for(region)); + let mut entry = (*src).clone(); + entry["offset"] = serde_json::json!(dst); + placed.push((entry, off, dst, len)); + cursor = dst + len; + } + let head_bytes: u64 = placed.iter().map(|p| p.3).sum(); + + // ── The new header: the target's, plus the section and its flag ── + let kind = head_arch.clone(); + let mut config: std::collections::BTreeMap = h_cfg + .as_object() + .unwrap() + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + config.insert("kind".to_string(), serde_json::json!(kind)); + // Where the head came from and, when the caller says, which one it is: + // the sidecar's source checkpoint name, and a version a catalog row can + // repeat (`speculator_version`) so a client knows its file's head is old. + if let Some(src) = h + .header + .get("source") + .and_then(|s| s.get("filename")) + .and_then(|f| f.as_str()) + { + if !src.is_empty() { + config.insert("head_source".to_string(), serde_json::json!(src)); + } + } + if let Some(v) = head_version { + config.insert("head_version".to_string(), serde_json::json!(v)); + } + let section = serde_json::json!({ + "arch": head_arch, + "config": config, + "tensors": placed.iter().map(|p| p.0.clone()).collect::>(), + }); + let flags = t_flags | HeaderFlags::HAS_SPECULATOR; + // The writer emits the struct's field order (`mmproj`, `speculator`, + // `calibration`, …); keep the section where a converted bundle has it. + let mut header = serde_json::Map::new(); + let mut inserted = false; + for (k, v) in t.header.as_object().unwrap() { + if !inserted && matches!(k.as_str(), "calibration" | "sig" | "provenance") { + header.insert("speculator".to_string(), section.clone()); + inserted = true; + } + if k == "flags" { + header.insert(k.clone(), serde_json::to_value(flags)?); + } else { + header.insert(k.clone(), v.clone()); + } + } + if !inserted { + header.insert("speculator".to_string(), section); + } + if !header.contains_key("flags") { + header.insert("flags".to_string(), serde_json::to_value(flags)?); + } + let header_json = serde_json::to_vec(&serde_json::Value::Object(header))?; + let header_len = header_json.len() as u64; + let blob_start = align_up(PREFIX_LEN + header_len, BLOB_ALIGNMENT); + + // ── Write: prefix, header, pad, the target's blob verbatim, the head ── + let mut w = std::io::BufWriter::with_capacity( + 8 << 20, + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(out) + .with_context(|| format!("creating {}", out.display()))?, + ); + w.write_all(&base_format::MAGIC)?; + w.write_all(&base_format::FORMAT_VERSION.to_le_bytes())?; + w.write_all(&header_len.to_le_bytes())?; + w.write_all(&header_json)?; + write_zeros(&mut w, blob_start - PREFIX_LEN - header_len)?; + { + let mut src = std::fs::File::open(&t.path)?; + src.seek(SeekFrom::Start(t.blob_start))?; + let copied = std::io::copy(&mut src.take(t_blob_len), &mut w)?; + if copied != t_blob_len { + bail!( + "{}: short read ({copied} of {t_blob_len} blob bytes)", + target.display() + ); + } + } + let mut written = t_blob_len; + { + let mut src = std::fs::File::open(&h.path)?; + for (_, off, dst, len) in &placed { + write_zeros(&mut w, dst - written)?; + src.seek(SeekFrom::Start(h.blob_start + off))?; + let copied = std::io::copy(&mut (&src).take(*len), &mut w)?; + if copied != *len { + bail!( + "{}: short read ({copied} of {len} bytes at {off})", + head.display() + ); + } + written = dst + len; + } + } + w.flush()?; + let out_bytes = blob_start + written; + Ok(Summary { + head_arch: head_arch.clone(), + kind, + n_head_tensors: placed.len(), + head_bytes, + out_bytes, + }) +} + +fn write_zeros(w: &mut W, n: u64) -> Result<()> { + static Z: [u8; 65536] = [0u8; 65536]; + let mut left = n; + while left > 0 { + let k = left.min(Z.len() as u64) as usize; + w.write_all(&Z[..k])?; + left -= k as u64; + } + Ok(()) +} + +/// Tiny bundles for tests: a header of one arch and config, an f16 entry, +/// a filled payload. Shared with hub.rs's tests. +#[cfg(test)] +pub(crate) mod fixtures { + use base_format::{ + AlignmentConfig, ComputeRegion, Header, HeaderFlags, ModelConfig, QuantScheme, SourceInfo, + TargetBackend, TensorDtype, TensorEntry, TensorFlags, TensorPayload, TokenizerBlob, + }; + use std::collections::BTreeMap; + + pub(crate) fn header(arch: &str, cfg: serde_json::Value) -> Header { + Header { + schema: 1, + arch: arch.to_string(), + quant_scheme: QuantScheme::F16, + min_hw: "apple_m1".to_string(), + created: "2026-10-29T00:00:00Z".to_string(), + base_rt_version: "0.1.0-test".to_string(), + source: SourceInfo { + format: "test".to_string(), + sha256: "0".repeat(64), + filename: "synthetic".to_string(), + }, + tokenizer: TokenizerBlob { + fields: BTreeMap::new(), + }, + config: ModelConfig { + fields: cfg + .as_object() + .unwrap() + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(), + }, + metadata: BTreeMap::new(), + target_backend: TargetBackend::Metal, + quant_profile: String::new(), + alignment: AlignmentConfig::default(), + flags: HeaderFlags::empty(), + layers: vec![], + tensors: vec![], + mmproj: None, + speculator: None, + calibration: None, + provenance: None, + sig: None, + } + } + + pub(crate) fn entry(name: &str, shape: Vec) -> TensorEntry { + TensorEntry { + name: name.to_string(), + dtype: TensorDtype::F16, + shape, + offset: 0, + length: 0, + scale_offset: None, + scale_length: None, + bias_offset: None, + bias_length: None, + awq_scale_offset: None, + awq_scale_length: None, + group_size: None, + layout: None, + residency: None, + compute_region: ComputeRegion::Gpu, + scale_dtype: None, + symmetric: false, + flags: TensorFlags::empty(), + checksum_xxh64: None, + source_ggml_type: None, + } + } + + pub(crate) fn payload(name: &str, n: usize, fill: u8) -> TensorPayload { + TensorPayload { + entry: entry(name, vec![n as u64 / 2]), + data: vec![fill; n], + } + } +} + +#[cfg(test)] +mod tests { + use super::fixtures::*; + use super::*; + use base_format::{BaseReader, BaseWriter, TensorDtype, TensorEntry}; + + /// A tiny Gemma-4-shaped target (two main tensors, one mmproj tensor) + /// and a head sidecar over it. + fn fixtures(dir: &Path) -> (PathBuf, PathBuf) { + let target = dir.join("target.base"); + let mut w = BaseWriter::create( + &target, + header( + "gemma4", + serde_json::json!({"hidden_size": 8, "vocab_size": 16, "num_hidden_layers": 2}), + ), + ) + .unwrap(); + w.add_tensor(payload("token_embedding.weight", 256, 0xA1)); + w.add_tensor(payload("layers.0.attention.q.weight", 100, 0xA2)); + w.set_mmproj_arch("gemma4-vision"); + w.add_mmproj_tensor(payload("vision.patch.weight", 50, 0xB1)); + w.finish().unwrap(); + + let head = dir.join("head.base"); + let mut w = BaseWriter::create( + &head, + header( + "gemma4-mtp", + serde_json::json!({"hidden_size": 4, "backbone_hidden_size": 8, "vocab_size": 16, "num_hidden_layers": 4, "speculator_kind": "mtp-shared-kv"}), + ), + ) + .unwrap(); + w.add_tensor(payload("output.weight", 128, 0xC1)); + w.add_tensor(payload("layers.0.input_norm.weight", 8, 0xC2)); + w.add_tensor(payload("fc_embed.weight", 64, 0xC3)); + w.finish().unwrap(); + (target, head) + } + + fn bytes_of(path: &Path, blob_start: u64, e: &TensorEntry) -> Vec { + let mut f = std::fs::File::open(path).unwrap(); + f.seek(SeekFrom::Start(blob_start + e.offset)).unwrap(); + let mut v = vec![0u8; e.length as usize]; + f.read_exact(&mut v).unwrap(); + v + } + + #[test] + fn embeds_the_head_behind_an_untouched_target_blob() { + let dir = tempfile::tempdir().unwrap(); + let (target, head) = fixtures(dir.path()); + let out = dir.path().join("out.base"); + let s = add_speculator(&target, &head, &out, Some("1")).unwrap(); + assert_eq!( + ( + s.head_arch.as_str(), + s.kind.as_str(), + s.n_head_tensors, + s.head_bytes + ), + ("gemma4-mtp", "gemma4-mtp", 3, 200) + ); + assert_eq!(s.out_bytes, std::fs::metadata(&out).unwrap().len()); + + let before = BaseReader::open(&target).unwrap(); + let after = BaseReader::open(&out).unwrap(); + let (hb, ha) = (before.header(), after.header()); + // The target's own sections are the same entries over the same bytes. + let same = |a: &TensorEntry, b: &TensorEntry| { + serde_json::to_string(a).unwrap() == serde_json::to_string(b).unwrap() + }; + assert_eq!(hb.tensors.len(), ha.tensors.len()); + for (a, b) in hb.tensors.iter().zip(&ha.tensors) { + assert!(same(a, b), "{}", a.name); + assert_eq!( + before.tensor_bytes(&a.name).unwrap(), + after.tensor_bytes(&b.name).unwrap() + ); + } + let (ma, mb) = (hb.mmproj.as_ref().unwrap(), ha.mmproj.as_ref().unwrap()); + assert_eq!( + (ma.arch.as_str(), ma.tensors.len()), + (mb.arch.as_str(), mb.tensors.len()) + ); + assert!(same(&ma.tensors[0], &mb.tensors[0])); + assert_eq!(hb.config.fields, ha.config.fields); + assert_eq!(hb.arch, ha.arch); + assert!( + ha.flags.contains(HeaderFlags::HAS_SPECULATOR) + && !hb.flags.contains(HeaderFlags::HAS_SPECULATOR) + ); + + // The section is the sidecar's header, `kind` added, names kept, + // bytes and checksums intact, placed by the target's alignment. + let spec = ha.speculator.as_ref().unwrap(); + assert_eq!(spec.arch, "gemma4-mtp"); + assert_eq!(spec.config["kind"], serde_json::json!("gemma4-mtp")); + assert_eq!(spec.config["backbone_hidden_size"], serde_json::json!(8)); + assert_eq!( + spec.config["speculator_kind"], + serde_json::json!("mtp-shared-kv") + ); + let hh = BaseReader::open(&head).unwrap(); + let t_blob_end = hb + .mmproj + .as_ref() + .unwrap() + .tensors + .iter() + .chain(&hb.tensors) + .map(|e| e.offset + e.length) + .max() + .unwrap(); + let mut cursor = t_blob_end; + for (src, dst) in hh.header().tensors.iter().zip(&spec.tensors) { + assert_eq!(src.name, dst.name); + assert_eq!(src.checksum_xxh64, dst.checksum_xxh64); + assert_eq!(src.length, dst.length); + assert_eq!( + dst.offset, + align_up(cursor, ha.alignment.align_for(ComputeRegion::Gpu)), + "{}", + dst.name + ); + assert_eq!( + bytes_of(&head, hh.blob_offset(), src), + bytes_of(&out, after.blob_offset(), dst), + "{}", + dst.name + ); + after.verify_tensor(&dst.name).unwrap(); + cursor = dst.offset + dst.length; + } + assert_eq!( + after.blob_offset() + cursor, + s.out_bytes, + "nothing follows the head" + ); + } + + #[test] + fn refuses_what_it_cannot_embed_into_or_from() { + let dir = tempfile::tempdir().unwrap(); + let (target, head) = fixtures(dir.path()); + let out = dir.path().join("out.base"); + + // Output in the way. + std::fs::write(&out, b"x").unwrap(); + assert!(add_speculator(&target, &head, &out, None) + .unwrap_err() + .to_string() + .contains("refusing to overwrite")); + std::fs::remove_file(&out).unwrap(); + + // A head whose width is not the target's, and a target that is not + // the head's backbone. + let other = dir.path().join("other.base"); + let mut w = BaseWriter::create( + &other, + header( + "gemma4-mtp", + serde_json::json!({"hidden_size": 4, "backbone_hidden_size": 12, "vocab_size": 16}), + ), + ) + .unwrap(); + w.add_tensor(payload("output.weight", 64, 1)); + w.finish().unwrap(); + assert!(add_speculator(&target, &other, &out, None) + .unwrap_err() + .to_string() + .contains("hidden width")); + assert!(add_speculator(&head, &head, &out, None) + .unwrap_err() + .to_string() + .contains("but the head drafts for")); + + // A quantized head: the sidecar is embedded as it is, and a head + // with scale sub-regions is not the f16 one the rule keeps. + let q = dir.path().join("q.base"); + let mut w = BaseWriter::create( + &q, + header( + "gemma4-mtp", + serde_json::json!({"hidden_size": 4, "backbone_hidden_size": 8, "vocab_size": 16}), + ), + ) + .unwrap(); + let mut p = payload("output.weight", 64, 1); + p.entry.dtype = TensorDtype::BaseQ4; + p.entry.scale_offset = Some(0); + p.entry.scale_length = Some(0); + w.add_tensor(p); + w.finish().unwrap(); + assert!(add_speculator(&target, &q, &out, None) + .unwrap_err() + .to_string() + .contains("quantized")); + + // Embedding twice. + add_speculator(&target, &head, &out, None).unwrap(); + let twice = dir.path().join("twice.base"); + assert!(add_speculator(&out, &head, &twice, None) + .unwrap_err() + .to_string() + .contains("already carries")); + } +} diff --git a/base-convert/crates/base-convert/src/hub.rs b/base-convert/crates/base-convert/src/hub.rs index c4f18ee..8cbc573 100644 --- a/base-convert/crates/base-convert/src/hub.rs +++ b/base-convert/crates/base-convert/src/hub.rs @@ -449,7 +449,101 @@ fn auto_pull_and_resolve( /// with several installed without naming one). An uninstalled model is fetched /// on demand — preferring the pre-converted basecompute mirror, else converting /// the source repo — so `basert chat`/`serve ` Just Works. -fn resolve_hub_model(token: &str, default_variant: Option<&str>) -> Result { +/// What `--speculate` asks of the model's own MTP head in a runtime +/// command's args: bare `mtp-head`, or `auto`, which takes that head before +/// any drafter. None when speculation is off or not asked for, a drafter is +/// named, or the strategy needs no head (`prompt-lookup`). +pub(crate) fn mtp_speculation_asked(rest: &[String]) -> Option<&str> { + let mut it = rest.iter(); + while let Some(a) = it.next() { + let v = if a == "--speculate" { + it.next().map(String::as_str) + } else { + a.strip_prefix("--speculate=") + }; + if let Some(v) = v { + return matches!(v, "mtp-head" | "auto").then_some(v); + } + } + None +} + +/// An installed catalog file that cannot serve the speculation asked for, +/// though the catalog's current file could: it carries no MTP head, or one +/// of another version than the row's `speculator_version`, while the row for +/// the same id and variant says the published file has one (`speculator: +/// mtp`, or `gemma4-mtp` for Gemma 4's embedded head) and is another file +/// (its sha256 differs). Only a file the catalog installed qualifies — +/// pulling it again replaces like with like; a conversion or a file from +/// elsewhere is left alone. Returns the variant to pull again. +fn stale_head_variant(reg: &MergedRegistry, id: &str, path: &Path) -> Option { + let sidecar = cache::read_sidecar(path.parent()?).ok()??; + if sidecar.source_kind != "catalog" { + return None; + } + let row = reg.catalog.rows(id).find(|e| e.quant == sidecar.variant)?; + if !matches!(row.speculator.as_deref(), Some("mtp" | "gemma4-mtp")) { + return None; + } + let same_file = matches!((&row.sha256, &sidecar.base_sha256), (Some(want), Some(got)) if want.eq_ignore_ascii_case(got)); + if same_file { + return None; + } + let header = base_format::BaseReader::read_header(path).ok()?; + let stale = match (&header.speculator, &row.speculator_version) { + // No head at all. + (None, _) => true, + // A head of another version than the one the catalog publishes. + (Some(have), Some(want)) => { + have.config.get("head_version").and_then(|v| v.as_str()) != Some(want.as_str()) + } + // A head, and the catalog names no version to hold it to. + (Some(_), None) => false, + }; + stale.then_some(sidecar.variant) +} + +/// The installed artifact to serve, pulled again first when `--speculate +/// ` asks for its MTP head and the catalog's current file has one +/// this file lacks (stale_head_variant). Without a speculation request the +/// file is served as it is, with no word said. +fn serve_ready( + reg: &MergedRegistry, + id: &str, + path: PathBuf, + mtp: Option<&str>, +) -> Result { + let Some(directive) = mtp else { + return Ok(path); + }; + let Some(variant) = stale_head_variant(reg, id, &path) else { + return Ok(path); + }; + eprintln!( + "{id} [{variant}]: the installed file carries no MTP head and the catalog's current one does — \ + pulling it again for --speculate {directive}" + ); + cmd_pull(PullArgs { + id: format!("{id}:{variant}"), + profile: None, + target: TargetScheme::BaseQ4, + revision: "main".to_string(), + file: None, + force: true, + dry_run: false, + }) + .with_context(|| format!("pulling {id}:{variant} again"))?; + let reg2 = MergedRegistry::load()?; + installed_best_variant(®2, id, &variant).with_context(|| { + format!("pulled {id}:{variant} again but could not locate the installed artifact") + }) +} + +fn resolve_hub_model( + token: &str, + default_variant: Option<&str>, + mtp: Option<&str>, +) -> Result { let (id, inline) = token .split_once(':') .map_or((token, None), |(i, v)| (i, Some(v))); @@ -470,13 +564,13 @@ fn resolve_hub_model(token: &str, default_variant: Option<&str>) -> Result) -> Result) -> Result> { + // `--speculate mtp-head` / `auto` asks for the model's own head: a stale + // catalog file without one is pulled again on the way (serve_ready). + let mtp = mtp_speculation_asked(rest); rest.iter() .map(|arg| { if looks_like_hub_id(arg) { - Ok(resolve_hub_model(arg, default_variant)? + Ok(resolve_hub_model(arg, default_variant, mtp)? .to_string_lossy() .into_owned()) } else if let Some((kind, id)) = speculate_drafter_id(arg) { @@ -498,7 +595,7 @@ fn resolve_model_args(rest: &[String], default_variant: Option<&str>) -> Result< // it drafts for: catalogue lookup, pull, convert-on-pull, // variant selection, hub cache. A drafter was the one model a // user had to fetch and convert by hand and then name by path. - let path = resolve_hub_model(id, default_variant)?; + let path = resolve_hub_model(id, default_variant, None)?; Ok(format!("{kind}:{}", path.to_string_lossy())) } else { Ok(arg.clone()) @@ -615,11 +712,10 @@ pub fn dispatch_external(argv: Vec) -> Result<()> { (vec![format!("basert-{cmd}"), format!("baseRT_{cmd}")], rest) }; - // Prefer a binary sitting next to `basert` (how the release ships); fall - // back to a bare name, which `Command` resolves against `PATH`. - let exe_dir = std::env::current_exe() - .ok() - .and_then(|p| p.parent().map(Path::to_path_buf)); + // Prefer a binary sitting next to `basert` (how the release ships; for the + // versions layout, in the version this launcher runs from); fall back to a + // bare name, which `Command` resolves against `PATH`. + let exe_dir = crate::selfupdate::bundle_dir().ok(); let harness_environment_override = std::env::var_os(COMPUTEARENA_HARNESS_ENV).is_some() || std::env::var_os(LEGACY_COMPUTEARENA_HARNESS_ENV).is_some(); let bundled_harness = if is_computearena { @@ -674,11 +770,15 @@ const RUNTIME_COMMANDS: [&str; 5] = ["chat", "complete", "bench", "transcribe", pub fn cmd_pull(args: PullArgs) -> Result<()> { let reg = MergedRegistry::load()?; - if let Some((id, variant)) = args.id.split_once(':') { - let fetcher = HfFetcher::new(cache::hf_staging_dir(®.root))?; - return pull_variant(®, &args, id, variant, &fetcher); - } - pull_id(®, &args) + // Resolved against whichever catalog the attempt is given: the cached one + // first, the live one if the cached rows prove to be behind the Hub. + with_fresh_catalog_on_mismatch(®, MergedRegistry::load_fresh, |reg| { + if let Some((id, variant)) = args.id.split_once(':') { + let fetcher = HfFetcher::new(cache::hf_staging_dir(®.root))?; + return pull_variant(reg, &args, id, variant, &fetcher); + } + pull_id(reg, &args) + }) } /// Where `basert pull :` gets the variant from (see @@ -1230,6 +1330,78 @@ fn print_plan(r: &ModelRef, want: &str) { } /// Fast path: download a pre-converted `.base` from the catalog. +/// A pulled file whose bytes are not the catalog row's (`expected`): typed, +/// so a caller can tell a catalog that is behind the Hub from a bad download +/// (with_fresh_catalog_on_mismatch). +#[derive(Debug)] +struct ShaMismatch { + id: String, + file: String, + expected: String, + got: String, +} + +impl std::fmt::Display for ShaMismatch { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "sha256 mismatch for {} ({}): expected {}, got {}", + self.id, self.file, self.expected, self.got + ) + } +} + +impl std::error::Error for ShaMismatch {} + +/// Runs `attempt` against `reg`; when it fails because the file's bytes are +/// not the row's sha256, fetches the live catalog past the cache (`refetch`) +/// and runs it once more if the live row names another sha256. The cached +/// catalog is served under its TTL without a look at the hosted one, and a +/// bundle's row and file change together on the Hub, so between that change +/// and the cache's expiry a fresh pull downloads the new bytes and checks +/// them against the old row. The live catalog settles it: a Hub file that +/// disagrees with the live row too stays an error, as does a cache that +/// cannot be refreshed. +fn with_fresh_catalog_on_mismatch( + reg: &MergedRegistry, + refetch: impl FnOnce() -> Option, + attempt: impl Fn(&MergedRegistry) -> Result<()>, +) -> Result<()> { + let err = match attempt(reg) { + Ok(()) => return Ok(()), + Err(e) => e, + }; + let Some(m) = err.downcast_ref::() else { + return Err(err); + }; + let Some(fresh) = refetch() else { + return Err(err); + }; + let changed = fresh.catalog.rows(&m.id).any(|e| { + e.file == m.file + && e.sha256 + .as_deref() + .is_some_and(|s| !s.eq_ignore_ascii_case(&m.expected)) + }); + if !changed { + return Err(err); + } + eprintln!( + " catalog: the cached catalog was behind the Hub ({}'s sha256 changed); pulling again from the live one", + m.file + ); + attempt(&fresh) +} + +/// Where a pull puts the downloaded `.base` until it has passed the checks +/// that make it the installed one: beside it (`model.base.pending`), on the +/// same filesystem, so the final move is a rename. +fn pending_path(out: &Path) -> PathBuf { + let mut p = out.as_os_str().to_owned(); + p.push(".pending"); + PathBuf::from(p) +} + fn pull_catalog(root: &Path, r: &ModelRef, fetcher: &dyn Fetcher) -> Result<()> { let ModelRef::Catalog { id, @@ -1254,28 +1426,40 @@ fn pull_catalog(root: &Path, r: &ModelRef, fetcher: &dyn Fetcher) -> Result<()> std::fs::create_dir_all(&vdir)?; let out = cache::base_artifact_path(&vdir); announce_parts(&artifact, &out); - // Moves the staged download into place (same filesystem), so the pulled - // artifact exists exactly once on disk. + // The download lands beside the installed file (same filesystem, moved + // from the staging area, so it exists once on disk) and replaces it only + // once its sha256 is the row's: a forced pull that fails the check — a + // `basert serve` refreshing a file for its head, `pull --force` — leaves + // the file and its sidecar as they were rather than an empty slot. + let pending = pending_path(&out); parts::install( fetcher, hf_repo, revision, &artifact, - &out, + &pending, parts_sha256.as_deref(), )?; - let got_sha = crate::compute_sha256_streaming(&out)?; + let got_sha = crate::compute_sha256_streaming(&pending)?; if let Some(expected) = sha256 { if !got_sha.eq_ignore_ascii_case(expected) { - std::fs::remove_file(&out).ok(); + std::fs::remove_file(&pending).ok(); // The staged bytes are equally bad — no resume value in them, so // clear them and let a retry start from scratch. fetch::cleanup_staging(fetcher, hf_repo); - bail!("sha256 mismatch for {id}: expected {expected}, got {got_sha}"); + return Err(ShaMismatch { + id: id.clone(), + file: file.clone(), + expected: expected.clone(), + got: got_sha, + } + .into()); } eprintln!(" sha256: verified"); } + std::fs::rename(&pending, &out) + .with_context(|| format!("moving {} into place", pending.display()))?; write_sidecar_for( &vdir, @@ -1336,22 +1520,32 @@ fn pull_base_direct( let out = cache::base_artifact_path(&vdir); announce_parts(artifact, &out); - // Moves the staged download into place (same filesystem), so the pulled - // artifact exists exactly once on disk. A part set is reassembled beside - // `out` and renamed into place once complete. + // As pull_catalog: the download lands beside the installed file (moved + // from the staging area; a part set is reassembled there first) and + // replaces it only once it passes the catalog's sha256, so a failed + // forced pull leaves the installed file and its sidecar as they were. let parts_sha256 = listed.and_then(|e| e.parts_sha256.as_deref()); - parts::install(fetcher, repo, revision, artifact, &out, parts_sha256)?; + let pending = pending_path(&out); + parts::install(fetcher, repo, revision, artifact, &pending, parts_sha256)?; - let sha = crate::compute_sha256_streaming(&out).ok(); + let sha = crate::compute_sha256_streaming(&pending).ok(); if let Some(expected) = listed.and_then(|e| e.sha256.as_deref()) { let got = sha.as_deref().unwrap_or("nothing readable"); if !got.eq_ignore_ascii_case(expected) { - std::fs::remove_file(&out).ok(); + std::fs::remove_file(&pending).ok(); fetch::cleanup_staging(fetcher, repo); - bail!("sha256 mismatch for {id} ({file}): expected {expected}, got {got}"); + return Err(ShaMismatch { + id: id.to_string(), + file: file.clone(), + expected: expected.to_string(), + got: got.to_string(), + } + .into()); } eprintln!(" sha256: verified"); } + std::fs::rename(&pending, &out) + .with_context(|| format!("moving {} into place", pending.display()))?; write_sidecar_for( &vdir, id, @@ -1434,6 +1628,8 @@ fn pull_and_convert( let conv = ConvertArgs { input: snapshot, output: Some(out.clone()), + mtp_head: None, + mtp_head_version: None, target: args.target, calib: None, calib_tokens: 512, @@ -2793,6 +2989,149 @@ mod tests { ); } + /// A cached catalog behind the Hub — the row's sha256 is the old file's, + /// the Hub serves the new one — fails a fresh pull's checksum; the pull + /// then fetches the live catalog and, when the live row names the new + /// sha256, pulls once more against it. A live row that agrees with the + /// stale one (the Hub really disagrees), or no live catalog at all, leaves + /// the error, after one attempt. + #[test] + fn a_stale_catalog_row_is_retried_from_the_live_catalog() { + use sha2::{Digest, Sha256}; + let tmp = tempfile::tempdir().unwrap(); + fixture_repo( + tmp.path(), + "basecompute/cat-files", + &[("cat-Medium.base", b"medium")], + ); + let fetcher = MockFetcher::new(tmp.path()); + let real = format!("{:x}", Sha256::digest(b"medium")); + let row = |sha: &str| { + catalog(&format!( + r#"{{"id":"basecompute/cat","hf_repo":"basecompute/cat-files","quant":"default-medium","file":"cat-Medium.base","sha256":"{sha}"}}"# + )) + }; + let root = tmp.path().join("hub"); + let stale = MergedRegistry::new(root.clone(), row(&"0".repeat(64))); + let out = root.join("basecompute/cat/default-medium/model.base"); + let attempts = std::cell::Cell::new(0); + let attempt = |reg: &MergedRegistry| { + attempts.set(attempts.get() + 1); + let r = reg + .resolve("basecompute/cat", "main", Some("default-medium"), false) + .unwrap(); + pull_catalog(®.root, &r, &fetcher) + }; + + // The live catalog knows the new file: the retry installs it. + with_fresh_catalog_on_mismatch( + &stale, + || Some(MergedRegistry::new(root.clone(), row(&real))), + attempt, + ) + .unwrap(); + assert_eq!(attempts.get(), 2); + assert_eq!(std::fs::read(&out).unwrap(), b"medium"); + let s = cache::read_sidecar(out.parent().unwrap()).unwrap().unwrap(); + assert_eq!(s.base_sha256.as_deref(), Some(real.as_str())); + std::fs::remove_dir_all(&root).unwrap(); + + // The live row agrees with the stale one: the Hub is wrong, one attempt. + attempts.set(0); + let err = with_fresh_catalog_on_mismatch( + &stale, + || Some(MergedRegistry::new(root.clone(), row(&"0".repeat(64)))), + attempt, + ) + .unwrap_err(); + assert!(err.to_string().contains("sha256 mismatch"), "{err}"); + assert_eq!(attempts.get(), 1); + assert!(!out.exists()); + + // No live catalog to be had: the error as it was. + attempts.set(0); + let err = with_fresh_catalog_on_mismatch(&stale, || None, attempt).unwrap_err(); + assert!(err.to_string().contains("sha256 mismatch"), "{err}"); + assert_eq!(attempts.get(), 1); + } + + /// A pull that fails the catalog's sha256 leaves an installed file and + /// its sidecar as they were, on both pull paths: the download is checked + /// beside them and moved into place only when it is the row's file. A + /// forced pull is what `basert serve` runs to refresh a file for its MTP + /// head, so a catalog and a Hub file that disagree must not cost the + /// model that was being served. + #[test] + fn a_failed_checksum_leaves_the_installed_file_alone() { + let tmp = tempfile::tempdir().unwrap(); + fixture_repo( + tmp.path(), + "basecompute/cat-files", + &[("cat-Medium.base", b"medium")], + ); + let fetcher = MockFetcher::new(tmp.path()); + let reg = MergedRegistry::new( + tmp.path().join("hub"), + catalog(&format!( + r#"{{"id":"basecompute/cat","hf_repo":"basecompute/cat-files","quant":"default-medium","file":"cat-Medium.base","sha256":"{}"}}"#, + "0".repeat(64) + )), + ); + // What an earlier pull installed, with the sidecar it recorded. + let vdir = cache::variant_dir(®.root, "basecompute/cat", "default-medium").unwrap(); + std::fs::create_dir_all(&vdir).unwrap(); + let out = cache::base_artifact_path(&vdir); + std::fs::write(&out, b"the installed one").unwrap(); + write_sidecar_for( + &vdir, + "basecompute/cat", + "catalog", + "basecompute/cat-files", + None, + "main", + "default-medium", + None, + Some("installed-sha".into()), + Some("cat-Medium.base"), + ) + .unwrap(); + let untouched = |what: &str| { + assert_eq!( + std::fs::read(&out).unwrap(), + b"the installed one", + "{what}: the file" + ); + let s = cache::read_sidecar(&vdir).unwrap().unwrap(); + assert_eq!( + s.base_sha256.as_deref(), + Some("installed-sha"), + "{what}: the sidecar" + ); + assert!( + !pending_path(&out).exists(), + "{what}: no download left beside it" + ); + }; + + // The catalog path, forced (the refresh `basert serve` runs). + let r = reg + .resolve("basecompute/cat", "main", Some("default-medium"), true) + .unwrap(); + assert!(matches!(r, ModelRef::Catalog { .. })); + let err = pull_catalog(®.root, &r, &fetcher).unwrap_err(); + assert!(err.to_string().contains("sha256 mismatch"), "{err}"); + untouched("catalog path"); + + // The direct path, by file name, forced. + let args = PullArgs { + force: true, + ..named("cat-Medium.base", false, "basecompute/cat") + }; + let err = pull_named(®, &fetcher, &args).unwrap_err(); + assert!(err.to_string().contains("sha256 mismatch"), "{err}"); + untouched("direct path"); + } + /// The catalog's own file is held to the catalog's sha256 when named. #[test] fn a_named_catalog_file_is_checked_against_the_catalog() { @@ -3500,7 +3839,7 @@ mod tests { fn resolve_hub_model_rejects_empty_variant() { // A trailing `:` (empty variant) is a typo — error before any pull, // rather than silently resolving to the q4 default. - let err = resolve_hub_model("basecompute/Gemma-4-E2B-it:", None) + let err = resolve_hub_model("basecompute/Gemma-4-E2B-it:", None, None) .unwrap_err() .to_string(); assert!(err.contains("empty variant"), "{err}"); @@ -3563,4 +3902,171 @@ mod tests { assert_eq!(variant, "bf16"); assert!(path.is_none() && guard.is_none()); } + + #[test] + fn mtp_speculation_is_asked_by_mtp_head_or_auto() { + let asked = |v: &[&str]| { + mtp_speculation_asked(&v.iter().map(|s| s.to_string()).collect::>()) + .map(str::to_string) + }; + assert_eq!( + asked(&["m.base", "--speculate", "mtp-head"]).as_deref(), + Some("mtp-head") + ); + assert_eq!( + asked(&["--speculate=auto", "m.base"]).as_deref(), + Some("auto") + ); + assert_eq!(asked(&["m.base", "--speculate", "dflash:org/d"]), None); + assert_eq!(asked(&["m.base", "--speculate", "off"]), None); + assert_eq!(asked(&["m.base", "--speculate", "prompt-lookup"]), None); + assert_eq!(asked(&["m.base", "--speculate"]), None); + assert_eq!(asked(&["m.base"]), None); + } + + /// A catalog file without the head its row now promises is pulled again + /// for speculation; one with a head, one the catalog promises no head + /// for, the catalog's own file, or an install from elsewhere are not. + #[test] + fn a_catalog_file_without_the_head_its_row_promises_is_stale() { + use base_format::{SpeculatorBundle, FORMAT_VERSION, MAGIC}; + use std::io::Write; + let tmp = tempfile::tempdir().unwrap(); + let root = tmp.path().to_path_buf(); + let id = "basecompute/gemma-4-E4B-it"; + let (old, new) = ("b".repeat(64), "a".repeat(64)); + // A header-only bundle (read_header never reaches the blob) and the + // sidecar a pull records beside it. + // `head`: None for no head, Some(version) for one (None = unversioned). + let install = |variant: &str, head: Option>, source_kind: &str| -> PathBuf { + let vdir = cache::variant_dir(&root, id, variant).unwrap(); + std::fs::create_dir_all(&vdir).unwrap(); + let path = cache::base_artifact_path(&vdir); + let mut h = + crate::embed::fixtures::header("gemma4", serde_json::json!({"hidden_size": 8})); + if let Some(version) = head { + let mut config: std::collections::BTreeMap = + [("kind".to_string(), serde_json::json!("gemma4-mtp"))] + .into_iter() + .collect(); + if let Some(v) = version { + config.insert("head_version".to_string(), serde_json::json!(v)); + } + h.speculator = Some(SpeculatorBundle { + arch: "gemma4-mtp".into(), + config, + tensors: vec![], + }); + } + let json = h.to_canonical_json().unwrap(); + let mut f = std::fs::File::create(&path).unwrap(); + f.write_all(&MAGIC).unwrap(); + f.write_all(&FORMAT_VERSION.to_le_bytes()).unwrap(); + f.write_all(&(json.len() as u64).to_le_bytes()).unwrap(); + f.write_all(&json).unwrap(); + cache::write_sidecar( + &vdir, + &HubSidecar { + id: id.into(), + source_kind: source_kind.into(), + hf_repo: id.into(), + source_repo: None, + revision: "main".into(), + variant: variant.into(), + profile: None, + pulled_at: "0".into(), + base_sha256: Some(old.clone()), + file: Some(format!("{variant}.base")), + }, + ) + .unwrap(); + path + }; + let reg = |variant: &str, speculator: &str, sha: &str| { + MergedRegistry::new( + root.clone(), + catalog(&format!( + r#"{{"id":"{id}","hf_repo":"{id}","file":"{variant}.base","quant":"{variant}","sha256":"{sha}"{speculator}}}"# + )), + ) + }; + let promised = r#","speculator":"gemma4-mtp""#; + + let headless = install("default-q8", None, "catalog"); + assert_eq!( + stale_head_variant(®("default-q8", promised, &new), id, &headless).as_deref(), + Some("default-q8") + ); + assert_eq!( + stale_head_variant( + ®("default-q8", r#","speculator":"mtp""#, &new), + id, + &headless + ) + .as_deref(), + Some("default-q8") + ); + assert_eq!( + stale_head_variant(®("default-q8", "", &new), id, &headless), + None, + "the catalog promises no head" + ); + assert_eq!( + stale_head_variant(®("default-q8", promised, &old), id, &headless), + None, + "the catalog's file is this one" + ); + assert_eq!( + stale_head_variant(®("default-q4", promised, &new), id, &headless), + None, + "another variant's row" + ); + + let with_head = install("default-q4", Some(None), "catalog"); + assert_eq!( + stale_head_variant(®("default-q4", promised, &new), id, &with_head), + None, + "it has its head" + ); + + // A versioned head: stale only when the catalog names another version. + let v1 = install("cuda-q8", Some(Some("1")), "catalog"); + let versioned = + |v: &str| format!(r#","speculator":"gemma4-mtp","speculator_version":"{v}""#); + assert_eq!( + stale_head_variant(®("cuda-q8", &versioned("1"), &new), id, &v1), + None, + "the catalog's head is this version" + ); + assert_eq!( + stale_head_variant(®("cuda-q8", &versioned("2"), &new), id, &v1).as_deref(), + Some("cuda-q8"), + "the catalog's head is a newer version" + ); + assert_eq!( + stale_head_variant(®("cuda-q8", promised, &new), id, &v1), + None, + "the catalog names no version to hold it to" + ); + assert_eq!( + stale_head_variant(®("default-q4", &versioned("1"), &new), id, &with_head) + .as_deref(), + Some("default-q4"), + "an unversioned head against a versioned row" + ); + + let converted = install("default-f16", None, "huggingface"); + assert_eq!( + stale_head_variant(®("default-f16", promised, &new), id, &converted), + None, + "not the catalog's install" + ); + + // Without a speculation request the file is served as it is. + let r = reg("default-q8", promised, &new); + assert_eq!( + serve_ready(&r, id, headless.clone(), None).unwrap(), + headless + ); + } } diff --git a/base-convert/crates/base-convert/src/main.rs b/base-convert/crates/base-convert/src/main.rs index ed07d37..534ab49 100644 --- a/base-convert/crates/base-convert/src/main.rs +++ b/base-convert/crates/base-convert/src/main.rs @@ -2,9 +2,11 @@ use anyhow::{bail, Context, Result}; use clap::{Parser, Subcommand, ValueEnum}; use std::path::PathBuf; +mod embed; mod gpt_oss; mod hub; mod lora; +mod selfupdate; mod serve; mod superfluid; @@ -35,6 +37,9 @@ Convert & author: verify Verify a signed `.base` file keygen Generate a signing keypair +Keep up to date: + self Update basert (`basert self update`), or switch between installed versions + Options: -h, --help Print help -V, --version Print version @@ -67,6 +72,9 @@ enum Cmd { Verify(VerifyArgs), /// Inspect a `.base` file (summary of header + tensors + slots). Inspect(InspectArgs), + /// Embed a converted MTP head sidecar in a bundle as its built-in + /// speculator head (Gemma 4's separate `-assistant` checkpoint). + AddSpeculator(embed::AddSpeculatorArgs), /// Generate an ed25519 keypair for signing. Keygen(KeygenArgs), /// Download a model into the local hub cache (pre-converted `.base` from @@ -80,6 +88,9 @@ enum Cmd { CatalogManifest(CatalogManifestArgs), /// Install or check superfluid, the server `basert serve` runs. Superfluid(superfluid::SuperfluidArgs), + /// Update basert itself, or switch between the versions installed here. + #[command(name = "self")] + SelfUpdate(selfupdate::SelfArgs), /// Runtime commands — `serve`, `chat`, `complete`, `bench`, … — handled /// by the BaseRT runtime (dispatched in `hub::dispatch_external`). #[command(external_subcommand)] @@ -118,6 +129,18 @@ struct ConvertArgs { #[arg(long)] synthetic: bool, + /// Embed this converted Gemma 4 MTP head (a `gemma4-mtp` sidecar: + /// `basert convert google/gemma-4--it-assistant`) in the bundle as + /// its speculator head, as `basert add-speculator` does to an existing + /// bundle. The output must not exist yet. + #[arg(long, value_name = "HEAD.base")] + mtp_head: Option, + + /// With `--mtp-head`: the head's version, recorded in the bundle + /// (`add-speculator --head-version`). + #[arg(long, value_name = "V", requires = "mtp_head")] + mtp_head_version: Option, + /// Canonical-quant profile JSON (e.g. /// `tools/base-convert/profiles/dense-q4mix.json`). When set, /// per-tensor quant decisions come from the profile rules; the @@ -499,6 +522,7 @@ fn main() -> Result<()> { { print_banner(); } + selfupdate::at_launch(raw_arg1.as_deref()); let args = Args::parse(); match args.cmd { Cmd::Convert(a) => cmd_convert(a), @@ -506,12 +530,14 @@ fn main() -> Result<()> { Cmd::Sign(a) => cmd_sign(a), Cmd::Verify(a) => cmd_verify(a), Cmd::Inspect(a) => cmd_inspect(a), + Cmd::AddSpeculator(a) => embed::cmd_add_speculator(a), Cmd::Keygen(a) => cmd_keygen(a), Cmd::Pull(a) => hub::cmd_pull(a), Cmd::List(a) => hub::cmd_list(a), Cmd::CatalogScan(a) => hub::cmd_catalog_scan(a.org, a.out, a.dry_run), Cmd::CatalogManifest(a) => hub::cmd_catalog_manifest(a.bundle), Cmd::Superfluid(a) => superfluid::cmd(a), + Cmd::SelfUpdate(a) => selfupdate::cmd(a), Cmd::External(argv) => hub::dispatch_external(argv), } } @@ -532,14 +558,51 @@ fn cmd_keygen(args: KeygenArgs) -> Result<()> { Ok(()) } -fn cmd_convert(args: ConvertArgs) -> Result<()> { +fn cmd_convert(mut args: ConvertArgs) -> Result<()> { + let Some(head) = args.mtp_head.take() else { + return cmd_convert_bundle(args); + }; + let args_head_version = args.mtp_head_version.take(); + // `--mtp-head`: the model is converted beside its final name, then the + // head is embedded behind it into the final file (add_speculator copies + // the blob), and the headless conversion goes — briefly the one extra + // copy on disk, as the writer's own temp blob is. + let output = args + .output + .clone() + .unwrap_or_else(|| default_output(&args.input)); + let headless = output.with_extension("headless.base"); + for p in [&output, &headless] { + if p.exists() { + bail!("{}: exists; refusing to overwrite it", p.display()); + } + } + args.output = Some(headless.clone()); + cmd_convert_bundle(args)?; + let embedded = embed::add_speculator(&headless, &head, &output, args_head_version.as_deref()); + let _ = std::fs::remove_file(&headless); + let s = embedded?; + eprintln!( + " mtp: {} head ({} tensors, {} bytes) embedded as `{}`", + s.head_arch, s.n_head_tensors, s.head_bytes, s.kind + ); + Ok(()) +} + +/// `.base`, the output `convert` writes when none is named. +fn default_output(input: &std::path::Path) -> PathBuf { + let mut p = input.to_path_buf(); + p.set_extension("base"); + p +} + +fn cmd_convert_bundle(args: ConvertArgs) -> Result<()> { eprintln!("base-convert v{}", env!("CARGO_PKG_VERSION")); eprintln!(" input: {}", args.input.display()); - let output = args.output.clone().unwrap_or_else(|| { - let mut p = args.input.clone(); - p.set_extension("base"); - p - }); + let output = args + .output + .clone() + .unwrap_or_else(|| default_output(&args.input)); eprintln!(" output: {}", output.display()); let ctx = QuantContext::from_args(&args)?; @@ -1763,11 +1826,12 @@ fn convert_hf(input: &std::path::Path, output: &std::path::Path, ctx: &QuantCont let mapper: &dyn base_arch::HfMapper = if let Some(m) = base_arch::dflash::hf_mapper_for_architectures(&hf.config) { - eprintln!( - " arch: {} (block drafter sidecar: {})", - model_type, - m.canonical_arch() - ); + let kind = if base_arch::dflash::is_mtp_sidecar_arch(m.canonical_arch()) { + "MTP head sidecar" + } else { + "block drafter sidecar" + }; + eprintln!(" arch: {} ({kind}: {})", model_type, m.canonical_arch()); m } else if let Some(m) = base_arch::dflash::hf_mapper_for_eagle3(&hf.config) { eprintln!(" arch: {} (EAGLE-3 head sidecar)", model_type); @@ -3554,6 +3618,7 @@ impl<'a> FcSplitProvider<'a> { let mut splits = std::collections::BTreeMap::new(); let mut rewritten = Vec::with_capacity(source_names.len()); let mtp = canonical_arch.starts_with("qwen35"); + let mtp_sidecar = base_arch::dflash::is_mtp_sidecar_arch(canonical_arch); let eagle3 = base_arch::dflash::is_eagle3_arch(canonical_arch); let drafter = base_arch::dflash::is_block_drafter_arch(canonical_arch) || eagle3; for n in source_names { @@ -3585,6 +3650,18 @@ impl<'a> FcSplitProvider<'a> { splits.insert(h.clone(), (n.clone(), 1, 2)); rewritten.push(e); rewritten.push(h); + } else if mtp_sidecar && n == "pre_projection.weight" { + // [hidden, 2 * backbone]: the reference concatenates + // (token embedding, target hidden) on the input axis, so + // column half 0 is the embedding projection. + let shape = inner.source_shape(n)?; + if shape.len() != 2 || shape[1] % 2 != 0 { + bail!("{n}: expected a [hidden, 2 * backbone] Gemma 4 MTP pre_projection, got {shape:?}"); + } + splits.insert("fc_embed.weight".to_string(), (n.clone(), 0, 2)); + splits.insert("fc_hidden.weight".to_string(), (n.clone(), 1, 2)); + rewritten.push("fc_embed.weight".to_string()); + rewritten.push("fc_hidden.weight".to_string()); } else if drafter && n == "fc.weight" { let shape = inner.source_shape(n)?; if shape.len() != 2 @@ -3890,6 +3967,7 @@ fn convert_generic( if !fcs.splits.is_empty() { if base_arch::dflash::is_block_drafter_arch(canonical_arch) || base_arch::dflash::is_eagle3_arch(canonical_arch) + || base_arch::dflash::is_mtp_sidecar_arch(canonical_arch) { eprintln!( " drafter: split fused projections into {} column blocks (f16)", @@ -4214,9 +4292,20 @@ fn convert_generic( // block step is small next to the target's verify) except for what // --quantize-drafter names: see drafter_tensor_stays_f16. let is_drafter_arch = base_arch::dflash::is_block_drafter_arch(canonical_arch) - || base_arch::dflash::is_eagle3_arch(canonical_arch); + || base_arch::dflash::is_eagle3_arch(canonical_arch) + || base_arch::dflash::is_mtp_sidecar_arch(canonical_arch); for (src_name, canonical) in &mapped { + // The Gemma 4 MTP sidecar scores with its own 262k-row table at the + // head's width (`output.weight`, read on every chained draft: 134 MB + // f16 on the E4B head, 537 MB on the 26B-A4B's): under + // --quantize-drafter it follows --target too, as the embedded heads' + // `mtp.lm_head.weight` does; its fc_embed / fc_hidden / post_fc and + // norms stay f16 (drafter_tensor_stays_f16 quantizes `layers.*` only, + // and `fc.*` taps, which the head has none of). + let mtp_sidecar_head = + base_arch::dflash::is_mtp_sidecar_arch(canonical_arch) && canonical == "output.weight"; let drafter_f16 = is_drafter_arch + && !(ctx.quantize_drafter && mtp_sidecar_head) && drafter_tensor_stays_f16(ctx.quantize_drafter, ctx.keep_drafter_fc_f16, canonical); let is_spec = spec_names.contains(canonical); // --quantize-drafter also quantizes an in-bundle speculator's transformer @@ -5250,6 +5339,41 @@ fn eagle3_canonical(name: &str) -> Option { }) } +/// Gemma 4 MTP head sidecar tensor → runtime name. The decoder layers take +/// the Gemma 4 layer rules (four norms, layer_scalar, q-only attention). The +/// head's own tensors get names that do NOT collide with the target's, since +/// the runtime resolves what the sidecar lacks through the target: its +/// `embed_tokens` is the DRAFT logit projection (`output.weight`, hidden-width +/// rows — the INPUT embedding stays the target's `token_embedding.weight`), +/// `model.norm` is the head's final norm (`output_norm.weight`, the name the +/// encoder's output stage reads; the target's final norm is applied to the +/// tapped hidden in the prologue through the target's store), and +/// `pre_projection` is split by input column into the embedding half and the +/// hidden half (`fc_embed.weight` / `fc_hidden.weight`, as the Qwen MTP fc). +fn gemma4_mtp_canonical(name: &str) -> Option { + match name { + "model.embed_tokens.weight" => Some("output.weight".to_string()), + // The head's final norm takes the encoder's output-stage name: the + // runtime norms the head's own residual with it before `output.weight` + // (the TARGET's final norm is applied to the tapped hidden in the + // prologue, reached through the target's own store). + "model.norm.weight" => Some("output_norm.weight".to_string()), + "fc_embed.weight" | "fc_hidden.weight" => Some(name.to_string()), + "post_projection.weight" => Some("post_fc.weight".to_string()), + "pre_projection.weight" => None, // split into fc_embed / fc_hidden by FcSplitProvider + _ => { + if name.starts_with("model.layers.") { + match to_canonical_name(name, "gemma4")? { + Canonical::Main(c) => Some(c), + _ => None, + } + } else { + None + } + } + } +} + /// Block-drafter (DFlash / DSpark) sidecar tensor → canonical name. The /// decoder layer goes through the qwen3 main-layer rules (so the runtime's /// per-layer aliases apply as for the target's own layers); `norm.weight` @@ -5380,6 +5504,9 @@ fn to_canonical_name(name: &str, arch: &str) -> Option { if base_arch::dflash::is_eagle3_arch(arch) { return eagle3_canonical(name).map(Canonical::Main); } + if base_arch::dflash::is_mtp_sidecar_arch(arch) { + return gemma4_mtp_canonical(name).map(Canonical::Main); + } // Qwen3.5/3.6/3.8 checkpoints ship a Multi-Token-Prediction head (`mtp.*` // — one extra decoder layer + fc / pre-fc norms / output norm, its own // experts on the MoE variants). It is the bundle's speculator: routed @@ -6835,6 +6962,35 @@ mod canonical_name_tests { } } + #[test] + fn gemma4_mtp_head_tensors_take_names_apart_from_the_targets() { + // The runtime resolves what the sidecar lacks through the target, so + // the head's own embedding / norm must not take the target's names. + let a = "gemma4-mtp"; + for (src, want) in [ + ("model.embed_tokens.weight", "output.weight"), + ("model.norm.weight", "output_norm.weight"), + ("post_projection.weight", "post_fc.weight"), + ("fc_embed.weight", "fc_embed.weight"), + ("fc_hidden.weight", "fc_hidden.weight"), + ] { + assert_eq!(main_canon(src, a).as_deref(), Some(want), "{src}"); + } + // pre_projection is split into fc_embed / fc_hidden before naming; + // the clustered vocabulary is not read. + for src in [ + "pre_projection.weight", + "masked_embedding.centroids.weight", + "masked_embedding.token_ordering", + ] { + assert_eq!(main_canon(src, a), None, "{src}"); + } + // The layers take the Gemma 4 rules. + let q = "model.layers.3.self_attn.q_proj.weight"; + assert!(main_canon(q, a).is_some()); + assert_eq!(main_canon(q, a), main_canon(q, "gemma4")); + } + #[test] fn block_drafter_qk_norm_check_sees_the_mapped_names() { // check_block_drafter_qk_norms looks for these exact canonical names; diff --git a/base-convert/crates/base-convert/src/selfupdate.rs b/base-convert/crates/base-convert/src/selfupdate.rs new file mode 100644 index 0000000..0842697 --- /dev/null +++ b/base-convert/crates/base-convert/src/selfupdate.rs @@ -0,0 +1,2685 @@ +//! `basert self update|check|list|use|install`: basert updating itself. +//! +//! The install script's layout: each release unpacked whole under +//! `/versions//`, a `versions/current` link to the one in use, and in +//! `` itself (the directory on PATH) a link per file of the release +//! through `versions/current`. An update unpacks the new release beside the +//! old, checks that its launcher runs and that a superfluid serves its engine, +//! then turns the one link: the launcher, the runtime tools, the engine +//! library and its kernel files change together, and a process already running +//! keeps the files it started with. The version replaced stays, for `basert +//! self use`; older ones are removed, unless a server still runs on one or it +//! holds a `keep` file. +//! +//! A copy the install script placed before this layout (the release's files +//! directly in ``, beside a `.release` stamp) is moved under it by the +//! first update, as the version to go back to. A copy a package manager placed +//! (an `installed-by` file beside it says which), cargo's or a source build is +//! updated the way it was installed, and these commands say how. +//! +//! superfluid follows the engine. When the superfluid `basert serve` runs +//! cannot serve the new engine, the newest superfluid release that states the +//! engine's ABI number is downloaded and asked before anything changes; when +//! no release does, basert stays where it is. When it can, a newer superfluid +//! that also serves the engine is installed as well. +//! +//! A command run on a terminal looks for a newer release once a day, in a +//! process of its own so that nothing waits on it, and a later command prints +//! one line when there is one (a few days after it is published, so that +//! early reports land first). `BASERT_NO_UPDATE_CHECK=1` turns this off. + +use std::fs; +use std::io::IsTerminal; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use anyhow::{bail, Context, Result}; +use clap::{Args, Subcommand}; +use serde_json::{json, Value}; + +use crate::superfluid; + +/// The releases the newest is chosen from: the open repo's, where the +/// install script downloads from. +const RELEASES: &str = "https://api.github.com/repos/basecompute/baseRT/releases?per_page=30"; +/// Another release listing in GitHub's form, a URL or a file: a mirror's, or +/// the tests'. Its assets may be named by path. +pub const RELEASES_ENV: &str = "BASERT_RELEASES"; +/// Set (and not `0`), no command looks for a newer release by itself. +pub const NO_CHECK_ENV: &str = "BASERT_NO_UPDATE_CHECK"; +/// The file beside a launcher a package manager installed, one line +/// `: `: the copy is that manager's to update, with that +/// command. +pub const INSTALLED_BY: &str = "installed-by"; +/// The release a directory holds, as its tag: in the root, the one in use (the +/// install script reads it to skip a download); in a version, that version. +const STAMP: &str = ".release"; +/// A version holding this file is never removed. +const KEEP: &str = "keep"; +/// In a version, a file per `basert serve` running on its engine, named by pid. +const SERVING: &str = ".serving"; +/// A release younger than this is not announced. +const HOLD_OFF: Duration = Duration::from_secs(3 * 24 * 60 * 60); +/// How often a command looks for a newer release, and how often it says so. +const CHECK_EVERY: Duration = Duration::from_secs(24 * 60 * 60); +const VERSION: &str = env!("CARGO_PKG_VERSION"); + +/// `basert self `. +#[derive(Args, Debug)] +pub struct SelfArgs { + #[command(subcommand)] + pub action: Action, +} + +#[derive(Subcommand, Debug)] +pub enum Action { + /// Install the newest release for this machine and make it the one in use. + /// + /// The superfluid `basert serve` runs is brought to the newest release + /// that serves the new engine; when none does yet, nothing changes. + Update { + /// Go to this release instead of the newest (an older one too), e.g. 0.4.3. + #[arg(long = "version", value_name = "V")] + version: Option, + /// Say what would be done, and change nothing. + #[arg(long)] + dry_run: bool, + /// For a copy Homebrew, npm, pipx, uv or cargo installed: run that + /// manager's update without asking (needed without a terminal). + #[arg(long, short = 'y')] + yes: bool, + }, + /// Say whether a newer release exists, without installing it. + Check { + /// Record the answer for the notice a later command prints, and print nothing. + #[arg(long, hide = true)] + record: bool, + /// Start `--record` in a process of its own and return at once. + #[arg(long, hide = true)] + background: bool, + }, + /// The versions installed here, and which is in use. + List, + /// Make an installed version the one in use (the one it replaces stays). + Use { + /// The version, as `basert self list` shows it. + version: String, + }, + /// Place an unpacked release bundle here and make it the one in use. + /// + /// What the install script runs; a copy it placed before is kept under + /// versions/ as the version to go back to. + Install { + /// The unpacked bundle (moved into place). + #[arg(long, value_name = "DIR")] + from: PathBuf, + /// The install directory (default: $BASERT_INSTALL_DIR, else ~/.basert). + #[arg(long, value_name = "DIR")] + root: Option, + }, +} + +pub fn cmd(args: SelfArgs) -> Result<()> { + let mut say = |line: &str| eprintln!("{line}"); + match args.action { + Action::Update { + version, + dry_run, + yes, + } => update(version.as_deref(), dry_run, yes, &mut say), + Action::Check { record: true, .. } => { + record(); + Ok(()) + } + Action::Check { + background: true, .. + } => background(), + Action::Check { .. } => check(), + Action::List => list(), + Action::Use { version } => use_version(&version, &mut say), + Action::Install { from, root } => install_from(&from, root, &mut say), + } +} + +// --- where this basert came from --------------------------------------------- + +/// How the running basert was installed, which decides who updates it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Installed { + /// `/versions//basert`: the versions layout, this command's to update. + Managed { root: PathBuf, version: String }, + /// The install script's earlier layout: the release's files directly in + /// `root`, beside a `.release` stamp. + InPlace { root: PathBuf }, + /// A package manager's copy, cargo's, a source build: `how` it is updated, + /// and the command that does it when there is one. + Other { + exe: PathBuf, + how: String, + command: Option, + }, +} + +impl Installed { + /// Whether a package manager (cargo among them) placed this copy, so its + /// folder is that manager's: an upgrade replaces it, an uninstall leaves + /// behind what basert put there. + pub fn by_package_manager(&self) -> bool { + matches!( + self, + Installed::Other { + command: Some(_), + .. + } + ) + } +} + +pub fn installed_as(exe: &Path) -> Installed { + let real = fs::canonicalize(exe).unwrap_or_else(|_| exe.to_path_buf()); + let dir = real.parent().unwrap_or(Path::new("/")).to_path_buf(); + if let (Some(version), Some(versions)) = + (dir.file_name().and_then(|n| n.to_str()), dir.parent()) + { + if versions.file_name().is_some_and(|n| n == "versions") && version_key(version).is_some() { + if let Some(root) = versions.parent() { + return Installed::Managed { + root: root.to_path_buf(), + version: version.to_string(), + }; + } + } + } + // A package keeps the release's layout too, so it is told apart by its + // file, not by where it is. + let named = package_manager(&dir); + if named.is_none() && dir.join(STAMP).is_file() { + return Installed::InPlace { root: dir }; + } + match delegate(&real, named) { + Some(d) => Installed::Other { + exe: real, + how: format!("{} installed it: `{}` updates it", d.manager(), d.command()), + command: Some(d.command().to_string()), + }, + None => Installed::Other { + how: match real.to_string_lossy().contains("/target/") { + true => "it is a source build: build it again", + false => "the install script did not place it: update it the way it was installed", + } + .to_string(), + exe: real, + command: None, + }, + } +} + +/// The manager and command an `installed-by` file in `dir` names. +fn package_manager(dir: &Path) -> Option<(String, String)> { + let line = fs::read_to_string(dir.join(INSTALLED_BY)).ok()?; + let (manager, command) = line.trim().split_once(": ")?; + Some((manager.trim().to_string(), command.trim().to_string())) +} + +// --- a copy a package manager installed ---------------------------------------- + +/// How a copy another tool installed is updated: by that tool's own command, +/// which `self update` runs (after asking), or which it only names, saying +/// why. The command is built here from the manager's name, never read from +/// the `installed-by` file: that file's own text is for showing. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Delegate { + Run { + manager: String, + /// The command as a user would type it. + command: String, + /// What is run: the manager's binary as found from the copy's own + /// path, so it is the installation that owns the copy. + argv: Vec, + /// Put first on PATH (npm's `node`, beside it). + path_first: Option, + }, + Name { + manager: String, + command: String, + why: String, + }, +} + +impl Delegate { + fn manager(&self) -> &str { + match self { + Delegate::Run { manager, .. } | Delegate::Name { manager, .. } => manager, + } + } + + fn command(&self) -> &str { + match self { + Delegate::Run { command, .. } | Delegate::Name { command, .. } => command, + } + } +} + +/// The delegate for the launcher at the real path `exe`, from the manager its +/// `installed-by` file names, or from where it is for packages published +/// before they carried the file; `None` for a copy no package manager placed. +fn delegate(exe: &Path, named: Option<(String, String)>) -> Option { + let p = exe.to_string_lossy().into_owned(); + let dir = exe.parent().unwrap_or(Path::new("/")); + // pipx and uv keep the wheel in a venv of their own, whose file says pip. + if p.contains("/pipx/venvs/") { + return Some(on_path("pipx", &["upgrade", "basert"])); + } + if p.contains("/uv/tools/") { + return Some(on_path("uv", &["tool", "upgrade", "basert"])); + } + let manager = named.as_ref().map(|(m, _)| m.to_ascii_lowercase()); + Some(match manager.as_deref() { + Some("homebrew" | "brew") => homebrew(&p), + Some("npm") => npm(&p), + Some("pip") => pip(&p), + Some("cargo") => cargo(dir), + Some(_) => { + let (manager, command) = named?; + Delegate::Name { + manager, + command, + why: "basert runs only Homebrew, npm, pipx, uv and cargo itself".to_string(), + } + } + None if p.contains("/Cellar/") => homebrew(&p), + None if p.contains("/node_modules/") => npm(&p), + None if p.contains("/site-packages/") => pip(&p), + None if p.contains("/.cargo/bin/") => cargo(dir), + None => return None, + }) +} + +/// Whether this user may write `path`, as the manager would need to. +fn writable(path: &Path) -> bool { + use std::os::unix::ffi::OsStrExt; + let Ok(c) = std::ffi::CString::new(path.as_os_str().as_bytes()) else { + return false; + }; + // SAFETY: `c` is a NUL-terminated path that outlives the call. + unsafe { libc::access(c.as_ptr(), libc::W_OK) == 0 } +} + +/// `tool args…` with `tool` from PATH, or named when PATH has none. +fn on_path(tool: &str, args: &[&str]) -> Delegate { + let command = std::iter::once(tool) + .chain(args.iter().copied()) + .collect::>() + .join(" "); + let found = std::env::var_os("PATH").and_then(|path| { + std::env::split_paths(&path) + .map(|d| d.join(tool)) + .find(|p| superfluid::is_executable(p)) + }); + match found { + Some(bin) => Delegate::Run { + manager: tool.to_string(), + command, + argv: std::iter::once(bin.display().to_string()) + .chain(args.iter().map(|a| a.to_string())) + .collect(), + path_first: None, + }, + None => Delegate::Name { + manager: tool.to_string(), + why: format!("no {tool} on PATH"), + command, + }, + } +} + +/// `brew upgrade ` by the Homebrew whose Cellar holds the copy, +/// unless the formula is pinned or that Homebrew is another user's. +fn homebrew(p: &str) -> Delegate { + let manager = "Homebrew".to_string(); + let Some(i) = p.find("/Cellar/") else { + return Delegate::Name { + manager, + command: "brew upgrade basert".to_string(), + why: "the Homebrew it came from cannot be told from where it is".to_string(), + }; + }; + let prefix = Path::new(&p[..i]); + let formula = p[i + "/Cellar/".len()..] + .split('/') + .next() + .unwrap_or("basert"); + let command = format!("brew upgrade {formula}"); + let brew = prefix.join("bin").join("brew"); + let why = if !superfluid::is_executable(&brew) { + format!("there is no {}", brew.display()) + } else if prefix.join("var/homebrew/pinned").join(formula).exists() { + format!("it is pinned in Homebrew: `brew unpin {formula}` first") + } else if !writable(&prefix.join("Cellar")) { + format!("the Homebrew in {} is another user's", prefix.display()) + } else { + return Delegate::Run { + manager, + command, + argv: vec![brew.display().to_string(), "upgrade".into(), formula.into()], + path_first: None, + }; + }; + Delegate::Name { + manager, + command, + why, + } +} + +/// `npm install --global` by the npm whose global folder holds the copy; a +/// project's dependency (or pnpm's, yarn's, bun's) is that project's to update. +fn npm(p: &str) -> Delegate { + let manager = "npm".to_string(); + let command = "npm install --global @basecompute/basert@latest".to_string(); + let Some(i) = p.find("/lib/node_modules/") else { + return Delegate::Name { + manager, + command: "npm install @basecompute/basert@latest".to_string(), + why: "it is a project's dependency (or pnpm's, yarn's or bun's): update it in that project" + .to_string(), + }; + }; + let prefix = Path::new(&p[..i]); + let bin = prefix.join("bin"); + let npm = bin.join("npm"); + let why = if !superfluid::is_executable(&npm) { + format!("there is no {}", npm.display()) + } else if !writable(&prefix.join("lib").join("node_modules")) { + format!( + "npm's global folder in {} needs root to write: run it yourself", + prefix.display() + ) + } else { + return Delegate::Run { + manager, + command, + argv: vec![ + npm.display().to_string(), + "install".into(), + "--global".into(), + "@basecompute/basert@latest".into(), + ], + path_first: Some(bin), + }; + }; + Delegate::Name { + manager, + command, + why, + } +} + +/// pip is only named: the environment may be a project's, which a lock file +/// (Poetry, uv, pip-tools) pins, or the system's, which refuses or needs root. +fn pip(p: &str) -> Delegate { + // ...//lib/python3.x/site-packages/baseRT/_engine/basert + let python = p + .find("/lib/python") + .map(|i| Path::new(&p[..i]).join("bin").join("python")) + .filter(|py| superfluid::is_executable(py)); + Delegate::Name { + manager: "pip".to_string(), + command: match python { + Some(py) => format!("{} -m pip install --upgrade basert", py.display()), + None => "pip install --upgrade basert".to_string(), + }, + why: "its environment may be a project's, pinned by a lock file, or the system's: \ + run it in that environment yourself" + .to_string(), + } +} + +/// `cargo install basert-cli` by the cargo beside the copy. +fn cargo(dir: &Path) -> Delegate { + let cargo = dir.join("cargo"); + if !superfluid::is_executable(&cargo) { + return on_path("cargo", &["install", "basert-cli"]); + } + Delegate::Run { + manager: "cargo".to_string(), + command: "cargo install basert-cli".to_string(), + argv: vec![ + cargo.display().to_string(), + "install".into(), + "basert-cli".into(), + ], + path_first: Some(dir.to_path_buf()), + } +} + +/// `self update` of a copy another tool installed: its manager's command, run +/// after asking (or with `--yes`) when it is one basert runs, else named. +fn update_by_manager( + exe: &Path, + version: Option<&str>, + dry_run: bool, + yes: bool, + say: &mut dyn FnMut(&str), +) -> Result<()> { + let real = fs::canonicalize(exe).unwrap_or_else(|_| exe.to_path_buf()); + let dir = real.parent().unwrap_or(Path::new("/")).to_path_buf(); + let Some(plan) = delegate(&real, package_manager(&dir)) else { + let Installed::Other { how, .. } = installed_as(&real) else { + bail!("{} is not a package manager's copy", real.display()); + }; + bail!( + "{} was not installed by the install script or a package manager, so `basert self` does not change it; {how}", + real.display() + ); + }; + let (manager, command, argv, path_first) = match (plan, version) { + ( + Delegate::Name { + manager, + command, + why, + }, + _, + ) => bail!( + "{} was installed by {manager}, which updates it: run `{command}` ({why})", + real.display() + ), + ( + Delegate::Run { + manager, command, .. + }, + Some(_), + ) => bail!( + "{} was installed by {manager}, which chooses its version: `{command}` updates it, \ + and a version is pinned there", + real.display() + ), + ( + Delegate::Run { + manager, + command, + argv, + path_first, + }, + None, + ) => (manager, command, argv, path_first), + }; + if dry_run { + say(&format!( + "{} was installed by {manager}: `self update` would run {}", + real.display(), + argv.join(" ") + )); + return Ok(()); + } + // The manager replaces this directory: a server's workers load the engine + // from it again with each model. cargo replaces the launcher alone; its + // engine is wherever BASERT_LIB names. + let busy = match manager.as_str() { + "cargo" => Vec::new(), + _ => servers_on_package(&dir), + }; + if !busy.is_empty() { + let pids = busy + .iter() + .map(i32::to_string) + .collect::>() + .join(", "); + bail!( + "a server runs on the engine in {} (pid {pids}), which `{command}` replaces: stop it, then run this again", + dir.display() + ); + } + if !yes { + if !(std::io::stdin().is_terminal() && std::io::stderr().is_terminal()) { + bail!( + "{} was installed by {manager}: run `{command}`, or `basert self update --yes` to let basert run it", + real.display() + ); + } + if !superfluid::confirm(&format!( + "{} was installed by {manager}. Run `{}`? [Y/n] ", + real.display(), + argv.join(" ") + ))? { + bail!("nothing was run; `{command}` updates it"); + } + } + if manager == "cargo" { + say("cargo builds basert from source: this takes a few minutes"); + } + say(&format!("running {}", argv.join(" "))); + let mut run = Command::new(&argv[0]); + run.args(&argv[1..]); + if let Some(first) = path_first { + let rest = std::env::var_os("PATH").unwrap_or_default(); + let path = + std::env::join_paths(std::iter::once(first).chain(std::env::split_paths(&rest)))?; + run.env("PATH", path); + } + let status = run + .status() + .with_context(|| format!("running {}", argv[0]))?; + if !status.success() { + bail!("`{}` failed ({status})", argv.join(" ")); + } + say("the next `basert serve` brings superfluid to a release that serves the new engine (`basert superfluid install` does it now)"); + Ok(()) +} + +/// The servers on the engine in a package's directory `dir`: basert's and +/// superfluid's programs run from it (only theirs: a cargo install shares +/// `~/.cargo/bin` with other tools), and those `basert serve` recorded in the +/// state directory. +fn servers_on_package(dir: &Path) -> Vec { + let ours = |exe: &Path| { + exe.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with("basert") || n.starts_with("superfluid")) + }; + let mut pids = processes_running(dir, &ours); + if let Some(records) = serving_records() { + for pid in servers_recorded(&records, dir) { + if !pids.contains(&pid) { + pids.push(pid); + } + } + } + pids +} + +fn this_exe() -> Result { + std::env::current_exe().context("this program's own path") +} + +/// The directory the install script made: `` for the versions layout, +/// else this launcher's own directory as it was started. superfluid is looked +/// for beside it. +pub fn install_dir() -> Result { + let exe = this_exe()?; + if let Installed::Managed { root, .. } = installed_as(&exe) { + return Ok(root); + } + exe.parent() + .map(Path::to_path_buf) + .context("this program's own directory") +} + +/// The directory this launcher's release files are in: for the versions +/// layout the version it runs from, a real path no update changes under a +/// running server; else its own directory as it was started. +pub fn bundle_dir() -> Result { + let exe = this_exe()?; + if let Installed::Managed { root, version } = installed_as(&exe) { + return Ok(Layout { root }.version_dir(&version)); + } + exe.parent() + .map(Path::to_path_buf) + .context("this program's own directory") +} + +// --- releases ---------------------------------------------------------------- + +/// The engine bundle name's platform for this machine: +/// `basert-engine--.tar.gz`. +pub fn platform() -> Result<&'static str> { + match (std::env::consts::OS, std::env::consts::ARCH) { + ("macos", "aarch64") => Ok("macos-arm64"), + ("linux", "aarch64") => Ok("linux-arm64-cuda"), + (os, arch) => bail!("no basert release is published for {os}/{arch}"), + } +} + +/// A basert release for this machine, as a listing names it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Release { + /// `0.4.3`. + pub version: String, + /// `v0.4.3`. + pub tag: String, + pub published: String, + /// The engine ABI number the release states beside its bundles + /// (`basert-engine-abi-.txt`), from 0.4.0 on. + pub abi: Option, + tarball: String, + tarball_url: String, + checksum_url: String, +} + +/// `0.4.3` (or `v0.4.3`) as `[0, 4, 3]`; `None` for anything else. +pub fn version_key(v: &str) -> Option> { + v.strip_prefix('v') + .unwrap_or(v) + .split('.') + .map(|part| part.parse().ok()) + .collect() +} + +/// The releases in a listing of GitHub's form that are neither drafts nor +/// pre-releases and publish this platform's bundle and its checksum, newest +/// first. +pub fn releases_in(listing: &Value, platform: &str) -> Vec { + let mut found: Vec = listing + .as_array() + .into_iter() + .flatten() + .filter(|r| !r["draft"].as_bool().unwrap_or(false)) + .filter(|r| !r["prerelease"].as_bool().unwrap_or(false)) + .filter_map(|r| { + let tag = r["tag_name"].as_str()?; + let version = tag.strip_prefix('v').unwrap_or(tag); + version_key(version)?; + let assets = r["assets"].as_array()?; + let url = |name: &str| { + assets.iter().find(|a| a["name"] == name)?["browser_download_url"] + .as_str() + .map(str::to_string) + }; + let tarball = format!("basert-engine-{platform}-{version}.tar.gz"); + let abi = assets.iter().find_map(|a| { + a["name"] + .as_str()? + .strip_prefix("basert-engine-abi-")? + .strip_suffix(".txt")? + .parse() + .ok() + }); + Some(Release { + version: version.to_string(), + tag: tag.to_string(), + published: r["published_at"].as_str().unwrap_or_default().to_string(), + abi, + tarball_url: url(&tarball)?, + checksum_url: url(&format!("{tarball}.sha256"))?, + tarball, + }) + }) + .collect(); + found.sort_by_key(|r| std::cmp::Reverse(version_key(&r.version))); + found +} + +/// The release listing: `$BASERT_RELEASES`, else the open repo's. +fn listing() -> Result { + let named = std::env::var(RELEASES_ENV).ok().filter(|v| !v.is_empty()); + superfluid::read_listing(named.as_deref().unwrap_or(RELEASES)) +} + +/// The engine ABI number the header in a release directory states. +fn abi_in(dir: &Path) -> Option { + fs::read_to_string(dir.join("include").join("baseRT").join("baseRT.h")) + .ok()? + .lines() + .find_map(|l| l.strip_prefix("#define BASERT_ABI_VERSION "))? + .trim() + .parse() + .ok() +} + +/// What the launcher `basert` says its version is (`basert 0.4.3`). +fn launcher_version(launcher: &Path) -> Result { + let out = Command::new(launcher) + .arg("--version") + .stdin(Stdio::null()) + .output() + .with_context(|| format!("running {} --version", launcher.display()))?; + let text = String::from_utf8_lossy(&out.stdout); + match text.split_whitespace().collect::>()[..] { + ["basert", v, ..] if out.status.success() && version_key(v).is_some() => Ok(v.to_string()), + _ => bail!( + "{} --version answered {:?}, not `basert `", + launcher.display(), + text.trim() + ), + } +} + +// --- the versions layout ----------------------------------------------------- + +/// A release file, by name: what the install script replaces in its install +/// directory. Anything else there (superfluid's directory, the user's own) is +/// left alone. +fn is_bundle_file(name: &str) -> bool { + matches!(name, "basert" | "include" | "LICENSE" | "NOTICE") + || name.starts_with("basert-") + || name.starts_with("baseRT_") + || name.starts_with("libbaseRT") + || (name.starts_with("baseRT") + && (name.ends_with(".metallib") || name.ends_with(".fatbin"))) +} + +/// What `` links to through `versions/current`: every name in the +/// release but its dot files and a `keep` file. +fn members(dir: &Path) -> Vec { + let mut names: Vec = fs::read_dir(dir) + .map(|rd| { + rd.flatten() + .filter_map(|e| e.file_name().to_str().map(str::to_string)) + .filter(|n| !n.starts_with('.') && n != KEEP) + .collect() + }) + .unwrap_or_default(); + names.sort(); + names +} + +#[derive(Debug, Clone)] +pub struct Layout { + pub root: PathBuf, +} + +impl Layout { + fn versions(&self) -> PathBuf { + self.root.join("versions") + } + + pub fn version_dir(&self, v: &str) -> PathBuf { + self.versions().join(v) + } + + fn launcher(&self, v: &str) -> PathBuf { + self.version_dir(v).join("basert") + } + + pub fn current(&self) -> Option { + let target = fs::read_link(self.versions().join("current")).ok()?; + target.file_name()?.to_str().map(str::to_string) + } + + fn previous(&self) -> Option { + let v = fs::read_to_string(self.versions().join("previous")).ok()?; + let v = v.trim(); + (!v.is_empty() && self.launcher(v).is_file()).then(|| v.to_string()) + } + + /// The complete versions here, newest first. + pub fn on_disk(&self) -> Vec { + let mut all: Vec = fs::read_dir(self.versions()) + .map(|rd| { + rd.flatten() + .filter_map(|e| e.file_name().to_str().map(str::to_string)) + .filter(|n| version_key(n).is_some() && self.launcher(n).is_file()) + .collect() + }) + .unwrap_or_default(); + all.sort_by_key(|v| std::cmp::Reverse(version_key(v))); + all + } + + /// The version the root's `.release` stamp names. + fn stamped(&self) -> Option { + let tag = fs::read_to_string(self.root.join(STAMP)).ok()?; + let v = tag.trim().trim_start_matches('v'); + version_key(v).map(|_| v.to_string()) + } + + /// The root's stamp: the release in use, by its tag. + fn stamp(&self, tag: &str) -> Result<()> { + let at = self.root.join(STAMP); + fs::write(&at, format!("{tag}\n")).with_context(|| format!("writing {}", at.display())) + } + + /// Point `current` at `v`, in one rename, remembering what it pointed at. + fn set_current(&self, v: &str) -> Result<()> { + if !superfluid::is_executable(&self.launcher(v)) { + bail!("{v} is not installed here (`basert self list` shows what is)"); + } + let was = self.current(); + relink(&self.versions().join("current"), Path::new(v))?; + if let Some(was) = was.filter(|was| was != v) { + self.set_previous(&was)?; + } + Ok(()) + } + + fn set_previous(&self, v: &str) -> Result<()> { + let at = self.versions().join("previous"); + fs::write(&at, format!("{v}\n")).with_context(|| format!("writing {}", at.display())) + } + + /// The root's links: one per file of the version in use, through + /// `versions/current`. A link to a file that version does not have is + /// removed, and so is a file the install script placed in the root before + /// (kept under `versions/` by then): a tool left behind would load the new + /// version's kernel files. Anything else in the way is left as it is, and + /// said. + fn link(&self) -> Result> { + let Some(v) = self.current() else { + bail!("no version is in use in {}", self.versions().display()); + }; + let through = Path::new("versions").join("current"); + let mut notes = Vec::new(); + let names = members(&self.version_dir(&v)); + for name in &names { + let link = self.root.join(name); + let target = through.join(name); + match fs::read_link(&link) { + Ok(to) if to == target => continue, + Ok(to) if to.starts_with("versions") => {} + // A file or link is renamed over in one step; a directory + // (include/) has to go first. + _ if placed_by_script(&link, name) => { + if fs::symlink_metadata(&link).is_ok_and(|m| m.is_dir()) { + remove_placed(&link)?; + } + } + Ok(to) => { + notes.push(format!( + "{} links to {}, not into versions/: left as it is", + link.display(), + to.display() + )); + continue; + } + Err(_) if fs::symlink_metadata(&link).is_ok() => { + notes.push(format!( + "{} is not a file of a release: left as it is", + link.display() + )); + continue; + } + Err(_) => {} + } + relink(&link, &target)?; + } + for entry in fs::read_dir(&self.root)?.flatten() { + let name = entry.file_name().to_string_lossy().to_string(); + if names.contains(&name) { + continue; + } + let path = entry.path(); + match fs::read_link(&path) { + Ok(to) if to.starts_with(&through) && !path.exists() => fs::remove_file(&path) + .with_context(|| format!("removing {}", path.display()))?, + _ if placed_by_script(&path, &name) => remove_placed(&path)?, + _ => {} + } + } + Ok(notes) + } + + /// Removes every version but the one in use, the one before it, one + /// holding a `keep` file and one a server still runs on. + fn prune(&self) -> Result> { + let (current, previous) = (self.current(), self.previous()); + let mut gone = Vec::new(); + for v in self.on_disk() { + let dir = self.version_dir(&v); + if Some(&v) == current.as_ref() + || Some(&v) == previous.as_ref() + || dir.join(KEEP).exists() + || serving(&dir) + { + continue; + } + fs::remove_dir_all(&dir).with_context(|| format!("removing {}", dir.display()))?; + gone.push(v); + } + Ok(gone) + } +} + +/// Whether `path` in the root is the install script's earlier copy of a +/// release file: a file or directory of a release's name, or a link of one to +/// a file beside it (the library's versioned names). +fn placed_by_script(path: &Path, name: &str) -> bool { + is_bundle_file(name) + && match fs::read_link(path) { + Ok(to) => to.components().count() == 1 && !to.starts_with("versions"), + Err(_) => fs::symlink_metadata(path).is_ok(), + } +} + +fn remove_placed(path: &Path) -> Result<()> { + let is_dir = fs::symlink_metadata(path).is_ok_and(|m| m.is_dir()); + match is_dir { + true => fs::remove_dir_all(path), + false => fs::remove_file(path), + } + .with_context(|| format!("removing {}", path.display())) +} + +/// `link` made to point at `target`, in one rename. +fn relink(link: &Path, target: &Path) -> Result<()> { + let name = link.file_name().and_then(|n| n.to_str()).unwrap_or("link"); + let tmp = link.with_file_name(format!(".{name}.tmp")); + let _ = fs::remove_file(&tmp); + std::os::unix::fs::symlink(target, &tmp) + .with_context(|| format!("creating {}", tmp.display()))?; + fs::rename(&tmp, link).with_context(|| format!("replacing {}", link.display())) +} + +/// A server records the engine it runs on, so that no update removes or +/// replaces it while it runs: `basert serve` calls this just before it becomes +/// superfluid (same pid). A version of the layout holds the record itself; a +/// package's directory is the package manager's, so its servers are recorded +/// in the state directory instead, a file per pid naming the engine. +pub fn note_serving(engine: Option<&Path>) { + let Some(engine) = engine else { return }; + // By its real path, and a library named by its file as its directory: + // the version is found however `$BASERT_LIB` or `--basert-lib` names it. + let engine = canonical(engine); + let engine = match engine.is_dir() { + true => engine, + false => match engine.parent() { + Some(dir) => dir.to_path_buf(), + None => return, + }, + }; + let pid = std::process::id().to_string(); + let in_layout = engine + .parent() + .and_then(Path::file_name) + .is_some_and(|n| n == "versions"); + let (dir, text) = match in_layout { + true => (engine.join(SERVING), String::new()), + false => match serving_records() { + Some(records) => (records, format!("{}\n", engine.display())), + None => return, + }, + }; + if fs::create_dir_all(&dir).is_ok() { + let _ = fs::write(dir.join(pid), text); + } +} + +/// Where `basert serve` records the servers on a package's engine. +fn serving_records() -> Option { + crate::serve::state_dir().ok().map(|d| d.join("serving")) +} + +fn canonical(path: &Path) -> PathBuf { + fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) +} + +/// The live servers `records` names on the engine in `dir`; the records of +/// servers that stopped are removed. +fn servers_recorded(records: &Path, dir: &Path) -> Vec { + let dir = canonical(dir); + let mut pids = Vec::new(); + for entry in fs::read_dir(records).into_iter().flatten().flatten() { + let pid = entry + .file_name() + .to_str() + .and_then(|n| n.parse::().ok()); + match pid { + Some(pid) if alive(pid) => { + let on = fs::read_to_string(entry.path()).unwrap_or_default(); + if Path::new(on.trim()) == dir { + pids.push(pid); + } + } + _ => { + let _ = fs::remove_file(entry.path()); + } + } + } + pids +} + +/// Whether a server still runs on the version in `dir`; the records of +/// servers that stopped are removed. +fn serving(dir: &Path) -> bool { + !servers_on_version(dir).is_empty() +} + +/// The live servers recorded on the version in `dir`; the records of servers +/// that stopped are removed. +fn servers_on_version(dir: &Path) -> Vec { + let mut pids = Vec::new(); + for entry in fs::read_dir(dir.join(SERVING)) + .into_iter() + .flatten() + .flatten() + { + match entry + .file_name() + .to_str() + .and_then(|n| n.parse::().ok()) + { + Some(pid) if alive(pid) => pids.push(pid), + _ => { + let _ = fs::remove_file(entry.path()); + } + } + } + pids +} + +/// The servers on the engine in `root`, the install script's copy from before +/// the layout, whose files moving it under the layout turns into links: a +/// server started from it reads its engine from `root` by that path, so its +/// next model would load the new release's. Those `basert serve` recorded on +/// it, and superfluid and the runtime tools run from it (not the launcher, +/// which loads no engine, and runs this). +fn servers_on_flat(root: &Path) -> Vec { + let on_engine = |exe: &Path| { + exe.file_name() + .and_then(|n| n.to_str()) + .is_some_and(|n| n.starts_with("superfluid") || n.starts_with("basert-")) + }; + let mut pids = processes_running(root, &on_engine); + if let Some(records) = serving_records() { + for pid in servers_recorded(&records, root) { + if !pids.contains(&pid) { + pids.push(pid); + } + } + } + pids +} + +/// That no server runs on the flat copy in `root` (see [`servers_on_flat`]). +fn flat_unused(root: &Path) -> Result<()> { + let pids = servers_on_flat(root); + if pids.is_empty() { + return Ok(()); + } + bail!( + "a server runs on the engine in {} (pid {}), whose files this would replace: stop it, \ + then run this again", + root.display(), + pid_list(&pids) + ) +} + +fn pid_list(pids: &[i32]) -> String { + pids.iter() + .map(i32::to_string) + .collect::>() + .join(", ") +} + +fn alive(pid: i32) -> bool { + // SAFETY: signal 0 delivers nothing; it only asks whether the process exists. + let r = unsafe { libc::kill(pid, 0) }; + r == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) +} + +/// Only one `basert self` changes a layout at a time. +struct Lock(PathBuf); + +impl Lock { + fn take(layout: &Layout) -> Result { + let versions = layout.versions(); + fs::create_dir_all(&versions) + .with_context(|| format!("creating {}", versions.display()))?; + let path = versions.join(".lock"); + for _ in 0..2 { + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&path) + { + Ok(mut f) => { + use std::io::Write; + let _ = writeln!(f, "{}", std::process::id()); + return Ok(Lock(path)); + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + let holder = fs::read_to_string(&path) + .ok() + .and_then(|s| s.trim().parse::().ok()); + if let Some(pid) = holder.filter(|&p| alive(p)) { + bail!( + "another `basert self` (pid {pid}) is changing {}; run this again once it is done", + layout.root.display() + ); + } + let _ = fs::remove_file(&path); + } + Err(e) => return Err(e).with_context(|| format!("creating {}", path.display())), + } + } + bail!("could not take {}", path.display()) + } +} + +impl Drop for Lock { + fn drop(&mut self) { + let _ = fs::remove_file(&self.0); + } +} + +/// `from` copied to `to`, links as links. +fn copy_entry(from: &Path, to: &Path) -> Result<()> { + let meta = fs::symlink_metadata(from).with_context(|| format!("reading {}", from.display()))?; + if meta.file_type().is_symlink() { + std::os::unix::fs::symlink(fs::read_link(from)?, to)?; + } else if meta.is_dir() { + fs::create_dir_all(to)?; + for entry in fs::read_dir(from)?.flatten() { + copy_entry(&entry.path(), &to.join(entry.file_name()))?; + } + } else { + fs::copy(from, to).with_context(|| format!("copying {}", from.display()))?; + } + Ok(()) +} + +/// The install script's earlier copy, kept under the layout as the version to +/// go back to. It is copied beside the layout and moved in whole, so a copy +/// cut short is never taken for the version; the files in the root stay until +/// the links replace them. +fn keep_in_place(layout: &Layout, say: &mut dyn FnMut(&str)) -> Result { + let old = match layout.stamped() { + Some(v) => v, + None => launcher_version(&layout.root.join("basert"))?, + }; + let dest = layout.version_dir(&old); + if superfluid::is_executable(&dest.join("basert")) { + return Ok(old); + } + let staging = layout + .versions() + .join(".staging") + .join(format!("kept-{old}")); + let _ = fs::remove_dir_all(&staging); + let copied = (|| -> Result<()> { + fs::create_dir_all(&staging)?; + for entry in fs::read_dir(&layout.root)?.flatten() { + let name = entry.file_name().to_string_lossy().to_string(); + if is_bundle_file(&name) { + copy_entry(&entry.path(), &staging.join(&name))?; + } + } + fs::write(staging.join(STAMP), format!("v{old}\n"))?; + let _ = fs::remove_dir_all(&dest); + fs::rename(&staging, &dest) + .with_context(|| format!("{} -> {}", staging.display(), dest.display())) + })(); + if copied.is_err() { + let _ = fs::remove_dir_all(&staging); + } + copied?; + say(&format!( + "the release in place in {} ({old}) is kept as {}", + layout.root.display(), + dest.display() + )); + Ok(old) +} + +/// Download `release`, check its SHA-256 against the one it publishes, unpack +/// it, check its launcher runs and is that version, and move it in as +/// `versions/`. A complete copy already there is used as it is. Whether +/// anything was downloaded. +fn place(layout: &Layout, release: &Release, say: &mut dyn FnMut(&str)) -> Result { + let dest = layout.version_dir(&release.version); + let stamped = fs::read_to_string(dest.join(STAMP)).unwrap_or_default(); + if superfluid::is_executable(&dest.join("basert")) && stamped.trim() == release.tag { + return Ok(false); + } + // Another copy is there (cut short, or another build of the tag): a server + // on it reads its engine by that path. + let pids = servers_on_version(&dest); + if !pids.is_empty() { + bail!( + "{} holds another copy of basert {} and a server runs on it (pid {}): stop it, \ + then run this again", + dest.display(), + release.version, + pid_list(&pids) + ); + } + let staging = layout.versions().join(".staging"); + fs::create_dir_all(&staging).with_context(|| format!("creating {}", staging.display()))?; + // Beside the layout, so the final rename stays on one filesystem. + let stage = tempfile::Builder::new() + .prefix("new-") + .tempdir_in(&staging) + .with_context(|| format!("creating a staging directory in {}", staging.display()))?; + let tarball = stage.path().join(&release.tarball); + let checksum = stage.path().join(format!("{}.sha256", release.tarball)); + say(&format!( + "downloading basert {} ({}) from {}", + release.version, release.tarball, release.tarball_url + )); + superfluid::download(&release.tarball_url, &tarball)?; + superfluid::download(&release.checksum_url, &checksum)?; + superfluid::check_sha256(&tarball, &checksum)?; + let unpacked = stage.path().join("unpacked"); + fs::create_dir(&unpacked)?; + superfluid::untar(&tarball, &unpacked)?; + let said = launcher_version(&unpacked.join("basert")) + .with_context(|| format!("{} holds no basert launcher that runs", release.tarball))?; + if said != release.version { + bail!( + "{}: its launcher says it is {said}, not {}", + release.tarball, + release.version + ); + } + fs::write(unpacked.join(STAMP), format!("{}\n", release.tag))?; + if dest.exists() { + fs::remove_dir_all(&dest).with_context(|| format!("removing {}", dest.display()))?; + } + fs::rename(&unpacked, &dest) + .with_context(|| format!("moving the release into {}", dest.display()))?; + Ok(true) +} + +// --- superfluid -------------------------------------------------------------- + +/// What superfluid needs for an engine, found before anything changes. +enum Pairing { + /// None is installed: `basert serve` installs one at its first start. + NotInstalled, + /// The installed one serves the engine (and stays). + Serves, + /// A release, downloaded and asked, to take the installed one's place. + Replace { + installed: superfluid::Install, + staged: superfluid::Staged, + /// This install's turn among the installs into that directory, + /// kept until the swap. + turn: Option, + }, +} + +/// What superfluid needs to serve the engine in `engine`, whose ABI number is +/// `abi`. When the installed one cannot, the newest release that states the +/// number is downloaded and asked, whatever its order (going back to an older +/// basert may take an older superfluid); that no release can is an error. With +/// `forward`, a newer release that serves it is taken even when the installed +/// one can. Nothing is replaced under a running server. +fn pair_superfluid( + engine: &Path, + abi: Option, + forward: bool, + say: &mut dyn FnMut(&str), +) -> Result { + let Some(installed) = superfluid::Install::locate()? else { + return Ok(Pairing::NotInstalled); + }; + installed.claim(); + // Installs into it take turns; what is there is asked once this one's + // turn has come. + let turn = installed.take_turn(say)?; + let home = crate::serve::superfluid_home().ok(); + let have = installed + .installed_release() + .unwrap_or_else(|| "of a release it does not say".to_string()); + let ready = installed.probe(Some(engine), home.as_deref()).ready; + if ready.is_ok() && !forward { + return Ok(Pairing::Serves); + } + let found = match abi { + Some(abi) => superfluid::newest_for(abi).map_err(|e| format!("{e:#}")), + None => Err("its engine states no ABI number".to_string()), + }; + let release = match (&ready, found) { + (Err(_), Ok(Some(release))) => release, + (Ok(()), Ok(Some(release))) if superfluid::newer(Some(&release.tag), &have) => release, + (Ok(()), _) => return Ok(Pairing::Serves), + (Err(why), Ok(None)) => bail!( + "superfluid {have} at {} cannot serve this engine ({why}), and no superfluid release \ + states its ABI number {} yet", + installed.root.display(), + abi.unwrap_or_default() + ), + (Err(why), Err(lookup)) => bail!( + "superfluid {have} at {} cannot serve this engine ({why}), and no release that can \ + was found ({lookup})", + installed.root.display() + ), + }; + let tag = release.tag.clone(); + let busy = running_from(&installed.root); + if !busy.is_empty() { + let pids = busy + .iter() + .map(i32::to_string) + .collect::>() + .join(", "); + if let Err(why) = &ready { + bail!( + "superfluid {have} at {} cannot serve this engine ({why}) and {tag} can, but a \ + server runs from it (pid {pids}): stop it and run this again", + installed.root.display() + ); + } + say(&format!( + "superfluid {tag} is out, but a server runs from {} (pid {pids}): {have} stays until the next update", + installed.root.display() + )); + return Ok(Pairing::Serves); + } + if !installed.writable() { + if let Err(why) = &ready { + bail!( + "superfluid {have} at {} cannot serve this engine ({why}), and that directory \ + cannot be written to put {tag} in its place", + installed.root.display() + ); + } + return Ok(Pairing::Serves); + } + let staged = installed.stage(&superfluid::Source::GitHub, &release, say)?; + match staged.fresh.probe(Some(engine), home.as_deref()).ready { + // Replaced, its directory goes whole: one that holds more than + // superfluid (a prefix `$BASERT_SUPERFLUID_DIR` names) stays. + Ok(()) => match (installed.replaceable(&staged.fresh.root), &ready) { + (Ok(()), _) => Ok(Pairing::Replace { + installed, + staged, + turn, + }), + (Err(e), Ok(())) => { + say(&format!("superfluid {tag} is out, but {have} stays: {e:#}")); + Ok(Pairing::Serves) + } + (Err(e), Err(why)) => bail!( + "superfluid {have} at {} cannot serve this engine ({why}) and {tag} can, but {e:#}", + installed.root.display() + ), + }, + Err(why) if ready.is_err() => { + bail!("superfluid {tag} cannot serve this engine either: {why}") + } + Err(why) => { + say(&format!( + "superfluid {tag} cannot serve this engine ({why}): {have} stays" + )); + Ok(Pairing::Serves) + } + } +} + +fn apply(pairing: Pairing, say: &mut dyn FnMut(&str)) -> Result<()> { + match pairing { + Pairing::Replace { + installed, + staged, + turn, + } => { + let swapped = installed.swap_in(staged, say); + drop(turn); + swapped + } + Pairing::NotInstalled | Pairing::Serves => Ok(()), + } +} + +// --- the commands -------------------------------------------------------------- + +/// The layout the running basert belongs to, and whether it is the install +/// script's earlier, in-place one; a copy installed otherwise is an error that +/// says how to update it. +fn own_layout() -> Result<(Layout, bool)> { + let exe = this_exe()?; + match installed_as(&exe) { + Installed::Managed { root, .. } => Ok((Layout { root }, false)), + Installed::InPlace { root } => Ok((Layout { root }, true)), + Installed::Other { exe, how, .. } => bail!( + "{} was not installed by the install script, so `basert self` does not change it; {how}", + exe.display() + ), + } +} + +/// `self update`. +fn update( + version: Option<&str>, + dry_run: bool, + yes: bool, + say: &mut dyn FnMut(&str), +) -> Result<()> { + let exe = this_exe()?; + if let Installed::Other { .. } = installed_as(&exe) { + return update_by_manager(&exe, version, dry_run, yes, say); + } + let (layout, in_place) = own_layout()?; + let platform = platform()?; + let _lock = match dry_run { + true => None, + false => Some(Lock::take(&layout)?), + }; + let found = releases_in(&listing()?, platform); + let chosen = match version { + Some(want) => { + let want = want.trim_start_matches('v'); + found.iter().find(|r| r.version == want).with_context(|| { + format!( + "no release {want} for {platform} (the newest is {})", + found.first().map_or("none", |r| r.version.as_str()) + ) + })? + } + None => found.first().with_context(|| { + format!("the release listing names no basert release for {platform}") + })?, + }; + let v = chosen.version.as_str(); + let here = match in_place { + true => layout.stamped(), + false => layout.current(), + } + .unwrap_or_else(|| VERSION.to_string()); + let engine_here = match in_place { + true => layout.root.clone(), + false => layout.version_dir(&here), + }; + + if v == here { + say(&format!( + "basert {here} is in use, {}", + if version.is_some() { + "the release named" + } else { + "the newest release" + } + )); + if dry_run { + return Ok(()); + } + // superfluid as well: the newest release that serves this engine. + let pairing = + pair_superfluid(&engine_here, abi_in(&engine_here).or(chosen.abi), true, say)?; + return apply(pairing, say); + } + if version.is_none() && version_key(v) < version_key(&here) { + say(&format!( + "basert {here} is in use, newer than the newest release {v}: nothing is changed (`--version {v}` goes to it)" + )); + return Ok(()); + } + let on_disk = layout.launcher(v).is_file(); + say(&format!( + "basert {here} -> {v} (published {}{})", + chosen.published.split('T').next().unwrap_or("?"), + if on_disk { ", already here" } else { "" } + )); + if dry_run { + say("dry run: nothing fetched or changed"); + return Ok(()); + } + + let kept = match in_place { + true => { + flat_unused(&layout.root)?; + Some(keep_in_place(&layout, say)?) + } + false => None, + }; + let placed = place(&layout, chosen, say)?; + let engine = layout.version_dir(v); + let pairing = match pair_superfluid(&engine, abi_in(&engine).or(chosen.abi), true, say) { + Ok(pairing) => pairing, + Err(e) => { + if placed { + let _ = fs::remove_dir_all(&engine); + } + return Err(e.context(format!( + "basert {v} was not made the one in use; {here} stays" + ))); + } + }; + apply(pairing, say)?; + switch(&layout, v, &chosen.tag, kept.as_deref(), say) +} + +/// The layout's `current` turned to `v`, the root's stamp and links after it, +/// older versions pruned; `kept` is the in-place copy just moved under it. +fn switch( + layout: &Layout, + v: &str, + tag: &str, + kept: Option<&str>, + say: &mut dyn FnMut(&str), +) -> Result<()> { + layout.set_current(v)?; + if let Some(old) = kept.filter(|old| *old != v) { + layout.set_previous(old)?; + } + layout.stamp(tag)?; + for note in layout.link()? { + say(¬e); + } + println!( + "basert {v} is in use ({})", + layout.versions().join("current").display() + ); + if let Some(prev) = layout.previous() { + say(&format!( + "{prev} stays installed: `basert self use {prev}` goes back to it" + )); + } + match layout.prune() { + Ok(gone) if !gone.is_empty() => say(&format!("removed {}", gone.join(", "))), + Ok(_) => {} + Err(e) => say(&format!( + "an older version was not removed ({e:#}); the next update tries again" + )), + } + say("a server already running keeps the engine it started with; starting it again uses this one"); + Ok(()) +} + +/// `self use `. +fn use_version(v: &str, say: &mut dyn FnMut(&str)) -> Result<()> { + let (layout, in_place) = own_layout()?; + if in_place { + bail!( + "basert {} is installed in place in {}, with no other version beside it \ + (`basert self update` moves it under versions/)", + layout.stamped().unwrap_or_else(|| VERSION.to_string()), + layout.root.display() + ); + } + let v = v.trim_start_matches('v'); + let _lock = Lock::take(&layout)?; + if !layout.launcher(v).is_file() { + bail!("{v} is not installed here (`basert self list` shows what is)"); + } + if layout.current().as_deref() == Some(v) { + println!("basert {v} is in use"); + return Ok(()); + } + let engine = layout.version_dir(v); + let pairing = pair_superfluid(&engine, abi_in(&engine), false, say) + .with_context(|| format!("basert {v} was not made the one in use"))?; + apply(pairing, say)?; + let tag = fs::read_to_string(engine.join(STAMP)) + .map(|t| t.trim().to_string()) + .unwrap_or_else(|_| format!("v{v}")); + switch(&layout, v, &tag, None, say) +} + +/// `self install --from `: the install script's last step. +fn install_from(from: &Path, root: Option, say: &mut dyn FnMut(&str)) -> Result<()> { + let root = match root { + Some(root) => root, + None => match std::env::var_os("BASERT_INSTALL_DIR").filter(|d| !d.is_empty()) { + Some(dir) => PathBuf::from(dir), + None => dirs::home_dir() + .context("no home directory")? + .join(".basert"), + }, + }; + let layout = Layout { root }; + let v = launcher_version(&from.join("basert"))?; + let tag = format!("v{v}"); + let _lock = Lock::take(&layout)?; + let in_place = fs::symlink_metadata(layout.root.join("basert")).is_ok_and(|m| m.is_file()) + && layout.stamped().is_some(); + let kept = match in_place { + true => { + flat_unused(&layout.root)?; + Some(keep_in_place(&layout, say)?) + } + false => None, + }; + let dest = layout.version_dir(&v); + // A server on that version reads its engine by its path, and its record + // is in it: it stays, and is made the one in use. + let pids = servers_on_version(&dest); + if !pids.is_empty() && superfluid::is_executable(&dest.join("basert")) { + say(&format!( + "basert {v} in {} stays as it is: a server runs on it (pid {}); {} is not placed", + dest.display(), + pid_list(&pids), + from.display() + )); + return switch(&layout, &v, &tag, kept.as_deref(), say); + } + let aside = layout + .versions() + .join(".staging") + .join(format!("replaced-{v}")); + let _ = fs::remove_dir_all(&aside); + let had = dest.exists(); + if had { + fs::create_dir_all(aside.parent().unwrap_or(&layout.root))?; + fs::rename(&dest, &aside).with_context(|| format!("moving {} aside", dest.display()))?; + } + let moved = fs::rename(from, &dest).or_else(|_| { + // Another filesystem: copied, then the bundle removed. + copy_entry(from, &dest).and_then(|()| { + fs::remove_dir_all(from).with_context(|| format!("removing {}", from.display())) + }) + }); + if let Err(e) = moved { + let _ = fs::remove_dir_all(&dest); + if had { + let _ = fs::rename(&aside, &dest); + } + return Err(e).with_context(|| format!("placing {} as {}", from.display(), dest.display())); + } + let _ = fs::remove_dir_all(&aside); + fs::write(dest.join(STAMP), format!("{tag}\n"))?; + switch(&layout, &v, &tag, kept.as_deref(), say) +} + +/// `self list`. +fn list() -> Result<()> { + match installed_as(&this_exe()?) { + Installed::Managed { root, .. } => { + let layout = Layout { root }; + let (current, previous) = (layout.current(), layout.previous()); + println!("versions in {}:", layout.versions().display()); + for v in layout.on_disk() { + let dir = layout.version_dir(&v); + let mut notes = Vec::new(); + if Some(&v) == current.as_ref() { + notes.push("in use".to_string()); + } + if Some(&v) == previous.as_ref() { + notes.push(format!( + "the one before: `basert self use {v}` goes back to it" + )); + } + if dir.join(KEEP).exists() { + notes.push("kept".to_string()); + } + if serving(&dir) { + notes.push("a server runs on it".to_string()); + } + let mark = if Some(&v) == current.as_ref() { + "*" + } else { + " " + }; + match notes.is_empty() { + true => println!("{mark} {v}"), + false => println!("{mark} {v} ({})", notes.join("; ")), + } + } + } + Installed::InPlace { root } => println!( + "basert {} is installed in place in {} (the install script's earlier layout; \ + `basert self update` moves it under versions/)", + Layout { root: root.clone() } + .stamped() + .unwrap_or_else(|| VERSION.to_string()), + root.display() + ), + Installed::Other { exe, how, .. } => { + println!("basert {VERSION} at {}: {how}", exe.display()) + } + } + match superfluid::Install::locate()? { + Some(i) => println!( + "superfluid {} at {}", + i.installed_release() + .unwrap_or_else(|| "of a release it does not say".to_string()), + i.root.display() + ), + None => { + println!("superfluid: not installed (`basert serve` installs it at its first start)") + } + } + Ok(()) +} + +// --- the check, and the notice at a launch ------------------------------------ + +/// What the last look at the release listings found, kept in the state +/// directory for the notice. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +struct Check { + checked_at: u64, + /// The newest basert release past the hold-off whose engine a superfluid + /// release serves. + newest: Option, + /// The newest superfluid release past the hold-off that serves the engine + /// beside this basert. + superfluid: Option, + notified_at: u64, +} + +impl Check { + fn file() -> Option { + crate::serve::state_dir() + .ok() + .map(|d| d.join("update-check.json")) + } + + fn read(path: &Path) -> Option { + let v: Value = serde_json::from_str(&fs::read_to_string(path).ok()?).ok()?; + Some(Check { + checked_at: v["checked_at"].as_u64().unwrap_or(0), + newest: v["newest"].as_str().map(str::to_string), + superfluid: v["superfluid"].as_str().map(str::to_string), + notified_at: v["notified_at"].as_u64().unwrap_or(0), + }) + } + + fn write(&self, path: &Path) -> Result<()> { + if let Some(dir) = path.parent() { + fs::create_dir_all(dir)?; + } + let v = json!({ + "checked_at": self.checked_at, + "newest": self.newest, + "superfluid": self.superfluid, + "notified_at": self.notified_at, + }); + fs::write(path, format!("{v:#}\n")).with_context(|| format!("writing {}", path.display())) + } + + /// The line a launch prints for what this found, if any: `installed` is + /// how this basert was installed, `superfluid_have` the release of the + /// superfluid beside it. + fn line(&self, installed: &Installed, superfluid_have: Option<&str>) -> Option { + let ours = matches!( + installed, + Installed::Managed { .. } | Installed::InPlace { .. } + ); + if let Some(n) = self + .newest + .as_deref() + .filter(|n| version_key(n) > version_key(VERSION)) + { + let command = match installed { + Installed::Other { command, .. } => command.as_deref()?, + _ => "basert self update", + }; + return Some(format!( + "basert {n} is available (this is {VERSION}): `{command}` installs it; {NO_CHECK_ENV}=1 silences this" + )); + } + let sf = self.superfluid.as_deref()?; + let have = superfluid_have?; + if !superfluid::newer(Some(sf), have) { + return None; + } + let command = if ours { + "basert self update" + } else { + "basert superfluid install" + }; + Some(format!( + "superfluid {sf}, the server `basert serve` runs, is available (installed: {have}): \ + `{command}` installs it; {NO_CHECK_ENV}=1 silences this" + )) + } +} + +fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +/// Seconds since the epoch of a timestamp of GitHub's (`2026-10-27T06:42:10Z`). +fn epoch_of(stamp: &str) -> Option { + let (date, time) = stamp.trim_end_matches('Z').split_once('T')?; + let mut d = date.split('-').map(|p| p.parse::().ok()); + let (y, m, day) = (d.next()??, d.next()??, d.next()??); + let mut t = time.split(':').map(|p| p.parse::().ok()); + let (h, min, s) = (t.next()??, t.next()??, t.next()??); + // Days from 1970-01-01 to the date (Howard Hinnant's days_from_civil). + let y = if m <= 2 { y - 1 } else { y }; + let era = if y >= 0 { y } else { y - 399 } / 400; + let yoe = y - era * 400; + let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) + 2) / 5 + day - 1; + let days = era * 146_097 + yoe * 365 + yoe / 4 - yoe / 100 + doy - 719_468; + u64::try_from(days * 86_400 + h * 3_600 + min * 60 + s).ok() +} + +/// Whether a release published at `stamp` is past the hold-off at `now`. +fn settled(stamp: &str, now: u64) -> bool { + epoch_of(stamp).is_none_or(|at| now.saturating_sub(at) >= HOLD_OFF.as_secs()) +} + +/// What the listings say now: basert's releases, newest first, each with +/// whether a superfluid release serves its engine, and superfluid's listing +/// when it could be read. +struct Found { + releases: Vec<(Release, bool)>, + superfluid: Option, +} + +fn look() -> Result { + let releases = releases_in(&listing()?, platform()?); + let sf_listing = superfluid::releases().ok(); + let target = superfluid::target().ok(); + let releases = releases + .into_iter() + .map(|r| { + let served = match (r.abi, &sf_listing, target) { + (Some(abi), Some(l), Some(t)) => superfluid::newest_in(l, t, abi).is_some(), + _ => true, + }; + (r, served) + }) + .collect(); + Ok(Found { + releases, + superfluid: sf_listing, + }) +} + +impl Found { + /// The newest release a `self update` would go to, past the hold-off when + /// asked. + fn newest(&self, hold_off: Option) -> Option<&Release> { + self.releases + .iter() + .filter(|(r, served)| *served && hold_off.is_none_or(|now| settled(&r.published, now))) + .map(|(r, _)| r) + .next() + } + + /// The newest superfluid release that serves this basert's engine. + fn newest_superfluid(&self, hold_off: Option) -> Option { + let listing = self.superfluid.as_ref()?; + let keep = |r: &Value| { + hold_off.is_none_or(|now| settled(r["published_at"].as_str().unwrap_or_default(), now)) + }; + superfluid::newest_where( + listing, + superfluid::target().ok()?, + superfluid::ENGINE_ABI, + &keep, + ) + .map(|r| r.tag) + } + + fn record(&self, path: &Path) { + let now = now(); + let mut check = Check::read(path).unwrap_or_default(); + check.checked_at = now; + check.newest = self.newest(Some(now)).map(|r| r.version.clone()); + check.superfluid = self.newest_superfluid(Some(now)); + let _ = check.write(path); + } +} + +/// `self check`. +fn check() -> Result<()> { + let found = look()?; + if let Some(path) = Check::file() { + found.record(&path); + } + let installed = installed_as(&this_exe()?); + let command = match &installed { + Installed::Other { command, .. } => command.clone(), + _ => Some("basert self update".to_string()), + }; + match found.releases.first() { + Some((r, served)) => { + let after = match (version_key(&r.version) > version_key(VERSION), served) { + (true, true) => command + .map(|c| format!(": `{c}` installs it")) + .unwrap_or_default(), + (_, false) => format!( + "; it states engine ABI {}, which no superfluid release serves yet", + r.abi.unwrap_or_default() + ), + (false, true) => String::new(), + }; + println!( + "basert {VERSION} is this one; the newest release is {} (published {}){after}", + r.version, + r.published.split('T').next().unwrap_or("?"), + ); + } + None => { + println!("basert {VERSION} is this one; the listing names no release for this machine") + } + } + let newest_sf = found.newest_superfluid(None); + match superfluid::Install::locate()? { + Some(i) => println!( + "superfluid {} at {}; the newest release that serves this engine is {}", + i.installed_release() + .unwrap_or_else(|| "of a release it does not say".to_string()), + i.root.display(), + newest_sf.as_deref().unwrap_or("not known") + ), + None => println!( + "superfluid: not installed; the newest release that serves this engine is {}", + newest_sf.as_deref().unwrap_or("not known") + ), + } + Ok(()) +} + +/// `self check --record`: the look a launch started, recorded for the notice. +fn record() { + let Some(path) = Check::file() else { return }; + // A failed look is tried again a day later: the launch that started this + // claimed the day. + if let Ok(found) = look() { + found.record(&path); + } +} + +/// `self check --background`: `--record` in a process of its own, which +/// outlives this one (and the launch that ran this, which waits only for this). +fn background() -> Result<()> { + use std::os::unix::process::CommandExt; + Command::new(this_exe()?) + .args(["self", "check", "--record"]) + .env(NO_CHECK_ENV, "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + // Its own process group: a Ctrl-C at the terminal is not for it. + .process_group(0) + .spawn() + .context("starting the update check")?; + Ok(()) +} + +fn check_disabled() -> bool { + std::env::var_os(NO_CHECK_ENV).is_some_and(|v| !v.is_empty() && v != "0") +} + +/// What a launch does before its command, on a terminal: start the day's look +/// for a newer release when it is due, and print the line for one found +/// before, once a day at most. `first` is the command's first argument. +pub fn at_launch(first: Option<&str>) { + if matches!( + first, + None | Some("self" | "-V" | "--version" | "-h" | "--help" | "help") + ) { + return; + } + if check_disabled() || !std::io::stderr().is_terminal() { + return; + } + let Ok(exe) = this_exe() else { return }; + let installed = installed_as(&exe); + if matches!(installed, Installed::Other { command: None, .. }) { + return; + } + let Some(path) = Check::file() else { return }; + let mut check = Check::read(&path).unwrap_or_default(); + let now = now(); + if now.saturating_sub(check.checked_at) >= CHECK_EVERY.as_secs() { + // Claimed before it starts, so the commands that follow start none. + check.checked_at = now; + let _ = check.write(&path); + let _ = Command::new(&exe) + .args(["self", "check", "--background"]) + .env(NO_CHECK_ENV, "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); + } + if now.saturating_sub(check.notified_at) < CHECK_EVERY.as_secs() { + return; + } + let have = superfluid::Install::locate() + .ok() + .flatten() + .and_then(|i| i.installed_release()); + if let Some(line) = check.line(&installed, have.as_deref()) { + eprintln!("{line}"); + // Read again: the look may have recorded since. + let mut latest = Check::read(&path).unwrap_or(check); + latest.notified_at = now; + let _ = latest.write(&path); + } +} + +// --- processes ------------------------------------------------------------------ + +/// The other processes running an executable under `dir`, by pid. +pub(crate) fn running_from(dir: &Path) -> Vec { + processes_running(dir, &|_| true) +} + +/// [`running_from`], among the executables `keep` accepts. +fn processes_running(dir: &Path, keep: &dyn Fn(&Path) -> bool) -> Vec { + let Ok(dir) = fs::canonicalize(dir) else { + return Vec::new(); + }; + other_processes() + .into_iter() + .filter(|(_, exe)| exe.starts_with(&dir) && keep(exe)) + .map(|(p, _)| p) + .collect() +} + +/// The other processes on this machine, each with its executable as a real path. +#[cfg(target_os = "linux")] +fn other_processes() -> Vec<(i32, PathBuf)> { + let me = std::process::id() as i32; + fs::read_dir("/proc") + .map(|rd| { + rd.flatten() + .filter_map(|e| { + let pid = e.file_name().to_str()?.parse::().ok()?; + (pid != me).then(|| (pid, e.path())) + }) + .filter_map(|(p, dir)| Some((p, fs::read_link(dir.join("exe")).ok()?))) + .collect() + }) + .unwrap_or_default() +} + +#[cfg(target_os = "macos")] +fn other_processes() -> Vec<(i32, PathBuf)> { + let me = std::process::id() as libc::c_int; + // SAFETY: with a null buffer, proc_listallpids only reports how many pids there are. + let n = unsafe { libc::proc_listallpids(std::ptr::null_mut(), 0) }; + if n <= 0 { + return Vec::new(); + } + let mut pids = vec![0 as libc::c_int; n as usize * 2 + 16]; + let bytes = (pids.len() * std::mem::size_of::()) as libc::c_int; + // SAFETY: the buffer is this function's, and its size in bytes is what is passed. + let got = unsafe { libc::proc_listallpids(pids.as_mut_ptr() as *mut libc::c_void, bytes) }; + pids.truncate(got.max(0) as usize); + pids.into_iter() + .filter(|&p| p > 0 && p != me) + .filter_map(|p| { + let mut buf = vec![0u8; 4096]; + // SAFETY: proc_pidpath writes at most the buffer's length, which it is told. + let len = unsafe { + libc::proc_pidpath(p, buf.as_mut_ptr() as *mut libc::c_void, buf.len() as u32) + }; + (len > 0).then(|| { + ( + p, + PathBuf::from(String::from_utf8_lossy(&buf[..len as usize]).to_string()), + ) + }) + }) + .collect() +} + +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn other_processes() -> Vec<(i32, PathBuf)> { + Vec::new() +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + fn executable(path: &Path, text: &str) { + use std::os::unix::fs::PermissionsExt; + fs::write(path, text).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); + } + + /// A release directory: a launcher script that says `version`, a library + /// and its link, a tool, the header stating `abi`. + fn release_dir(dir: &Path, version: &str, abi: u32) { + fs::create_dir_all(dir.join("include").join("baseRT")).unwrap(); + executable( + &dir.join("basert"), + &format!("#!/bin/sh\n[ \"$1\" = --version ] && echo \"basert {version}\"\n"), + ); + executable(&dir.join("basert-chat"), "#!/bin/sh\n"); + fs::write(dir.join(format!("libbaseRT.{version}.dylib")), version).unwrap(); + std::os::unix::fs::symlink( + format!("libbaseRT.{version}.dylib"), + dir.join("libbaseRT.dylib"), + ) + .unwrap(); + fs::write( + dir.join("include").join("baseRT").join("baseRT.h"), + format!("#pragma once\n#define BASERT_ABI_VERSION {abi}\n"), + ) + .unwrap(); + fs::write(dir.join(STAMP), format!("v{version}\n")).unwrap(); + } + + fn listed(tag: &str, published: &str, names: &[&str]) -> Value { + json!({ + "tag_name": tag, + "published_at": published, + "draft": false, + "prerelease": false, + "assets": names.iter().map(|n| json!({"name": n, "browser_download_url": format!("https://x/{tag}/{n}")})).collect::>(), + }) + } + + #[test] + fn the_listing_gives_this_platforms_releases_newest_first_with_their_abi() { + let listing = json!([ + listed( + "v0.4.2", + "2026-10-27T06:42:10Z", + &[ + "basert-engine-abi-1.txt", + "basert-engine-macos-arm64-0.4.2.tar.gz", + "basert-engine-macos-arm64-0.4.2.tar.gz.sha256", + ] + ), + listed( + "v0.4.10", + "2026-12-01T00:00:00Z", + &[ + "basert-engine-abi-2.txt", + "basert-engine-macos-arm64-0.4.10.tar.gz", + "basert-engine-macos-arm64-0.4.10.tar.gz.sha256", + ] + ), + // No checksum: not a release to install. + listed( + "v0.5.0", + "2026-12-02T00:00:00Z", + &["basert-engine-macos-arm64-0.5.0.tar.gz"] + ), + // Another platform only. + listed( + "v0.4.11", + "2026-12-03T00:00:00Z", + &[ + "basert-engine-linux-arm64-cuda-0.4.11.tar.gz", + "basert-engine-linux-arm64-cuda-0.4.11.tar.gz.sha256", + ] + ), + // A staging tag. + listed( + "v0.4.12-staging", + "2026-12-04T00:00:00Z", + &[ + "basert-engine-macos-arm64-v0.4.12-staging.tar.gz", + "basert-engine-macos-arm64-v0.4.12-staging.tar.gz.sha256", + ] + ), + ]); + let found = releases_in(&listing, "macos-arm64"); + let versions: Vec<(&str, Option)> = + found.iter().map(|r| (r.version.as_str(), r.abi)).collect(); + assert_eq!(versions, [("0.4.10", Some(2)), ("0.4.2", Some(1))]); + assert_eq!(found[1].tag, "v0.4.2"); + assert_eq!( + found[1].tarball_url, + "https://x/v0.4.2/basert-engine-macos-arm64-0.4.2.tar.gz" + ); + let mut pre = listed( + "v0.4.13", + "2026-12-05T00:00:00Z", + &[ + "basert-engine-macos-arm64-0.4.13.tar.gz", + "basert-engine-macos-arm64-0.4.13.tar.gz.sha256", + ], + ); + pre["prerelease"] = json!(true); + assert!(releases_in(&json!([pre]), "macos-arm64").is_empty()); + } + + #[test] + fn how_a_launcher_was_installed_is_read_from_where_it_is() { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().join(".basert"); + release_dir(&root.join("versions").join("0.4.3"), "0.4.3", 1); + assert_eq!( + installed_as(&root.join("versions").join("0.4.3").join("basert")), + Installed::Managed { + root: fs::canonicalize(&root).unwrap(), + version: "0.4.3".into() + } + ); + // Through the root's link, as PATH finds it. + fs::create_dir_all(root.join("versions")).unwrap(); + std::os::unix::fs::symlink("0.4.3", root.join("versions").join("current")).unwrap(); + std::os::unix::fs::symlink("versions/current/basert", root.join("basert")).unwrap(); + assert!( + matches!(installed_as(&root.join("basert")), Installed::Managed { version, .. } if version == "0.4.3") + ); + + let flat = tmp.path().join("flat"); + release_dir(&flat, "0.4.2", 1); + assert_eq!( + installed_as(&flat.join("basert")), + Installed::InPlace { + root: fs::canonicalize(&flat).unwrap() + } + ); + + let pip = tmp + .path() + .join("site-packages") + .join("baseRT") + .join("_engine"); + release_dir(&pip, "0.4.2", 1); + fs::remove_file(pip.join(STAMP)).unwrap(); + match installed_as(&pip.join("basert")) { + Installed::Other { command, .. } => { + assert_eq!(command.as_deref(), Some("pip install --upgrade basert")) + } + other => panic!("{other:?}"), + } + // The package's own file decides, whatever the path says. + fs::write(pip.join(INSTALLED_BY), "uv: uv tool upgrade basert\n").unwrap(); + fs::write(pip.join(STAMP), "v0.4.2\n").unwrap(); + match installed_as(&pip.join("basert")) { + Installed::Other { command, how, .. } => { + assert_eq!(command.as_deref(), Some("uv tool upgrade basert")); + assert!(how.starts_with("uv installed it"), "{how}"); + } + other => panic!("{other:?}"), + } + + let build = tmp.path().join("tools").join("target").join("release"); + fs::create_dir_all(&build).unwrap(); + executable(&build.join("basert"), "#!/bin/sh\n"); + assert!(matches!( + installed_as(&build.join("basert")), + Installed::Other { command: None, .. } + )); + } + + #[test] + fn the_files_of_a_release_are_told_from_the_users_own() { + for name in [ + "basert", + "basert-chat", + "basert-benchmark-harness", + "baseRT_cuda_bench", + "libbaseRT.dylib", + "libbaseRT.so.0", + "baseRT.metallib", + "baseRT_tensor.metallib", + "baseRT_cuda_kernels.fatbin", + "include", + "LICENSE", + "NOTICE", + ] { + assert!(is_bundle_file(name), "{name}"); + } + for name in [ + "superfluid", + "versions", + ".release", + "models", + "notes.txt", + "baseRT.h", + ] { + assert!(!is_bundle_file(name), "{name}"); + } + } + + #[test] + fn an_in_place_copy_moves_under_the_layout_and_the_root_links_through_current() { + let tmp = TempDir::new().unwrap(); + let layout = Layout { + root: tmp.path().to_path_buf(), + }; + release_dir(&layout.root, "0.4.2", 1); + fs::create_dir_all(layout.root.join("superfluid").join("bin")).unwrap(); + fs::write(layout.root.join("notes.txt"), "mine").unwrap(); + let mut said = Vec::new(); + let old = keep_in_place(&layout, &mut |l| said.push(l.to_string())).unwrap(); + assert_eq!(old, "0.4.2"); + let kept = layout.version_dir("0.4.2"); + assert_eq!(launcher_version(&kept.join("basert")).unwrap(), "0.4.2"); + assert_eq!( + fs::read_link(kept.join("libbaseRT.dylib")).unwrap(), + Path::new("libbaseRT.0.4.2.dylib") + ); + assert!(!kept.join("superfluid").exists() && !kept.join("notes.txt").exists()); + // Nothing in the root has changed yet. + assert!(fs::symlink_metadata(layout.root.join("basert")) + .unwrap() + .is_file()); + + release_dir(&layout.version_dir("0.4.3"), "0.4.3", 1); + fs::remove_file(layout.version_dir("0.4.3").join("basert-chat")).unwrap(); + layout.set_current("0.4.3").unwrap(); + layout.set_previous(&old).unwrap(); + let notes = layout.link().unwrap(); + assert!(notes.is_empty(), "{notes:?}"); + assert_eq!( + fs::read_link(layout.root.join("basert")).unwrap(), + Path::new("versions/current/basert") + ); + assert_eq!( + launcher_version(&layout.root.join("basert")).unwrap(), + "0.4.3" + ); + assert!(fs::read_link(layout.root.join("include")).is_ok()); + // A file the in-place copy had and the version in use does not: kept + // under versions/0.4.2, gone from the root. + assert!(fs::symlink_metadata(layout.root.join("basert-chat")).is_err()); + assert!(kept.join("basert-chat").is_file()); + assert!(fs::symlink_metadata(layout.root.join("libbaseRT.0.4.2.dylib")).is_err()); + assert_eq!( + fs::read_to_string(layout.root.join("notes.txt")).unwrap(), + "mine" + ); + assert!(layout.root.join("superfluid").join("bin").is_dir()); + + // Back, then forward: the links follow, a dropped file's link goes. + layout.set_current("0.4.2").unwrap(); + layout.link().unwrap(); + assert!(fs::read_link(layout.root.join("basert-chat")).is_ok()); + assert_eq!(layout.previous().as_deref(), Some("0.4.3")); + layout.set_current("0.4.3").unwrap(); + layout.link().unwrap(); + assert!(fs::symlink_metadata(layout.root.join("basert-chat")).is_err()); + assert_eq!(layout.on_disk(), ["0.4.3", "0.4.2"]); + } + + #[test] + fn pruning_keeps_the_current_the_previous_a_kept_and_a_served_version() { + let tmp = TempDir::new().unwrap(); + let layout = Layout { + root: tmp.path().to_path_buf(), + }; + for v in ["0.4.0", "0.4.1", "0.4.2", "0.4.3", "0.4.4", "0.4.5"] { + release_dir(&layout.version_dir(v), v, 1); + } + fs::write(layout.version_dir("0.4.1").join(KEEP), "").unwrap(); + // A server on 0.4.2 (this test's own pid stands in for it); one on 0.4.0 that stopped. + fs::create_dir_all(layout.version_dir("0.4.2").join(SERVING)).unwrap(); + fs::write( + layout + .version_dir("0.4.2") + .join(SERVING) + .join(std::process::id().to_string()), + "", + ) + .unwrap(); + fs::create_dir_all(layout.version_dir("0.4.0").join(SERVING)).unwrap(); + fs::write( + layout.version_dir("0.4.0").join(SERVING).join("2147483646"), + "", + ) + .unwrap(); + layout.set_current("0.4.4").unwrap(); + layout.set_current("0.4.5").unwrap(); + let gone = layout.prune().unwrap(); + assert_eq!(gone, ["0.4.3", "0.4.0"]); + assert_eq!(layout.on_disk(), ["0.4.5", "0.4.4", "0.4.2", "0.4.1"]); + } + + #[test] + fn a_server_on_a_version_is_noted_in_it_and_nowhere_else() { + let tmp = TempDir::new().unwrap(); + let version = tmp.path().join("versions").join("0.4.3"); + fs::create_dir_all(&version).unwrap(); + note_serving(Some(&version)); + assert!(version + .join(SERVING) + .join(std::process::id().to_string()) + .is_file()); + assert!(serving(&version)); + let flat = tmp.path().join("flat"); + fs::create_dir_all(&flat).unwrap(); + note_serving(Some(&flat)); + assert!(!flat.join(SERVING).exists()); + } + + #[test] + fn a_server_on_a_library_named_by_its_file_is_noted_in_its_version() { + let tmp = TempDir::new().unwrap(); + let version = tmp.path().join("versions").join("0.4.4"); + fs::create_dir_all(&version).unwrap(); + fs::write(version.join("libbaseRT.0.4.4.dylib"), "").unwrap(); + // What `$BASERT_LIB` may name, through the root's link as well. + std::os::unix::fs::symlink( + Path::new("versions/0.4.4/libbaseRT.0.4.4.dylib"), + tmp.path().join("libbaseRT.dylib"), + ) + .unwrap(); + note_serving(Some(&tmp.path().join("libbaseRT.dylib"))); + assert!(serving(&version)); + } + + #[test] + fn a_second_changer_waits_its_turn_and_a_dead_ones_lock_is_taken() { + let tmp = TempDir::new().unwrap(); + let layout = Layout { + root: tmp.path().to_path_buf(), + }; + let held = Lock::take(&layout).unwrap(); + let err = Lock::take(&layout).err().unwrap().to_string(); + assert!( + err.contains(&format!("pid {}", std::process::id())), + "{err}" + ); + drop(held); + fs::write(layout.versions().join(".lock"), "2147483646\n").unwrap(); + assert!(Lock::take(&layout).is_ok()); + } + + #[test] + fn github_timestamps_read_as_the_epoch() { + assert_eq!(epoch_of("1970-01-01T00:00:00Z"), Some(0)); + assert_eq!(epoch_of("2026-10-27T06:42:10Z"), Some(1_793_083_330)); + assert_eq!(epoch_of("2000-03-01T00:00:00Z"), Some(951_868_800)); + assert_eq!(epoch_of("soon"), None); + let at = epoch_of("2026-10-27T06:42:10Z").unwrap(); + assert!(!settled("2026-10-27T06:42:10Z", at + 2 * 86_400)); + assert!(settled("2026-10-27T06:42:10Z", at + 3 * 86_400)); + assert!(settled("", at)); + } + + #[test] + fn the_notice_names_the_command_that_updates_this_copy() { + let ahead = format!("{}.{}", VERSION, 1); + let managed = Installed::Managed { + root: "/x".into(), + version: VERSION.into(), + }; + let brew = Installed::Other { + exe: "/opt/homebrew/Cellar/basert/1/libexec/basert".into(), + how: String::new(), + command: Some("brew upgrade basert".into()), + }; + let unknown = Installed::Other { + exe: "/x".into(), + how: String::new(), + command: None, + }; + let newer = Check { + newest: Some(ahead.clone()), + ..Check::default() + }; + let line = newer.line(&managed, None).unwrap(); + assert!( + line.starts_with(&format!( + "basert {ahead} is available (this is {VERSION}): `basert self update`" + )), + "{line}" + ); + assert!(newer + .line(&brew, None) + .unwrap() + .contains("`brew upgrade basert`")); + assert_eq!(newer.line(&unknown, None), None); + let same = Check { + newest: Some(VERSION.into()), + ..Check::default() + }; + assert_eq!(same.line(&managed, None), None); + + let sf = Check { + newest: Some(VERSION.into()), + superfluid: Some("v0.1.9".into()), + ..Check::default() + }; + let line = sf.line(&managed, Some("v0.1.5")).unwrap(); + assert!(line.starts_with("superfluid v0.1.9, the server `basert serve` runs, is available (installed: v0.1.5): `basert self update`"), "{line}"); + assert!(sf + .line(&brew, Some("v0.1.5")) + .unwrap() + .contains("`basert superfluid install`")); + assert_eq!(sf.line(&managed, Some("v0.1.9")), None); + assert_eq!(sf.line(&managed, None), None); + } + + #[test] + fn a_release_states_its_abi_in_its_header() { + let tmp = TempDir::new().unwrap(); + release_dir(tmp.path(), "0.4.3", 7); + assert_eq!(abi_in(tmp.path()), Some(7)); + assert_eq!(abi_in(&tmp.path().join("nowhere")), None); + } + + /// A launcher (a script) at `rel` under `root`, its real path. + fn launcher_at(root: &Path, rel: &str) -> PathBuf { + let dir = root.join(rel); + fs::create_dir_all(&dir).unwrap(); + executable(&dir.join("basert"), "#!/bin/sh\n"); + fs::canonicalize(dir.join("basert")).unwrap() + } + + #[test] + fn homebrew_is_run_by_the_brew_whose_cellar_holds_the_copy_unless_pinned() { + let tmp = TempDir::new().unwrap(); + let exe = launcher_at(tmp.path(), "brew/Cellar/basert/0.4.3/libexec"); + let named = Some(("Homebrew".to_string(), "brew upgrade basert".to_string())); + match delegate(&exe, named.clone()).unwrap() { + Delegate::Name { why, .. } => assert!(why.starts_with("there is no "), "{why}"), + other => panic!("{other:?}"), + } + fs::create_dir_all(tmp.path().join("brew/bin")).unwrap(); + executable(&tmp.path().join("brew/bin/brew"), "#!/bin/sh\n"); + let brew = fs::canonicalize(tmp.path().join("brew/bin/brew")).unwrap(); + assert_eq!( + delegate(&exe, named.clone()), + Some(Delegate::Run { + manager: "Homebrew".into(), + command: "brew upgrade basert".into(), + argv: vec![ + brew.display().to_string(), + "upgrade".into(), + "basert".into() + ], + path_first: None, + }) + ); + // Without the file: the Cellar in its path says Homebrew. + assert!(matches!(delegate(&exe, None), Some(Delegate::Run { .. }))); + fs::create_dir_all(tmp.path().join("brew/var/homebrew/pinned")).unwrap(); + fs::write(tmp.path().join("brew/var/homebrew/pinned/basert"), "").unwrap(); + match delegate(&exe, named).unwrap() { + Delegate::Name { why, .. } => { + assert!(why.contains("`brew unpin basert` first"), "{why}") + } + other => panic!("{other:?}"), + } + } + + #[test] + fn npm_is_run_for_a_global_copy_only_and_puts_its_own_bin_first() { + let tmp = TempDir::new().unwrap(); + let named = Some(( + "npm".to_string(), + "npm install --global @basecompute/basert@latest".to_string(), + )); + let project = launcher_at( + tmp.path(), + "app/node_modules/@basecompute/basert-darwin-arm64/engine", + ); + match delegate(&project, named.clone()).unwrap() { + Delegate::Name { why, command, .. } => { + assert!(why.contains("update it in that project"), "{why}"); + assert_eq!(command, "npm install @basecompute/basert@latest"); + } + other => panic!("{other:?}"), + } + let global = launcher_at( + tmp.path(), + "node/lib/node_modules/@basecompute/basert-darwin-arm64/engine", + ); + fs::create_dir_all(tmp.path().join("node/bin")).unwrap(); + executable(&tmp.path().join("node/bin/npm"), "#!/bin/sh\n"); + let bin = fs::canonicalize(tmp.path().join("node/bin")).unwrap(); + match delegate(&global, named).unwrap() { + Delegate::Run { + argv, path_first, .. + } => { + assert_eq!(argv[0], bin.join("npm").display().to_string()); + assert_eq!( + argv[1..], + ["install", "--global", "@basecompute/basert@latest"] + ); + assert_eq!(path_first, Some(bin)); + } + other => panic!("{other:?}"), + } + } + + #[test] + fn pip_is_named_with_its_environments_python_and_pipx_and_uv_are_told_apart() { + let tmp = TempDir::new().unwrap(); + let named = Some(( + "pip".to_string(), + "pip install --upgrade basert".to_string(), + )); + let env = tmp.path().join("venv"); + let exe = launcher_at(&env, "lib/python3.12/site-packages/baseRT/_engine"); + fs::create_dir_all(env.join("bin")).unwrap(); + executable(&env.join("bin/python"), "#!/bin/sh\n"); + let python = fs::canonicalize(env.join("bin/python")).unwrap(); + match delegate(&exe, named.clone()).unwrap() { + Delegate::Name { command, .. } => { + assert_eq!( + command, + format!("{} -m pip install --upgrade basert", python.display()) + ) + } + other => panic!("{other:?}"), + } + let pipx = launcher_at( + tmp.path(), + ".local/pipx/venvs/basert/lib/python3.12/site-packages/baseRT/_engine", + ); + let d = delegate(&pipx, named.clone()).unwrap(); + assert_eq!((d.manager(), d.command()), ("pipx", "pipx upgrade basert")); + let uv = launcher_at( + tmp.path(), + ".local/share/uv/tools/basert/lib/python3.12/site-packages/baseRT/_engine", + ); + let d = delegate(&uv, named).unwrap(); + assert_eq!((d.manager(), d.command()), ("uv", "uv tool upgrade basert")); + } + + #[test] + fn cargo_is_run_by_the_cargo_beside_the_copy() { + let tmp = TempDir::new().unwrap(); + let exe = launcher_at(tmp.path(), ".cargo/bin"); + executable(&tmp.path().join(".cargo/bin/cargo"), "#!/bin/sh\n"); + let bin = fs::canonicalize(tmp.path().join(".cargo/bin")).unwrap(); + assert_eq!( + delegate(&exe, None), + Some(Delegate::Run { + manager: "cargo".into(), + command: "cargo install basert-cli".into(), + argv: vec![ + bin.join("cargo").display().to_string(), + "install".into(), + "basert-cli".into() + ], + path_first: Some(bin), + }) + ); + // A manager basert does not run is named with the file's own command. + let d = delegate(&exe, Some(("conda".into(), "conda update basert".into()))).unwrap(); + assert!( + matches!(&d, Delegate::Name { command, .. } if command == "conda update basert"), + "{d:?}" + ); + } + + #[test] + fn a_server_recorded_on_a_package_engine_is_found_and_a_stopped_ones_record_goes() { + let tmp = TempDir::new().unwrap(); + let records = tmp.path().join("serving"); + let engine = tmp.path().join("libexec"); + fs::create_dir_all(&records).unwrap(); + fs::create_dir_all(&engine).unwrap(); + let me = std::process::id() as i32; + fs::write( + records.join(me.to_string()), + format!("{}\n", canonical(&engine).display()), + ) + .unwrap(); + fs::write( + records.join("2147483646"), + format!("{}\n", canonical(&engine).display()), + ) + .unwrap(); + assert_eq!(servers_recorded(&records, &engine), [me]); + assert!(!records.join("2147483646").exists()); + assert!(servers_recorded(&records, tmp.path()).is_empty()); + } +} diff --git a/base-convert/crates/base-convert/src/serve.rs b/base-convert/crates/base-convert/src/serve.rs index deadf53..0432502 100644 --- a/base-convert/crates/base-convert/src/serve.rs +++ b/base-convert/crates/base-convert/src/serve.rs @@ -94,7 +94,8 @@ Accepted so that existing command lines keep working, and no longer used: State (the request log, uploaded files) is kept under $BASERT_STATE_DIR, by default the serve/ directory beside the model cache, one directory per port. The engine library is found beside basert (or in ../lib); BASERT_LIB names -another directory. BASERT_SUPERFLUID_DIR names where superfluid is installed; +another directory. BASERT_SUPERFLUID_DIR names a directory of superfluid's +own to run and install it in (one holding other files is never replaced); BASERT_SUPERFLUID_RELEASES another release listing (a URL or a file); BASERT_SUPERFLUID_SOURCE a directory holding the tested release's tarball, read instead of GitHub; BASERT_SUPERFLUID_AUTO_INSTALL=0 forbids a download @@ -427,8 +428,11 @@ pub fn run(rest: &[String]) -> Result<()> { let parsed = parse(rest); if parsed == Ok(Parsed::Version) { println!("basert serve {}", env!("CARGO_PKG_VERSION")); - match superfluid::Install::locate()? { - Some(i) => println!( + // The superfluid a start would run. + let engine = engine_dir().ok().flatten(); + let home = superfluid_home().ok(); + match superfluid::Install::choose(engine.as_deref(), home.as_deref())? { + Some((i, _)) => println!( "superfluid {} ({})", i.installed_release() .unwrap_or_else(|| "of a release it does not say".to_string()), @@ -505,8 +509,8 @@ pub fn run(rest: &[String]) -> Result<()> { tail.push(dir.display().to_string()); } if o.print_command { - let binary = match superfluid::Install::locate()? { - Some(i) => i.binary().display().to_string(), + let binary = match superfluid::Install::choose(engine.as_deref(), Some(&home))? { + Some((i, _)) => i.binary().display().to_string(), None => "superfluid (not installed; the newest release would be)".to_string(), }; let mut words = vec![binary, "serve".to_string()]; @@ -528,9 +532,22 @@ pub fn run(rest: &[String]) -> Result<()> { ); return Ok(()); } - let binary = superfluid::ensure(engine.as_deref(), &home, &mut |line| { + let serving = superfluid::ensure(engine.as_deref(), &home, &mut |line| { eprintln!("basert serve: {line}") })?; + let binary = serving.binary.clone(); + // A superfluid from before 0.1.6 serves a bundle with Google's Gemma 4 + // head inside but cannot use the head, and its own words do not say what + // to do: for `mtp-head`, which it would refuse, the start stops here with + // the way out; under `auto`, which it would quietly decode plainly, the + // start goes on with the notice said. + let directive = crate::hub::mtp_speculation_asked(&o.passthrough); + if let Some(note) = embedded_head_notice(&o.models, directive, serving.release.as_deref()) { + if directive == Some("mtp-head") { + fail(&format!("basert serve: {note}")); + } + eprintln!("basert serve: {note}"); + } std::fs::create_dir_all(&home) .with_context(|| format!("Error: could not create {}", home.display()))?; @@ -547,6 +564,18 @@ pub fn run(rest: &[String]) -> Result<()> { if o.verbose { command.env("SUPERFLUID_ENGINE_VERBOSE", "1"); } + // This process becomes the server: its pid marks the engine's version as + // in use, which no update removes while it runs. That engine is + // `$BASERT_LIB`'s when it names one (superfluid reads it itself). + let serving_engine = engine.clone().or_else(|| { + std::env::var_os("BASERT_LIB") + .filter(|v| !v.is_empty()) + .map(PathBuf::from) + }); + crate::selfupdate::note_serving(serving_engine.as_deref()); + // The server keeps the start's hold on its superfluid, so no install + // swaps it under the server. + serving.hold_through_exec(); use std::os::unix::process::CommandExt; let err = command.exec(); Err(err).with_context(|| format!("launching {}", binary.display())) @@ -561,6 +590,58 @@ fn shell_word(word: &str) -> String { } } +/// What `basert serve` says before launching a superfluid that cannot use +/// the Gemma 4 head embedded in a model: `--speculate mtp-head` or `auto` was +/// asked, a model's header carries a `gemma4-mtp` head, and `release` — the +/// superfluid about to run, as `vX.Y.Z` — predates 0.1.6, the first release +/// that takes that kind. Earlier ones serve the bundle but answer `mtp-head` +/// with "not an MTP head" and `auto` with nothing, so the way out is said +/// here. None when there is nothing to say, or the release is not a version +/// this can compare. +pub(crate) fn embedded_head_notice( + models: &[String], + speculate: Option<&str>, + release: Option<&str>, +) -> Option { + let directive = speculate.filter(|d| matches!(*d, "mtp-head" | "auto"))?; + let release = release?; + if !release_predates(release, (0, 1, 6))? { + return None; + } + let model = models + .iter() + .find(|m| embedded_head_kind(Path::new(m)).as_deref() == Some("gemma4-mtp"))?; + Some(format!( + "superfluid {release} cannot use the Gemma 4 head in {model}: `--speculate {directive}` needs \ + superfluid 0.1.6 or later (this one {}); run `basert superfluid install --force`, or \ + `basert self update`", + if directive == "mtp-head" { + "refuses the head" + } else { + "decodes plainly" + } + )) +} + +/// The `kind` of the head a bundle's header carries, if any. +fn embedded_head_kind(path: &Path) -> Option { + let h = base_format::BaseReader::read_header(path).ok()?; + h.speculator? + .config + .get("kind")? + .as_str() + .map(str::to_string) +} + +/// Whether `release` (`v0.1.5`, `0.1.5`) is older than `want`; None when it +/// is not three numbers. +fn release_predates(release: &str, want: (u64, u64, u64)) -> Option { + let mut it = release.trim().trim_start_matches('v').split('.'); + let mut num = || it.next()?.parse::().ok(); + let v = (num()?, num()?, num()?); + Some(v < want) +} + fn fail(message: &str) -> ! { eprintln!("{message}"); std::process::exit(1); @@ -572,7 +653,7 @@ pub(crate) fn superfluid_home() -> Result { Ok(state_dir()?.join("superfluid")) } -fn state_dir() -> Result { +pub(crate) fn state_dir() -> Result { if let Some(dir) = std::env::var_os("BASERT_STATE_DIR").filter(|v| !v.is_empty()) { return Ok(PathBuf::from(dir)); } @@ -594,7 +675,11 @@ pub(crate) fn engine_dir() -> Result> { return Ok(None); } let exe = std::env::current_exe().context("this program's own path")?; - let dir = exe.parent().context("this program's own directory")?; + // For the versions layout, the version this launcher runs from by its real + // path, not the install directory's links through `current`: an update + // turns those under a running server, whose workers load the engine + // again with each model. + let dir = &crate::selfupdate::bundle_dir()?; let mut candidates = vec![dir.to_path_buf(), dir.join("..").join("lib")]; candidates.extend( dir.ancestors() @@ -638,6 +723,83 @@ fn first_model_under(dir: &Path, depth: usize) -> Option { mod tests { use super::*; + /// `basert serve` says, before a superfluid from before 0.1.6 starts, + /// that it cannot use a model's embedded Gemma 4 head — only when + /// speculation is asked for, a model carries that head, and the release + /// is one it can compare and predates 0.1.6. + #[test] + fn a_superfluid_before_0_1_6_is_told_about_an_embedded_head() { + use base_format::{SpeculatorBundle, FORMAT_VERSION, MAGIC}; + use std::io::Write; + let tmp = tempfile::tempdir().unwrap(); + // A header-only bundle (read_header never reaches the blob). + let bundle = |name: &str, kind: Option<&str>| -> String { + let mut h = + crate::embed::fixtures::header("gemma4", serde_json::json!({"hidden_size": 8})); + if let Some(k) = kind { + h.speculator = Some(SpeculatorBundle { + arch: k.into(), + config: [("kind".to_string(), serde_json::json!(k))] + .into_iter() + .collect(), + tensors: vec![], + }); + } + let path = tmp.path().join(name); + let json = h.to_canonical_json().unwrap(); + let mut f = std::fs::File::create(&path).unwrap(); + f.write_all(&MAGIC).unwrap(); + f.write_all(&FORMAT_VERSION.to_le_bytes()).unwrap(); + f.write_all(&(json.len() as u64).to_le_bytes()).unwrap(); + f.write_all(&json).unwrap(); + path.display().to_string() + }; + let gemma = bundle("gemma.base", Some("gemma4-mtp")); + let qwen = bundle("qwen.base", Some("mtp")); + let plain = bundle("plain.base", None); + let note = |models: &[&str], spec: Option<&str>, rel: Option<&str>| { + let models: Vec = models.iter().map(|m| m.to_string()).collect(); + embedded_head_notice(&models, spec, rel) + }; + + let said = note(&[&gemma], Some("mtp-head"), Some("v0.1.5")).unwrap(); + assert!( + said.contains("v0.1.5") + && said.contains("gemma.base") + && said.contains("install --force") + && said.contains("refuses the head"), + "{said}" + ); + let said = note(&[&plain, &gemma], Some("auto"), Some("0.1.5")).unwrap(); + assert!(said.contains("decodes plainly"), "{said}"); + + let quiet: [(&[&str], Option<&str>, Option<&str>); 9] = [ + (&[&gemma], Some("mtp-head"), Some("v0.1.6")), + (&[&gemma], Some("auto"), Some("v0.2.0")), + (&[&gemma], Some("off"), Some("v0.1.5")), + (&[&gemma], Some("prompt-lookup"), Some("v0.1.5")), + (&[&gemma], None, Some("v0.1.5")), + (&[&qwen], Some("mtp-head"), Some("v0.1.5")), + (&[&plain], Some("mtp-head"), Some("v0.1.5")), + (&[&gemma], Some("mtp-head"), None), + ( + &[&gemma], + Some("mtp-head"), + Some("of a release it does not say"), + ), + ]; + for (models, spec, rel) in quiet { + assert!( + note(models, spec, rel).is_none(), + "{models:?} {spec:?} {rel:?}" + ); + } + assert_eq!(release_predates("v0.1.5", (0, 1, 6)), Some(true)); + assert_eq!(release_predates("0.1.6", (0, 1, 6)), Some(false)); + assert_eq!(release_predates("v0.10.0", (0, 1, 6)), Some(false)); + assert_eq!(release_predates("0.1.6-rc1", (0, 1, 6)), None); + } + fn line(args: &[&str]) -> Vec { args.iter().map(|a| a.to_string()).collect() } diff --git a/base-convert/crates/base-convert/src/superfluid.rs b/base-convert/crates/base-convert/src/superfluid.rs index 264dc76..7f11b69 100644 --- a/base-convert/crates/base-convert/src/superfluid.rs +++ b/base-convert/crates/base-convert/src/superfluid.rs @@ -13,12 +13,20 @@ //! names none. //! //! Where superfluid is looked for, in order: `$BASERT_SUPERFLUID_DIR`; -//! `superfluid/` beside basert, the one-shot installer's layout; a directory -//! of the user's own, where basert installs when the one beside it cannot be -//! written (a package manager's, an app bundle's). Each holds superfluid's -//! own layout (`bin/superfluid`, the runtime workers in `libexec/superfluid/`) -//! off PATH: a superfluid the user installed for themselves is neither -//! shadowed nor run in its place. +//! `superfluid/` beside basert, the one-shot installer's layout (and a +//! release bundle's, or a build's); basert's data directory +//! ([`user_data_dir`]), where basert installs when the one beside it cannot be +//! written (an app bundle's). A basert a package manager placed (Homebrew, +//! npm, pip, pipx, uv, cargo) installs only into its data directory, looked +//! in first: a package's folder is its manager's, replaced by an upgrade and +//! left behind by an uninstall. A superfluid already beside such a basert is +//! looked for after it and runs while it can serve, but nothing is installed +//! there. Each holds superfluid's own layout (`bin/superfluid`, the runtime +//! workers in `libexec/superfluid/`) off PATH: a superfluid the user +//! installed for themselves is neither shadowed nor run in its place. +//! An install replaces the directory whole, so one that holds anything but +//! superfluid (a prefix such as `~/.local`, which superfluid's own install +//! script fills) is never replaced. //! //! A download at a start is asked for on a terminal and made without asking //! otherwise, since an older Local calls `basert serve` without knowing @@ -42,7 +50,7 @@ use sha2::{Digest, Sha256}; /// : installed instead /// of the newest when the release listing cannot be read or names no release /// for this engine, and the release a `$BASERT_SUPERFLUID_SOURCE` holds. -pub const RELEASE: &str = "v0.1.3"; +pub const RELEASE: &str = "v0.1.6"; /// `BASERT_ABI_VERSION` of the engine shipped beside this basert, which a /// superfluid release names when it serves it (`superfluid-basert-abi-.txt`). @@ -63,6 +71,12 @@ pub const SOURCE_ENV: &str = "BASERT_SUPERFLUID_SOURCE"; pub const AUTO_INSTALL_ENV: &str = "BASERT_SUPERFLUID_AUTO_INSTALL"; /// Written into an install with the release it holds. const STAMP: &str = ".release"; +/// Written into an install basert made, as a package manager's copy carries +/// one: superfluid's own `self update` then leaves the copy to basert, which +/// picks the release that serves its engine, instead of moving it under +/// `~/.superfluid` and checking it against an engine that is not basert's. +const INSTALLED_BY: &str = "installed-by"; +const OWNED_BY: &str = "basert: basert superfluid install"; /// The name superfluid's basert runtime answers to in `runtimes --json`. const RUNTIME_ID: &str = "basert"; @@ -83,9 +97,9 @@ pub enum Action { force: bool, }, /// Show the release this basert was tested with, what is installed where, - /// and whether it can serve the engine beside basert. Exits 0 when an - /// installed superfluid can, 1 when none is installed, 2 when the - /// installed one cannot. + /// and whether it can serve the engine beside basert: the install a start + /// would run, the first that can. Exits 0 when an installed superfluid + /// can, 1 when none is installed, 2 when none installed can. Status, } @@ -108,16 +122,22 @@ pub fn cmd(args: SuperfluidArgs) -> Result<()> { ); return Ok(()); } - Some(i) if i.writable() => i, + Some(i) if i.takes_installs() && i.writable() => i, _ => Install::for_install()?, }; + // A server's workers start from this directory: replaced under + // it, the next one would be another release's. Looked at before + // the download, and again at the swap. + drop(dest.unused()?); dest.install(&source, &release, &mut |line| eprintln!("{line}"))?; println!("superfluid {tag} is installed at {}", dest.root.display()); Ok(()) } Action::Status => { println!("tested: {RELEASE}"); - let code = match Install::locate()? { + let engine = crate::serve::engine_dir().ok().flatten(); + let home = crate::serve::superfluid_home().ok(); + let code = match Install::choose(engine.as_deref(), home.as_deref())? { None => { let dest = Install::for_install()?; println!( @@ -126,7 +146,8 @@ pub fn cmd(args: SuperfluidArgs) -> Result<()> { ); 1 } - Some(i) => { + Some((i, ready)) => { + i.claim(); let have = i .installed_release() .unwrap_or_else(|| "a release that does not say".to_string()); @@ -135,9 +156,7 @@ pub fn cmd(args: SuperfluidArgs) -> Result<()> { i.root.display(), i.origin.said() ); - let engine = crate::serve::engine_dir().ok().flatten(); - let home = crate::serve::superfluid_home().ok(); - match i.probe(engine.as_deref(), home.as_deref()).ready { + match ready { Ok(()) => { println!("engine: ready to serve"); 0 @@ -167,10 +186,11 @@ pub fn target() -> Result<&'static str> { } } -/// basert's own place for a superfluid it installs when the directory beside -/// it cannot be written: under the user's local data directory -/// (`~/Library/Application Support/baseRT` on macOS, `$XDG_DATA_HOME/baseRT` -/// or `~/.local/share/baseRT` on Linux). +/// basert's own place for a superfluid it installs for a basert a package +/// manager placed, or when the directory beside basert cannot be written: +/// under the user's local data directory (`~/Library/Application +/// Support/baseRT` on macOS, `$XDG_DATA_HOME/baseRT` or `~/.local/share/baseRT` +/// on Linux). pub fn user_data_dir() -> Result { let data = dirs::data_local_dir() .context("no data directory to install superfluid in (set BASERT_SUPERFLUID_DIR)")?; @@ -184,6 +204,9 @@ pub enum Origin { Env, /// `superfluid/` beside basert: the one-shot installer's layout. BesideBasert, + /// `superfluid/` beside a basert a package manager placed: run when it + /// is there, never installed into. + BesidePackage, /// [`user_data_dir`]. UserData, } @@ -193,7 +216,7 @@ impl Origin { fn said(self) -> &'static str { match self { Origin::Env => "named by BASERT_SUPERFLUID_DIR", - Origin::BesideBasert => "beside basert", + Origin::BesideBasert | Origin::BesidePackage => "beside basert", Origin::UserData => "basert's data directory", } } @@ -209,7 +232,8 @@ pub struct Install { impl Install { /// Where superfluid is looked for, in order: `$BASERT_SUPERFLUID_DIR` - /// alone when set; else beside basert, then the user's data directory. + /// alone when set; else beside basert, then the user's data directory; + /// for a basert a package manager placed, the other way round. pub fn candidates() -> Result> { if let Some(dir) = std::env::var_os(DIR_ENV).filter(|d| !d.is_empty()) { return Ok(vec![Install { @@ -217,17 +241,33 @@ impl Install { origin: Origin::Env, }]); } + // The install directory, not the version a versions-layout basert + // runs from: superfluid stays put while basert's versions change. + let beside = crate::selfupdate::install_dir()?.join("superfluid"); + let data = Install { + root: user_data_dir()?, + origin: Origin::UserData, + }; let exe = std::env::current_exe().context("this program's own path")?; - let dir = exe.parent().context("this program's own directory")?; + if crate::selfupdate::installed_as(&exe).by_package_manager() { + // A package's folder is its manager's: Homebrew's keg and npm's + // package are replaced by an upgrade, pip's is left behind by an + // uninstall, cargo's is ~/.cargo/bin. One installed there before + // still runs, after the data directory's. + return Ok(vec![ + data, + Install { + root: beside, + origin: Origin::BesidePackage, + }, + ]); + } Ok(vec![ Install { - root: dir.join("superfluid"), + root: beside, origin: Origin::BesideBasert, }, - Install { - root: user_data_dir()?, - origin: Origin::UserData, - }, + data, ]) } @@ -236,12 +276,15 @@ impl Install { Ok(Self::candidates()?.into_iter().find(Install::is_installed)) } - /// Where an install goes: the first place that can be written, else the - /// last, whose own error then says why not. + /// Where an install goes: the first place that takes one and can be + /// written, else the last that takes one, whose own error then says why not. pub fn for_install() -> Result { - let candidates = Self::candidates()?; - let last = candidates.len() - 1; - Ok(candidates + let places: Vec = Self::candidates()? + .into_iter() + .filter(Install::takes_installs) + .collect(); + let last = places.len() - 1; + Ok(places .into_iter() .enumerate() .find(|(k, c)| *k == last || c.writable()) @@ -249,6 +292,11 @@ impl Install { .expect("at least one candidate")) } + /// Whether basert installs here: everywhere but a package's folder. + pub fn takes_installs(&self) -> bool { + self.origin != Origin::BesidePackage + } + /// Whether an install can be made or replaced here: its parent takes a /// new directory. pub fn writable(&self) -> bool { @@ -277,6 +325,58 @@ impl Install { is_executable(&self.binary()) } + /// Mark an install basert made (beside it or in its data directory, not + /// one `$BASERT_SUPERFLUID_DIR` names) as basert's, for a copy installed + /// before installs carried the file. Best effort: a directory that cannot + /// be written stays as it is. + pub fn claim(&self) { + if self.origin != Origin::Env + && self.is_installed() + && !self.root.join(INSTALLED_BY).exists() + { + let _ = fs::write(self.root.join(INSTALLED_BY), format!("{OWNED_BY}\n")); + } + } + + /// That this install's directory may be replaced whole, as an install + /// replaces it: it holds superfluid alone (see [`not_superfluids`]), or + /// nothing yet. A prefix such as `~/.local`, which superfluid's own install + /// script fills and `$BASERT_SUPERFLUID_DIR` may name, holds the user's + /// other files, which would be removed with it. `fresh` is the release + /// about to take its place. + pub fn replaceable(&self, fresh: &Path) -> Result<()> { + if fs::symlink_metadata(&self.root).is_err() { + return Ok(()); + } + // A program of superfluid's name (a `cargo install superfluid`) is + // not a directory to look in. + let found = match fs::metadata(&self.root).is_ok_and(|m| m.is_dir()) { + true => match not_superfluids(&self.root, fresh, Level::Top) { + Some(found) => found, + None => return Ok(()), + }, + false => self.root.clone(), + }; + let what = match found == self.root { + true => format!( + "{} is a file, not superfluid's directory", + self.root.display() + ), + false => format!( + "{} holds {}, which is not superfluid's", + self.root.display(), + found.display() + ), + }; + let then = match self.origin { + Origin::Env => format!( + "name a directory of superfluid's own in {DIR_ENV} (a new one: basert installs superfluid there)" + ), + _ => format!("move {} aside", found.display()), + }; + bail!("{what}, and replacing superfluid there would remove it: nothing is changed; {then}") + } + /// The release installed here: the stamp an install writes, else what the /// binary says of itself (a copy put here by hand). `None` when nothing is /// installed. @@ -332,6 +432,110 @@ impl Install { release: &Release, say: &mut dyn FnMut(&str), ) -> Result<()> { + let _turn = self.take_turn(say)?; + let staged = self.stage(source, release, say)?; + self.swap_in(staged, say) + } + + /// The lock file `kind` beside this install, made when missing; `None` + /// where it cannot be (a directory basert cannot write, where it + /// installs nothing either). + fn lock(&self, kind: &str) -> Option { + let name = self.root.file_name()?.to_string_lossy(); + let parent = self.parent(); + // Not there before a first install. + let _ = fs::create_dir_all(&parent); + let path = parent.join(format!(".{}.{kind}.lock", name.trim_start_matches('.'))); + fs::OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(path) + .ok() + } + + /// Installs into this directory take turns: one that finds another under + /// way says so and waits for it, then looks again at what is there. + pub fn take_turn(&self, say: &mut dyn FnMut(&str)) -> Result> { + let Some(file) = self.lock("install") else { + return Ok(None); + }; + if !flock(&file, libc::LOCK_EX | libc::LOCK_NB)? { + say(&format!( + "waiting for another basert installing superfluid into {}", + self.root.display() + )); + flock(&file, libc::LOCK_EX)?; + } + Ok(Some(Held(file))) + } + + /// Held by a start from asking this install until its server stops: no + /// install swaps it in between. Waits only while one swaps. + pub fn hold(&self) -> Option { + let file = self.lock("use")?; + flock(&file, libc::LOCK_SH).ok()?.then_some(Held(file)) + } + + /// That no server runs from this install (one `basert serve` started + /// holds it; others are found by what runs from the directory) and none + /// starts while the returned lock is held. + pub fn unused(&self) -> Result> { + let file = self.lock("use"); + let free = match &file { + Some(file) => flock(file, libc::LOCK_EX | libc::LOCK_NB)?, + None => true, + }; + let pids = crate::selfupdate::running_from(&self.root); + if free && pids.is_empty() { + return Ok(file.map(Held)); + } + let who = match pids.is_empty() { + true => "one basert serve started".to_string(), + false => format!( + "pid {}", + pids.iter() + .map(i32::to_string) + .collect::>() + .join(", ") + ), + }; + bail!( + "a server runs from {} ({who}): stop it, then run this again", + self.root.display() + ) + } + + /// The install a start would run for the engine in `engine`, asked as a + /// start asks but nothing installed: the first found that can serve, + /// else the first found, with why it cannot. + pub fn choose( + engine: Option<&Path>, + home: Option<&Path>, + ) -> Result)>> { + let mut first = None; + for install in Self::candidates()? + .into_iter() + .filter(Install::is_installed) + { + let ready = install.probe(engine, home).ready; + if ready.is_ok() { + return Ok(Some((install, ready))); + } + first.get_or_insert((install, ready)); + } + Ok(first) + } + + /// The first half of [`Install::install`]: `release` downloaded, checked + /// and unpacked beside this install, where it can be asked whether it + /// serves an engine before it takes this one's place. + pub fn stage( + &self, + source: &Source, + release: &Release, + say: &mut dyn FnMut(&str), + ) -> Result { let tag = &release.tag; let name = format!("superfluid-{}", target()?); let parent = self.parent(); @@ -350,38 +554,49 @@ impl Install { )); source.fetch(release, &format!("{name}.tar.gz"), &tarball)?; source.fetch(release, &format!("{name}.tar.gz.sha256"), &checksum)?; - let want = fs::read_to_string(&checksum) - .with_context(|| format!("reading {}", checksum.display()))? - .split_whitespace() - .next() - .unwrap_or_default() - .to_ascii_lowercase(); - let have = sha256_of(&tarball)?; - if want.len() != 64 || want != have { - bail!( - "{name}.tar.gz: SHA-256 is {have}, the release's checksum file says {want:?}: \ - the download is not the published file" - ); - } + check_sha256(&tarball, &checksum)?; let unpacked = stage.path().join("unpacked"); fs::create_dir(&unpacked)?; - let status = Command::new("tar") - .arg("-xzf") - .arg(&tarball) - .arg("-C") - .arg(&unpacked) - .status() - .context("running tar")?; - if !status.success() { - bail!("tar -xzf {} failed ({status})", tarball.display()); - } + untar(&tarball, &unpacked)?; let fresh = unpacked.join(&name); if !is_executable(&fresh.join("bin").join("superfluid")) { bail!("{name}.tar.gz holds no {name}/bin/superfluid"); } fs::write(fresh.join(STAMP), format!("{tag}\n"))?; + fs::write(fresh.join(INSTALLED_BY), format!("{OWNED_BY}\n"))?; + Ok(Staged { + _dir: stage, + fresh: Install { + root: fresh, + origin: self.origin, + }, + tag: tag.clone(), + replacing: self.installed_release(), + }) + } + /// The second half of [`Install::install`]: the staged release takes this + /// install's place. + pub fn swap_in(&self, staged: Staged, say: &mut dyn FnMut(&str)) -> Result<()> { + let tag = &staged.tag; + let fresh = &staged.fresh.root; + self.replaceable(fresh)?; + // The last look, held through the swap: no server runs from it, and + // none starts until it is done; and it holds what it held when the + // release was staged. + let _swapping = self.unused()?; + let now = self.installed_release(); + if now != staged.replacing { + bail!( + "{} changed while superfluid {tag} was fetched (it holds {}, not {}): nothing \ + is changed; run this again", + self.root.display(), + now.as_deref().unwrap_or("no superfluid"), + staged.replacing.as_deref().unwrap_or("none") + ); + } + let parent = self.parent(); let old = parent.join(format!(".superfluid-old-{}", std::process::id())); let had_old = self.root.exists(); if had_old { @@ -392,7 +607,7 @@ impl Install { ) })?; } - if let Err(e) = fs::rename(&fresh, &self.root) { + if let Err(e) = fs::rename(fresh, &self.root) { if had_old { let _ = fs::rename(&old, &self.root); } @@ -410,6 +625,171 @@ impl Install { } } +/// A release [`Install::stage`] unpacked beside an install: `fresh` can be +/// probed as an install of its own until [`Install::swap_in`] moves it into +/// place. Dropped instead, it is removed. +pub struct Staged { + _dir: tempfile::TempDir, + pub fresh: Install, + pub tag: String, + /// The release installed where it goes when it was staged. + replacing: Option, +} + +/// A lock on an install, in a file beside its directory (which an install +/// replaces whole). Installs into the directory take turns on one +/// ([`Install::take_turn`]); a start holds the other shared from asking the +/// install until its server stops ([`Install::hold`], +/// [`Serving::hold_through_exec`]), and an install holds it exclusive while +/// it swaps ([`Install::unused`]). +pub struct Held(fs::File); + +/// `flock(op)` on `file`: false when `LOCK_NB` is asked and another holds it. +fn flock(file: &fs::File, op: libc::c_int) -> std::io::Result { + use std::os::fd::AsRawFd; + loop { + // SAFETY: flock on a descriptor this process holds open. + if unsafe { libc::flock(file.as_raw_fd(), op) } == 0 { + return Ok(true); + } + let e = std::io::Error::last_os_error(); + match e.raw_os_error() { + Some(libc::EINTR) => continue, + Some(libc::EWOULDBLOCK) => return Ok(false), + _ => return Err(e), + } + } +} + +/// The superfluid a start runs, and its hold on it. +pub struct Serving { + pub binary: PathBuf, + /// The release the install says it is (`vX.Y.Z`), when it says. + pub release: Option, + held: Option, +} + +impl Serving { + /// The hold kept open across the `exec` that makes this process the + /// server, and by each process the server starts: no install swaps + /// superfluid under it while it runs. + pub fn hold_through_exec(self) { + use std::os::fd::IntoRawFd; + if let Some(Held(file)) = self.held { + let fd = file.into_raw_fd(); + // SAFETY: fcntl on a descriptor this process holds open. + unsafe { + let flags = libc::fcntl(fd, libc::F_GETFD); + if flags >= 0 { + libc::fcntl(fd, libc::F_SETFD, flags & !libc::FD_CLOEXEC); + } + } + } + } +} + +/// Where in a release [`not_superfluids`] looks. +#[derive(Debug, Clone, Copy)] +enum Level { + Top, + Bin, + Libexec, + Other, +} + +/// The first entry under `dir` that is not superfluid's, `at` that level of a +/// release: superfluid's is a name `fresh` (the same level of the release +/// about to replace it) has too, or one of superfluid's layout (`bin/` holding +/// `superfluid*`, `libexec/superfluid/`, the release's `LICENSE`, `NOTICE`, +/// `README.md` and `compat.json`, the `.release` and `installed-by` basert +/// writes). A link is superfluid's only where `fresh` has one (a release holds +/// none; superfluid's own install script links `bin/superfluid` to its +/// versions); a directory that cannot be read is not. +fn not_superfluids(dir: &Path, fresh: &Path, at: Level) -> Option { + let Ok(entries) = fs::read_dir(dir) else { + return Some(dir.to_path_buf()); + }; + for entry in entries { + let Ok(entry) = entry else { + return Some(dir.to_path_buf()); + }; + let path = entry.path(); + let name = entry.file_name().to_string_lossy().into_owned(); + let Ok(meta) = fs::symlink_metadata(&path) else { + return Some(path); + }; + let theirs = fs::symlink_metadata(fresh.join(&name)); + if meta.file_type().is_symlink() { + if !theirs.is_ok_and(|m| m.file_type().is_symlink()) { + return Some(path); + } + continue; + } + let layout = match at { + Level::Top => { + matches!( + name.as_str(), + "bin" | "libexec" | "LICENSE" | "NOTICE" | "README.md" | "compat.json" + ) || name == STAMP + || name == INSTALLED_BY + } + Level::Bin => name.starts_with("superfluid"), + Level::Libexec => name == "superfluid", + Level::Other => false, + }; + if !layout && theirs.is_err() { + return Some(path); + } + if meta.is_dir() { + let inner = match (at, name.as_str()) { + // superfluid's own directory: its workers and libraries. + (Level::Libexec, "superfluid") => continue, + (Level::Top, "bin") => Level::Bin, + (Level::Top, "libexec") => Level::Libexec, + _ => Level::Other, + }; + if let Some(found) = not_superfluids(&path, &fresh.join(&name), inner) { + return Some(found); + } + } + } + None +} + +/// That `tarball` is the file whose SHA-256 the release's `checksum` file +/// (` `) states. +pub(crate) fn check_sha256(tarball: &Path, checksum: &Path) -> Result<()> { + let name = tarball.file_name().unwrap_or_default().to_string_lossy(); + let want = fs::read_to_string(checksum) + .with_context(|| format!("reading {}", checksum.display()))? + .split_whitespace() + .next() + .unwrap_or_default() + .to_ascii_lowercase(); + let have = sha256_of(tarball)?; + if want.len() != 64 || want != have { + bail!( + "{name}: SHA-256 is {have}, the release's checksum file says {want:?}: \ + the download is not the published file" + ); + } + Ok(()) +} + +pub(crate) fn untar(tarball: &Path, into: &Path) -> Result<()> { + let status = Command::new("tar") + .arg("-xzf") + .arg(tarball) + .arg("-C") + .arg(into) + .status() + .context("running tar")?; + if !status.success() { + bail!("tar -xzf {} failed ({status})", tarball.display()); + } + Ok(()) +} + /// What a superfluid says of its basert runtime against an engine. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Probe { @@ -520,24 +900,31 @@ impl Source { .with_context(|| format!("copying {}", from.display())) .map(|_| ()) } - Source::GitHub => { - let url = release.url(file); - // GitHub answers an asset with a redirect to its storage, which - // the agent follows. - let response = agent(Duration::from_secs(600)) - .get(&url) - .call() - .with_context(|| format!("downloading {url}"))?; - let mut body = response.into_body().into_reader(); - let mut out = fs::File::create(dest) - .with_context(|| format!("creating {}", dest.display()))?; - std::io::copy(&mut body, &mut out).with_context(|| format!("reading {url}"))?; - Ok(()) - } + Source::GitHub => download(&release.url(file), dest), } } } +/// `url` into the file `dest`. A listing in a file (a mirror's, the tests') +/// may name its assets by path instead. +pub(crate) fn download(url: &str, dest: &Path) -> Result<()> { + if !url.starts_with("https://") && !url.starts_with("http://") { + return fs::copy(url, dest) + .with_context(|| format!("copying {url}")) + .map(|_| ()); + } + // GitHub answers an asset with a redirect to its storage, which the + // agent follows. + let response = agent(Duration::from_secs(600)) + .get(url) + .call() + .with_context(|| format!("downloading {url}"))?; + let mut body = response.into_body().into_reader(); + let mut out = fs::File::create(dest).with_context(|| format!("creating {}", dest.display()))?; + std::io::copy(&mut body, &mut out).with_context(|| format!("reading {url}"))?; + Ok(()) +} + impl Source { /// Where `file` of `release` comes from, for a message. fn describe(&self, release: &Release, file: &str) -> String { @@ -581,9 +968,20 @@ pub fn wanted(source: &Source, say: &mut dyn FnMut(&str)) -> Release { } /// superfluid's release listing, or the one `$BASERT_SUPERFLUID_RELEASES` names. -fn releases() -> Result { +pub(crate) fn releases() -> Result { let named = std::env::var(RELEASES_ENV).ok().filter(|v| !v.is_empty()); - let at = named.as_deref().unwrap_or(RELEASES); + read_listing(named.as_deref().unwrap_or(RELEASES)) +} + +/// The newest superfluid release that serves engine ABI `abi` here, from +/// [`releases`]: `None` when the listing names none. +pub fn newest_for(abi: u32) -> Result> { + let target = target()?; + Ok(newest_in(&releases()?, target, abi)) +} + +/// A release listing in GitHub's form, from a URL or a file. +pub(crate) fn read_listing(at: &str) -> Result { let text = if at.starts_with("https://") || at.starts_with("http://") { agent(Duration::from_secs(60)) .get(at) @@ -601,6 +999,16 @@ fn releases() -> Result { /// nor a pre-release and publishes, for `target`, the tarball, its checksum /// and the marker of engine ABI `abi`. pub fn newest_in(listing: &serde_json::Value, target: &str, abi: u32) -> Option { + newest_where(listing, target, abi, &|_| true) +} + +/// [`newest_in`] among the releases `keep` accepts (by their listing entry). +pub fn newest_where( + listing: &serde_json::Value, + target: &str, + abi: u32, + keep: &dyn Fn(&serde_json::Value) -> bool, +) -> Option { let tarball = format!("superfluid-{target}.tar.gz"); let checksum = format!("{tarball}.sha256"); let marker = format!("superfluid-basert-abi-{abi}.txt"); @@ -609,6 +1017,7 @@ pub fn newest_in(listing: &serde_json::Value, target: &str, abi: u32) -> Option< .iter() .filter(|r| !r["draft"].as_bool().unwrap_or(false)) .filter(|r| !r["prerelease"].as_bool().unwrap_or(false)) + .filter(|r| keep(r)) .filter_map(|r| { let tag = r["tag_name"].as_str()?; let assets = r["assets"].as_array()?; @@ -637,7 +1046,7 @@ fn release_key(tag: &str) -> Option> { } /// Whether the release `have` is newer than `than`. -fn newer(have: Option<&str>, than: &str) -> bool { +pub(crate) fn newer(have: Option<&str>, than: &str) -> bool { match (have.and_then(release_key), release_key(than)) { (Some(have), Some(than)) => have > than, _ => false, @@ -673,11 +1082,11 @@ pub fn policy() -> Download { } } -/// The superfluid a `basert serve` start runs, as a path to its binary: the +/// The superfluid a `basert serve` start runs, with its hold on it: the /// first install whose basert runtime superfluid itself reports ready for /// the engine in `engine`, under `home` as its `SUPERFLUID_HOME`. `say` gets /// a line for each step taken on the user's behalf. -pub fn ensure(engine: Option<&Path>, home: &Path, say: &mut dyn FnMut(&str)) -> Result { +pub fn ensure(engine: Option<&Path>, home: &Path, say: &mut dyn FnMut(&str)) -> Result { ensure_with(policy(), &Source::locate(), engine, home, say) } @@ -687,15 +1096,24 @@ pub fn ensure_with( engine: Option<&Path>, home: &Path, say: &mut dyn FnMut(&str), -) -> Result { - // Every install found is asked; the first that can serve runs. +) -> Result { + // Every install found is asked; the first that can serve runs. Each is + // held while asked, and the one that runs until its server stops. let mut unready: Vec<(Install, Option, String)> = Vec::new(); for install in Install::candidates()? .into_iter() .filter(Install::is_installed) { + install.claim(); + let held = install.hold(); match install.probe(engine, Some(home)).ready { - Ok(()) => return Ok(install.binary()), + Ok(()) => { + return Ok(Serving { + binary: install.binary(), + release: install.installed_release(), + held, + }) + } Err(why) => { let have = install.installed_release(); unready.push((install, have, why)); @@ -732,10 +1150,44 @@ pub fn ensure_with( } )); } - let dest = match unready.iter().map(|(i, _, _)| i).find(|i| i.writable()) { + let dest = match unready + .iter() + .map(|(i, _, _)| i) + .find(|i| i.takes_installs() && i.writable()) + { Some(found) => found.clone(), None => Install::for_install()?, }; + // A server's workers start from that directory: replaced under it, its + // next one would be another release's. Two baserts on engines that need + // different releases share one (a package's basert installs in its data + // directory), so the other's server may run from it. Looked at before + // the download, and again at the swap. + if let Err(e) = dest.unused() { + bail!( + "superfluid at {} cannot serve this engine and {tag} can, but {e:#}", + dest.root.display() + ); + } + // Installs into it take turns: one that finished while this waited may + // serve this engine, and then runs. + let turn = match (policy, &release) { + (Download::Never, _) | (_, None) => None, + _ => { + let turn = dest.take_turn(say)?; + if dest.is_installed() { + let held = dest.hold(); + if dest.probe(engine, Some(home)).ready.is_ok() { + return Ok(Serving { + binary: dest.binary(), + release: dest.installed_release(), + held, + }); + } + } + turn + } + }; let hint = format!( "install it with\n basert superfluid install\n(the newest superfluid release that serves \ this engine, from https://github.com/{REPO}/releases, into {})", @@ -768,9 +1220,16 @@ pub fn ensure_with( release } }; - dest.install(source, &release, say)?; + let staged = dest.stage(source, &release, say)?; + dest.swap_in(staged, say)?; + let held = dest.hold(); + drop(turn); match dest.probe(engine, Some(home)).ready { - Ok(()) => Ok(dest.binary()), + Ok(()) => Ok(Serving { + binary: dest.binary(), + release: dest.installed_release(), + held, + }), Err(why) => bail!( "superfluid {tag}, installed at {}, cannot serve this engine: {why}", dest.root.display() @@ -779,7 +1238,7 @@ pub fn ensure_with( } /// A yes/no question on stderr, answered on stdin; an empty answer is yes. -fn confirm(question: &str) -> Result { +pub(crate) fn confirm(question: &str) -> Result { eprint!("{question}"); std::io::stderr().flush()?; let mut answer = String::new(); @@ -790,14 +1249,14 @@ fn confirm(question: &str) -> Result { )) } -fn sha256_of(path: &Path) -> Result { +pub(crate) fn sha256_of(path: &Path) -> Result { let mut file = fs::File::open(path).with_context(|| format!("opening {}", path.display()))?; let mut hasher = Sha256::new(); std::io::copy(&mut file, &mut hasher)?; Ok(format!("{:x}", hasher.finalize())) } -fn is_executable(path: &Path) -> bool { +pub(crate) fn is_executable(path: &Path) -> bool { use std::os::unix::fs::PermissionsExt; fs::metadata(path).is_ok_and(|m| m.is_file() && m.permissions().mode() & 0o111 != 0) } @@ -875,6 +1334,49 @@ mod tests { } } + #[test] + fn a_copy_basert_made_before_is_claimed_and_one_the_user_named_is_not() { + let tmp = tempfile::tempdir().unwrap(); + for (origin, claimed) in [ + (Origin::BesideBasert, true), + (Origin::BesidePackage, true), + (Origin::UserData, true), + (Origin::Env, false), + ] { + let root = tmp.path().join(format!("{origin:?}")); + fs::create_dir_all(root.join("bin")).unwrap(); + fs::write( + root.join("bin").join("superfluid"), + fake_superfluid("0.1.3"), + ) + .unwrap(); + use std::os::unix::fs::PermissionsExt; + fs::set_permissions( + root.join("bin").join("superfluid"), + fs::Permissions::from_mode(0o755), + ) + .unwrap(); + let install = Install { + root: root.clone(), + origin, + }; + install.claim(); + assert_eq!(root.join(INSTALLED_BY).is_file(), claimed, "{origin:?}"); + } + // A file already there is left as it is. + let root = tmp.path().join("BesideBasert"); + fs::write(root.join(INSTALLED_BY), "someone: else\n").unwrap(); + Install { + root: root.clone(), + origin: Origin::BesideBasert, + } + .claim(); + assert_eq!( + fs::read_to_string(root.join(INSTALLED_BY)).unwrap(), + "someone: else\n" + ); + } + #[test] fn an_install_unpacks_the_release_and_stamps_it() { let tmp = tempfile::tempdir().unwrap(); @@ -896,14 +1398,27 @@ mod tests { .join("superfluid-worker-basert") .is_file()); assert_eq!(install.installed_release().as_deref(), Some(RELEASE)); + // basert's own copy, which superfluid's `self update` leaves to basert. + assert_eq!( + fs::read_to_string(install.root.join(INSTALLED_BY)).unwrap(), + format!("{OWNED_BY}\n") + ); assert_eq!(said.len(), 2, "{said:?}"); assert!(said[0].starts_with(&format!("downloading superfluid {RELEASE} ("))); - // Nothing of the staging is left behind. - let left: Vec<_> = fs::read_dir(tmp.path().join("bin")) + // Nothing of the staging is left behind; the locks beside it stay. + let mut left: Vec<_> = fs::read_dir(tmp.path().join("bin")) .unwrap() .map(|e| e.unwrap().file_name().into_string().unwrap()) .collect(); - assert_eq!(left, ["superfluid"]); + left.sort(); + assert_eq!( + left, + [ + ".superfluid.install.lock", + ".superfluid.use.lock", + "superfluid" + ] + ); } #[test] @@ -931,6 +1446,194 @@ mod tests { assert_eq!(install.installed_release().as_deref(), Some(RELEASE)); } + /// What `bin/superfluid` in `root` says it is. + fn says(root: &Path) -> String { + fs::read_to_string(root.join("bin").join("superfluid")).unwrap() + } + + #[test] + fn a_directory_holding_more_than_superfluid_is_not_replaced() { + let tmp = tempfile::tempdir().unwrap(); + let first = release(&tmp.path().join("first"), "0.1.3", true); + let next = release(&tmp.path().join("next"), "0.1.4", true); + let elsewhere = tmp.path().join("elsewhere"); + fs::create_dir_all(&elsewhere).unwrap(); + // What each directory holds beside (or instead of) superfluid: a + // prefix such as ~/.local, which superfluid's own install script + // fills and `$BASERT_SUPERFLUID_DIR` may name, and a directory of + // the user's own named before anything was installed in it. + type Extra = fn(&Path, &Path); + let cases: [(&str, bool, Extra, &str); 4] = [ + ( + "share", + true, + |root, _| { + fs::create_dir_all(root.join("share").join("app")).unwrap(); + fs::write(root.join("share").join("app").join("data.txt"), "mine").unwrap(); + }, + "share", + ), + ( + "tool", + true, + |root, _| { + fs::write(root.join("bin").join("other-tool"), "mine").unwrap(); + }, + "other-tool", + ), + ( + "link", + true, + |root, elsewhere| { + // superfluid's install script links bin/superfluid to its + // own versions. + fs::rename( + root.join("bin").join("superfluid"), + elsewhere.join("superfluid"), + ) + .unwrap(); + std::os::unix::fs::symlink( + elsewhere.join("superfluid"), + root.join("bin").join("superfluid"), + ) + .unwrap(); + }, + "superfluid", + ), + ( + "mine", + false, + |root, _| { + fs::create_dir_all(root).unwrap(); + fs::write(root.join("notes.txt"), "mine").unwrap(); + }, + "notes.txt", + ), + ]; + for (name, with_superfluid, extra, named) in cases { + let install = install_at(tmp.path().join(name)); + if with_superfluid { + install + .install(&Source::Dir(first.clone()), &Release::tested(), &mut |_| {}) + .unwrap(); + } + extra(&install.root, &elsewhere); + let before = says_all(&install.root); + let err = install + .install(&Source::Dir(next.clone()), &Release::tested(), &mut |_| {}) + .unwrap_err(); + let err = format!("{err:#}"); + assert!(err.contains("which is not superfluid's"), "{name}: {err}"); + assert!(err.contains(named), "{name}: {err}"); + assert!(err.contains("BASERT_SUPERFLUID_DIR"), "{name}: {err}"); + assert_eq!(says_all(&install.root), before, "{name}"); + // Nothing staged is left beside it (the locks stay). + let left: Vec<_> = fs::read_dir(tmp.path()) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .filter(|n| n.starts_with('.') && !n.ends_with(".lock")) + .collect(); + assert!(left.is_empty(), "{name}: {left:?}"); + } + } + + /// Every file under `root` with what it holds, links as their targets. + fn says_all(root: &Path) -> Vec<(PathBuf, String)> { + let mut all = Vec::new(); + let mut dirs = vec![root.to_path_buf()]; + while let Some(dir) = dirs.pop() { + for entry in fs::read_dir(&dir).unwrap().flatten() { + let path = entry.path(); + let meta = fs::symlink_metadata(&path).unwrap(); + if meta.file_type().is_symlink() { + all.push(( + path.clone(), + format!("-> {}", fs::read_link(&path).unwrap().display()), + )); + } else if meta.is_dir() { + dirs.push(path); + } else { + all.push((path.clone(), fs::read_to_string(&path).unwrap())); + } + } + } + all.sort(); + all + } + + #[test] + fn a_file_where_superfluid_would_go_is_left_alone_and_said_to_be_one() { + let tmp = tempfile::tempdir().unwrap(); + let assets = release(tmp.path(), "0.1.4", true); + // A program of superfluid's name (`cargo install superfluid` puts one + // beside a basert cargo installed). + let install = Install { + root: tmp.path().join("superfluid"), + origin: Origin::BesideBasert, + }; + fs::write(&install.root, "#!/bin/sh\n").unwrap(); + let err = install + .install(&Source::Dir(assets), &Release::tested(), &mut |_| {}) + .unwrap_err(); + let err = format!("{err:#}"); + assert!( + err.contains(&format!( + "{} is a file, not superfluid's directory", + install.root.display() + )), + "{err}" + ); + assert!( + err.contains(&format!("move {} aside", install.root.display())), + "{err}" + ); + assert_eq!(fs::read_to_string(&install.root).unwrap(), "#!/bin/sh\n"); + } + + #[test] + fn what_the_next_release_also_holds_is_superfluids() { + let tmp = tempfile::tempdir().unwrap(); + let (root, fresh) = (tmp.path().join("root"), tmp.path().join("fresh")); + for dir in [&root, &fresh] { + fs::create_dir_all(dir.join("share").join("superfluid")).unwrap(); + fs::write(dir.join("share").join("superfluid").join("t.json"), "").unwrap(); + fs::create_dir_all(dir.join("bin")).unwrap(); + fs::write(dir.join("bin").join("sf-helper"), "").unwrap(); + } + assert_eq!(not_superfluids(&root, &fresh, Level::Top), None); + fs::write(root.join("share").join("other.txt"), "").unwrap(); + assert_eq!( + not_superfluids(&root, &fresh, Level::Top), + Some(root.join("share").join("other.txt")) + ); + } + + #[test] + fn a_directory_holding_superfluid_alone_is_replaced() { + let tmp = tempfile::tempdir().unwrap(); + let first = release(&tmp.path().join("first"), "0.1.3", true); + let next = release(&tmp.path().join("next"), "0.1.4", true); + let install = install_at(tmp.path().join("superfluid")); + install + .install(&Source::Dir(first), &Release::tested(), &mut |_| {}) + .unwrap(); + // All a release holds, the release's own files included, and what + // basert writes beside them. + for file in ["LICENSE", "NOTICE", "README.md", "compat.json"] { + fs::write(install.root.join(file), "release").unwrap(); + } + for tool in ["superfluid-noded", "superfluid-workerd"] { + fs::write(install.root.join("bin").join(tool), "release").unwrap(); + } + assert!(install.root.join(STAMP).is_file()); + assert!(install.root.join(INSTALLED_BY).is_file()); + install + .install(&Source::Dir(next), &Release::tested(), &mut |_| {}) + .unwrap(); + assert!(says(&install.root).contains("superfluid 0.1.4")); + assert!(!install.root.join("LICENSE").exists()); + } + #[test] fn a_missing_asset_is_an_error_that_names_it() { let tmp = tempfile::tempdir().unwrap(); diff --git a/base-convert/crates/base-convert/tests/self_update.rs b/base-convert/crates/base-convert/tests/self_update.rs new file mode 100644 index 0000000..4161de4 --- /dev/null +++ b/base-convert/crates/base-convert/tests/self_update.rs @@ -0,0 +1,1529 @@ +//! `basert self`, end to end: the launcher, placed in a directory the way the +//! install script placed it before the versions layout, updates itself from a +//! release listing in a file, goes back and forward, and brings a superfluid +//! beside it to one that serves the engine. The releases are fake (a launcher +//! that is a shell script, a library of a few bytes); the superfluid is a +//! script that serves one engine ABI number and reads the engine's from its +//! header. + +use std::fs; +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::fs::symlink; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +use sha2::{Digest, Sha256}; +use tempfile::TempDir; + +/// The version of the launcher under test, and the one a fake release is. +const HERE: &str = env!("CARGO_PKG_VERSION"); + +fn next_version() -> String { + let mut parts: Vec = HERE.split('.').map(|p| p.parse().unwrap()).collect(); + *parts.last_mut().unwrap() += 1; + parts + .iter() + .map(u64::to_string) + .collect::>() + .join(".") +} + +fn platform() -> &'static str { + match (std::env::consts::OS, std::env::consts::ARCH) { + ("macos", "aarch64") => "macos-arm64", + ("linux", "aarch64") => "linux-arm64-cuda", + other => panic!("no basert release for {other:?}"), + } +} + +fn superfluid_target() -> &'static str { + match (std::env::consts::OS, std::env::consts::ARCH) { + ("macos", "aarch64") => "aarch64-apple-darwin", + ("linux", "aarch64") => "aarch64-unknown-linux-gnu", + ("linux", "x86_64") => "x86_64-unknown-linux-gnu", + other => panic!("no superfluid release for {other:?}"), + } +} + +fn executable(path: &Path, text: &str) { + use std::os::unix::fs::PermissionsExt; + fs::write(path, text).unwrap(); + fs::set_permissions(path, fs::Permissions::from_mode(0o755)).unwrap(); +} + +fn sha256(path: &Path) -> String { + format!("{:x}", Sha256::digest(fs::read(path).unwrap())) +} + +/// The engine's files: a library with its link, a tool, the header stating `abi`. +fn engine_files(dir: &Path, version: &str, abi: u32) { + fs::create_dir_all(dir.join("include").join("baseRT")).unwrap(); + fs::write(dir.join(format!("libbaseRT.{version}.dylib")), version).unwrap(); + symlink( + format!("libbaseRT.{version}.dylib"), + dir.join("libbaseRT.dylib"), + ) + .unwrap(); + executable( + &dir.join("basert-chat"), + &format!("#!/bin/sh\necho chat {version}\n"), + ); + fs::write( + dir.join("include").join("baseRT").join("baseRT.h"), + format!("#pragma once\n#define BASERT_ABI_VERSION {abi}\n"), + ) + .unwrap(); + fs::write(dir.join("LICENSE"), "engine").unwrap(); +} + +/// A superfluid at `root` (its layout: bin/, libexec/superfluid/) that is +/// `version` and serves engine ABI `serves`: `runtimes --json` reports the +/// basert runtime ready when the header beside `$BASERT_LIB` states it. +fn superfluid_at(root: &Path, version: &str, serves: u32) { + fs::create_dir_all(root.join("bin")).unwrap(); + fs::create_dir_all(root.join("libexec").join("superfluid")).unwrap(); + executable( + &root.join("bin").join("superfluid"), + &format!( + "#!/bin/sh\n\ + if [ \"$1\" = --version ]; then echo \"superfluid {version}\"; exit 0; fi\n\ + abi=$(sed -n 's/^#define BASERT_ABI_VERSION //p' \"$BASERT_LIB/include/baseRT/baseRT.h\" 2>/dev/null)\n\ + if [ \"$abi\" = \"{serves}\" ]; then\n\ + echo '[{{\"id\":\"basert\",\"state\":\"ready\",\"ready\":true,\"abi\":{serves}}}]'\n\ + else\n\ + echo \"[{{\\\"id\\\":\\\"basert\\\",\\\"state\\\":\\\"not ready\\\",\\\"ready\\\":\\\"drives engine ABI {serves}, the engine states $abi\\\"}}]\"\n\ + fi\n" + ), + ); + executable( + &root + .join("libexec") + .join("superfluid") + .join("superfluid-worker-basert"), + "#!/bin/sh\n", + ); +} + +/// A listing's assets, each named by its path. +fn listed_assets(paths: &[&Path]) -> serde_json::Value { + paths + .iter() + .map(|p| { + serde_json::json!({ + "name": p.file_name().unwrap().to_str().unwrap(), + "browser_download_url": p.to_str().unwrap(), + }) + }) + .collect() +} + +struct Fixture { + tmp: TempDir, + root: PathBuf, + releases: Vec, + superfluid_releases: Vec, +} + +impl Fixture { + /// The launcher under test placed in `/.basert` as the install + /// script did before versions (flat, with a `.release` stamp), and a + /// superfluid beside it that is v0.1.3 and serves ABI 1. + fn in_place() -> Fixture { + let tmp = TempDir::new().unwrap(); + let root = tmp.path().join(".basert"); + fs::create_dir_all(&root).unwrap(); + let launcher = root.join("basert"); + let built = Path::new(env!("CARGO_BIN_EXE_basert")); + // Copied, as package_copy explains. + fs::copy(built, &launcher).unwrap(); + engine_files(&root, HERE, 1); + fs::write(root.join(".release"), format!("v{HERE}\n")).unwrap(); + superfluid_at(&root.join("superfluid"), "0.1.3", 1); + fs::write(root.join("superfluid").join(".release"), "v0.1.3\n").unwrap(); + fs::write(root.join("notes.txt"), "mine").unwrap(); + let mut f = Fixture { + tmp, + root, + releases: Vec::new(), + superfluid_releases: Vec::new(), + }; + f.superfluid_release("v0.1.3", 1); + f + } + + fn path(&self, rel: &str) -> PathBuf { + self.tmp.path().join(rel) + } + + /// A basert release `version` for this platform whose engine states `abi`. + fn release(&mut self, version: &str, abi: u32) { + let tree = self.path(&format!("build-{version}")); + fs::create_dir_all(&tree).unwrap(); + engine_files(&tree, version, abi); + executable( + &tree.join("basert"), + &format!("#!/bin/sh\n[ \"$1\" = --version ] && echo \"basert {version}\" && exit 0\necho \"basert {version}: $*\"\n"), + ); + let assets = self.path(&format!("assets-{version}")); + fs::create_dir_all(&assets).unwrap(); + let name = format!("basert-engine-{}-{version}.tar.gz", platform()); + let tarball = assets.join(&name); + assert!(Command::new("tar") + .arg("-czf") + .arg(&tarball) + .arg("-C") + .arg(&tree) + .arg(".") + .status() + .unwrap() + .success()); + let checksum = assets.join(format!("{name}.sha256")); + fs::write(&checksum, format!("{} {name}\n", sha256(&tarball))).unwrap(); + let marker = assets.join(format!("basert-engine-abi-{abi}.txt")); + fs::write(&marker, format!("{abi}\n")).unwrap(); + self.releases.push(serde_json::json!({ + "tag_name": format!("v{version}"), + "published_at": "2026-01-01T00:00:00Z", + "draft": false, + "prerelease": false, + "assets": listed_assets(&[&tarball, &checksum, &marker]), + })); + } + + /// A superfluid release `tag` that serves engine ABI `serves`. + fn superfluid_release(&mut self, tag: &str, serves: u32) { + let name = format!("superfluid-{}", superfluid_target()); + let tree = self.path(&format!("sf-build-{tag}")); + superfluid_at(&tree.join(&name), tag.trim_start_matches('v'), serves); + let assets = self.path(&format!("sf-assets-{tag}")); + fs::create_dir_all(&assets).unwrap(); + let tarball = assets.join(format!("{name}.tar.gz")); + assert!(Command::new("tar") + .arg("-czf") + .arg(&tarball) + .arg("-C") + .arg(&tree) + .arg(&name) + .status() + .unwrap() + .success()); + let checksum = assets.join(format!("{name}.tar.gz.sha256")); + fs::write(&checksum, format!("{} {name}.tar.gz\n", sha256(&tarball))).unwrap(); + let marker = assets.join(format!("superfluid-basert-abi-{serves}.txt")); + fs::write(&marker, format!("{serves}\n")).unwrap(); + self.superfluid_releases.push(serde_json::json!({ + "tag_name": tag, + "published_at": "2026-01-01T00:00:00Z", + "draft": false, + "prerelease": false, + "assets": listed_assets(&[&tarball, &checksum, &marker]), + })); + } + + /// `launcher self ` against this fixture's listings, home and state. + fn run(&self, launcher: &Path, args: &[&str]) -> (i32, String, String) { + let mut words = vec!["self"]; + words.extend_from_slice(args); + self.launch(launcher, &words) + } + + /// `launcher ` against this fixture's listings, home and state. + fn launch(&self, launcher: &Path, args: &[&str]) -> (i32, String, String) { + self.launch_with(launcher, args, &[]) + } + + /// [`Fixture::launch`] with `env` set as well. + fn launch_with( + &self, + launcher: &Path, + args: &[&str], + env: &[(&str, &Path)], + ) -> (i32, String, String) { + let out = self.command(launcher, args, env).output().unwrap(); + ( + out.status.code().unwrap_or(-1), + String::from_utf8_lossy(&out.stdout).into_owned(), + String::from_utf8_lossy(&out.stderr).into_owned(), + ) + } + + /// `launcher ` with this fixture's listings, home and state, and + /// `env`, to run or to start. + fn command(&self, launcher: &Path, args: &[&str], env: &[(&str, &Path)]) -> Command { + let listing = self.path("releases.json"); + fs::write(&listing, serde_json::to_string(&self.releases).unwrap()).unwrap(); + let sf_listing = self.path("superfluid-releases.json"); + fs::write( + &sf_listing, + serde_json::to_string(&self.superfluid_releases).unwrap(), + ) + .unwrap(); + let mut command = Command::new(launcher); + command + .args(args) + .env("HOME", self.path("home")) + .env("XDG_DATA_HOME", self.path("home/.local/share")) + .env("BASERT_STATE_DIR", self.path("state")) + .env("BASERT_RELEASES", &listing) + .env("BASERT_SUPERFLUID_RELEASES", &sf_listing) + .env("BASERT_NO_UPDATE_CHECK", "1") + .env_remove("BASERT_LIB") + .env_remove("BASERT_SUPERFLUID_DIR") + .env_remove("BASERT_SUPERFLUID_SOURCE") + .env_remove("BASERT_SUPERFLUID_AUTO_INSTALL") + .env_remove("BASERT_INSTALL_DIR") + .envs(env.iter().copied()); + command + } + + fn version_dir(&self, v: &str) -> PathBuf { + self.root.join("versions").join(v) + } + + fn current(&self) -> Option { + fs::read_link(self.root.join("versions").join("current")) + .ok() + .map(|t| t.to_string_lossy().into_owned()) + } + + fn says(&self, program: &str) -> String { + let out = Command::new(self.root.join(program)) + .arg("--version") + .output() + .unwrap(); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + fn superfluid_release_installed(&self) -> String { + fs::read_to_string(self.root.join("superfluid").join(".release")) + .unwrap() + .trim() + .to_string() + } + + /// The superfluid in basert's data directory under this fixture's home. + fn data_superfluid(&self) -> PathBuf { + let data = match std::env::consts::OS { + "macos" => self.path("home/Library/Application Support"), + _ => self.path("home/.local/share"), + }; + data.join("baseRT").join("superfluid") + } +} + +#[test] +fn an_in_place_install_updates_into_the_versions_layout_and_goes_back_and_forward() { + let mut f = Fixture::in_place(); + let next = next_version(); + f.release(HERE, 1); + f.release(&next, 1); + + let (code, out, err) = f.run(&f.root.join("basert"), &["update", "--dry-run"]); + assert_eq!(code, 0, "{err}"); + assert!(err.contains(&format!("basert {HERE} -> {next}")), "{err}"); + assert!(err.contains("dry run: nothing fetched or changed"), "{err}"); + assert!(f.current().is_none() && !f.version_dir(&next).exists()); + + let (code, out2, err) = f.run(&f.root.join("basert"), &["update"]); + assert_eq!(code, 0, "{out}{out2}{err}"); + assert!(out2.contains(&format!("basert {next} is in use")), "{out2}"); + assert!( + err.contains(&format!( + "{HERE} stays installed: `basert self use {HERE}` goes back to it" + )), + "{err}" + ); + assert_eq!(f.current().as_deref(), Some(next.as_str())); + // The in-place copy is the version to go back to, the launcher under test among it. + assert!(fs::symlink_metadata(f.version_dir(HERE).join("basert")) + .unwrap() + .is_file()); + assert_eq!( + fs::read_link(f.version_dir(HERE).join("libbaseRT.dylib")).unwrap(), + Path::new(&format!("libbaseRT.{HERE}.dylib")) + ); + // The root is links through `current`: what PATH runs is the new release. + assert_eq!( + fs::read_link(f.root.join("basert")).unwrap(), + Path::new("versions/current/basert") + ); + assert_eq!(f.says("basert"), format!("basert {next}")); + assert_eq!( + fs::read_link(f.root.join("libbaseRT.dylib")).unwrap(), + Path::new("versions/current/libbaseRT.dylib") + ); + assert!(fs::symlink_metadata(f.root.join(format!("libbaseRT.{HERE}.dylib"))).is_err()); + assert_eq!( + fs::read_to_string(f.root.join(".release")).unwrap().trim(), + format!("v{next}") + ); + // Not the release's: left alone. superfluid already served the engine. + assert_eq!( + fs::read_to_string(f.root.join("notes.txt")).unwrap(), + "mine" + ); + assert_eq!(f.superfluid_release_installed(), "v0.1.3"); + + // The launcher under test, now a version of the layout. + let old = f.version_dir(HERE).join("basert"); + let (code, out, err) = f.run(&old, &["list"]); + assert_eq!(code, 0, "{err}"); + assert!(out.contains(&format!("* {next} (in use)")), "{out}"); + assert!( + out.contains(&format!( + " {HERE} (the one before: `basert self use {HERE}` goes back to it)" + )), + "{out}" + ); + assert!(out.contains("superfluid v0.1.3 at "), "{out}"); + + let (code, out, err) = f.run(&old, &["use", HERE]); + assert_eq!(code, 0, "{err}"); + assert!(out.contains(&format!("basert {HERE} is in use")), "{out}"); + assert_eq!(f.current().as_deref(), Some(HERE)); + assert_eq!(f.says("basert"), format!("basert {HERE}")); + + // Forward again from the root: the release on disk is used as it is. + let (code, out, err) = f.run(&f.root.join("basert"), &["update"]); + assert_eq!(code, 0, "{out}{err}"); + assert!(err.contains(", already here)"), "{err}"); + assert!(!err.contains("downloading"), "{err}"); + assert_eq!(f.current().as_deref(), Some(next.as_str())); + + let (code, _, err) = f.run(&old, &["update"]); + assert_eq!(code, 0, "{err}"); + assert!( + err.contains(&format!("basert {next} is in use, the newest release")), + "{err}" + ); +} + +#[test] +fn a_superfluid_that_cannot_serve_the_new_engine_is_replaced_by_the_newest_that_can() { + let mut f = Fixture::in_place(); + let next = next_version(); + f.release(&next, 2); + f.superfluid_release("v0.1.4", 2); + f.superfluid_release("v0.1.5", 2); + + let (code, out, err) = f.run(&f.root.join("basert"), &["update"]); + assert_eq!(code, 0, "{out}{err}"); + assert_eq!(f.current().as_deref(), Some(next.as_str())); + assert_eq!(f.superfluid_release_installed(), "v0.1.5"); + assert!(err.contains("installed superfluid v0.1.5"), "{err}"); + + // Back to the ABI 1 engine: the newest superfluid for it is older, and is what it takes. + let (code, out, err) = f.run(&f.version_dir(HERE).join("basert"), &["use", HERE]); + assert_eq!(code, 0, "{out}{err}"); + assert_eq!(f.current().as_deref(), Some(HERE)); + assert_eq!(f.superfluid_release_installed(), "v0.1.3"); +} + +#[test] +fn with_no_superfluid_for_the_new_engine_nothing_changes() { + let mut f = Fixture::in_place(); + let next = next_version(); + f.release(&next, 2); + + let (code, _, err) = f.run(&f.root.join("basert"), &["update"]); + assert_eq!(code, 1, "{err}"); + assert!( + err.contains("no superfluid release states its ABI number 2 yet"), + "{err}" + ); + assert!( + err.contains(&format!( + "basert {next} was not made the one in use; {HERE} stays" + )), + "{err}" + ); + assert!(fs::symlink_metadata(f.root.join("basert")) + .unwrap() + .is_file()); + assert!(f.current().is_none()); + assert!(!f.version_dir(&next).exists()); + assert_eq!(f.superfluid_release_installed(), "v0.1.3"); +} + +/// `/.local` as superfluid's own install script fills it, which +/// `BASERT_SUPERFLUID_DIR` may name: superfluid `version` serving `serves` in +/// `bin/` and `libexec/superfluid/`, beside the user's other tools and files. +fn shared_prefix(f: &Fixture, version: &str, serves: u32) -> PathBuf { + let prefix = f.path("home/.local"); + superfluid_at(&prefix, version, serves); + executable(&prefix.join("bin").join("other-tool"), "#!/bin/sh\n"); + fs::create_dir_all(prefix.join("share").join("app")).unwrap(); + fs::write(prefix.join("share").join("app").join("data.txt"), "mine").unwrap(); + prefix +} + +fn prefix_untouched(prefix: &Path, version: &str) { + assert!(prefix.join("bin").join("other-tool").is_file()); + assert_eq!( + fs::read_to_string(prefix.join("share").join("app").join("data.txt")).unwrap(), + "mine" + ); + let out = Command::new(prefix.join("bin").join("superfluid")) + .arg("--version") + .output() + .unwrap(); + assert_eq!( + String::from_utf8_lossy(&out.stdout).trim(), + format!("superfluid {version}") + ); +} + +#[test] +fn a_superfluid_dir_that_holds_other_files_keeps_its_superfluid_when_a_newer_one_is_out() { + let mut f = Fixture::in_place(); + f.release(HERE, 1); + f.superfluid_release("v0.1.4", 1); + let prefix = shared_prefix(&f, "0.1.3", 1); + let (code, out, err) = f.launch_with( + &f.root.join("basert"), + &["self", "update"], + &[("BASERT_SUPERFLUID_DIR", prefix.as_path())], + ); + assert_eq!(code, 0, "{out}{err}"); + assert!( + err.contains("superfluid v0.1.4 is out, but v0.1.3 stays"), + "{err}" + ); + assert!(err.contains("which is not superfluid's"), "{err}"); + prefix_untouched(&prefix, "0.1.3"); +} + +#[test] +fn a_superfluid_dir_that_holds_other_files_is_not_replaced_for_a_new_engine() { + let mut f = Fixture::in_place(); + let next = next_version(); + f.release(&next, 2); + f.superfluid_release("v0.1.4", 2); + let prefix = shared_prefix(&f, "0.1.3", 1); + let (code, out, err) = f.launch_with( + &f.root.join("basert"), + &["self", "update"], + &[("BASERT_SUPERFLUID_DIR", prefix.as_path())], + ); + assert_eq!(code, 1, "{out}{err}"); + assert!(err.contains("which is not superfluid's"), "{err}"); + assert!( + err.contains(&format!( + "basert {next} was not made the one in use; {HERE} stays" + )), + "{err}" + ); + prefix_untouched(&prefix, "0.1.3"); + assert!(fs::symlink_metadata(f.root.join("basert")) + .unwrap() + .is_file()); + assert!(!f.version_dir(&next).exists()); +} + +#[test] +fn a_newer_superfluid_comes_as_well_when_basert_is_up_to_date() { + let mut f = Fixture::in_place(); + f.release(HERE, 1); + f.superfluid_release("v0.1.4", 1); + let (code, out, err) = f.run(&f.root.join("basert"), &["update"]); + assert_eq!(code, 0, "{out}{err}"); + assert!( + err.contains(&format!("basert {HERE} is in use, the newest release")), + "{err}" + ); + assert_eq!(f.superfluid_release_installed(), "v0.1.4"); + // Still in place: nothing of basert changed. + assert!(fs::symlink_metadata(f.root.join("basert")) + .unwrap() + .is_file()); +} + +#[test] +fn the_install_script_places_a_bundle_over_an_in_place_copy() { + let f = Fixture::in_place(); + let next = next_version(); + let bundle = f.path("stage"); + fs::create_dir_all(&bundle).unwrap(); + engine_files(&bundle, &next, 1); + executable( + &bundle.join("basert"), + &format!("#!/bin/sh\n[ \"$1\" = --version ] && echo \"basert {next}\"\n"), + ); + let launcher = Path::new(env!("CARGO_BIN_EXE_basert")); + let root = f.root.to_str().unwrap().to_string(); + let (code, out, err) = f.run( + launcher, + &[ + "install", + "--from", + bundle.to_str().unwrap(), + "--root", + &root, + ], + ); + assert_eq!(code, 0, "{out}{err}"); + assert!(!bundle.exists()); + assert_eq!(f.current().as_deref(), Some(next.as_str())); + assert_eq!(f.says("basert"), format!("basert {next}")); + assert!(fs::symlink_metadata(f.version_dir(HERE).join("basert")) + .unwrap() + .is_file()); + assert_eq!( + fs::read_to_string(f.root.join(".release")).unwrap().trim(), + format!("v{next}") + ); + assert_eq!( + fs::read_to_string(f.root.join("versions").join("previous")) + .unwrap() + .trim(), + HERE + ); +} + +#[test] +fn serve_from_the_layout_runs_the_superfluid_in_the_root_on_its_own_versions_engine() { + let mut f = Fixture::in_place(); + f.release(&next_version(), 1); + assert_eq!(f.run(&f.root.join("basert"), &["update"]).0, 0); + // Back to the launcher under test, which PATH then finds through the root's links. + let (code, _, err) = f.run(&f.version_dir(HERE).join("basert"), &["use", HERE]); + assert_eq!(code, 0, "{err}"); + let model = f.path("m.base"); + fs::write(&model, "").unwrap(); + let (code, out, err) = f.launch( + &f.root.join("basert"), + &["serve", model.to_str().unwrap(), "--print-command"], + ); + assert_eq!(code, 0, "{out}{err}"); + let root = fs::canonicalize(&f.root).unwrap(); + let superfluid = root.join("superfluid").join("bin").join("superfluid"); + assert!( + out.contains(&format!(" {} serve ", superfluid.display())), + "{out}" + ); + // The version's own directory by its real path: an update turns the + // root's links, never this. + let engine = root.join("versions").join(HERE); + assert!( + out.trim_end() + .ends_with(&format!("--basert-lib {}", engine.display())), + "{out}" + ); +} + +/// The launcher under test placed at `rel` under the fixture, as a package +/// manager places it (the release's files beside it, an `installed-by` file +/// when `named`), and a manager script at `manager` (relative too) that logs +/// its arguments and the first PATH entry to `/manager.log`. +fn package_copy(f: &Fixture, rel: &str, named: Option<&str>, manager: &str) -> PathBuf { + let dir = f.path(rel); + fs::create_dir_all(&dir).unwrap(); + let launcher = dir.join("basert"); + let built = Path::new(env!("CARGO_BIN_EXE_basert")); + // A copy, not a hard link: macOS may give a hard-linked program's path as + // any of its links, so one test's launcher would seem to run from another's. + fs::copy(built, &launcher).unwrap(); + engine_files(&dir, HERE, 1); + if let Some(line) = named { + fs::write(dir.join("installed-by"), format!("{line}\n")).unwrap(); + } + let script = f.path(manager); + fs::create_dir_all(script.parent().unwrap()).unwrap(); + executable( + &script, + &format!( + "#!/bin/sh\necho \"$*|${{PATH%%:*}}\" >> {}\n", + f.path("manager.log").display() + ), + ); + launcher +} + +fn manager_log(f: &Fixture) -> String { + fs::read_to_string(f.path("manager.log")).unwrap_or_default() +} + +#[test] +fn a_homebrew_copy_is_upgraded_by_its_own_brew_when_asked() { + let f = Fixture::in_place(); + let exe = package_copy( + &f, + "brew/Cellar/basert/0.4.3/libexec", + Some("Homebrew: brew upgrade basert"), + "brew/bin/brew", + ); + // No terminal and no --yes: named, not run. + let (code, _, err) = f.run(&exe, &["update"]); + assert_eq!(code, 1, "{err}"); + assert!( + err.contains("installed by Homebrew: run `brew upgrade basert`, or `basert self update --yes` to let basert run it"), + "{err}" + ); + assert_eq!(manager_log(&f), ""); + + let (code, _, err) = f.run(&exe, &["update", "--dry-run"]); + assert_eq!(code, 0, "{err}"); + assert!(err.contains("`self update` would run "), "{err}"); + assert!(err.contains("brew/bin/brew upgrade basert"), "{err}"); + + let (code, _, err) = f.run(&exe, &["update", "--yes"]); + assert_eq!(code, 0, "{err}"); + assert!( + manager_log(&f).starts_with("upgrade basert|"), + "{}", + manager_log(&f) + ); + + // A version is the manager's to choose. + let (code, _, err) = f.run(&exe, &["update", "--yes", "--version", "0.4.2"]); + assert_eq!(code, 1); + assert!(err.contains("which chooses its version"), "{err}"); + + // A server recorded on this engine: nothing is run. + let records = f.path("state").join("serving"); + fs::create_dir_all(&records).unwrap(); + let engine = fs::canonicalize(exe.parent().unwrap()).unwrap(); + fs::write( + records.join(std::process::id().to_string()), + format!("{}\n", engine.display()), + ) + .unwrap(); + let (code, _, err) = f.run(&exe, &["update", "--yes"]); + assert_eq!(code, 1); + assert!( + err.contains(&format!( + "a server runs on the engine in {} (pid {})", + engine.display(), + std::process::id() + )), + "{err}" + ); + fs::remove_file(records.join(std::process::id().to_string())).unwrap(); + + // Pinned: named, with the way out. + fs::create_dir_all(f.path("brew/var/homebrew/pinned")).unwrap(); + fs::write(f.path("brew/var/homebrew/pinned/basert"), "").unwrap(); + let (code, _, err) = f.run(&exe, &["update", "--yes"]); + assert_eq!(code, 1); + assert!( + err.contains("it is pinned in Homebrew: `brew unpin basert` first"), + "{err}" + ); + assert_eq!(manager_log(&f).lines().count(), 1); +} + +#[test] +fn a_global_npm_copy_is_updated_by_that_npm_with_its_bin_first_on_path() { + let f = Fixture::in_place(); + let exe = package_copy( + &f, + "node/lib/node_modules/@basecompute/basert-darwin-arm64/engine", + Some("npm: npm install --global @basecompute/basert@latest"), + "node/bin/npm", + ); + let (code, _, err) = f.run(&exe, &["update", "--yes"]); + assert_eq!(code, 0, "{err}"); + let bin = fs::canonicalize(f.path("node/bin")).unwrap(); + assert_eq!( + manager_log(&f).trim(), + format!( + "install --global @basecompute/basert@latest|{}", + bin.display() + ) + ); +} + +#[test] +fn a_projects_npm_copy_and_a_pip_copy_are_named_not_run() { + let f = Fixture::in_place(); + let exe = package_copy( + &f, + "app/node_modules/@basecompute/basert-darwin-arm64/engine", + Some("npm: npm install --global @basecompute/basert@latest"), + "app/node_modules/.bin/npm", + ); + let (code, _, err) = f.run(&exe, &["update", "--yes"]); + assert_eq!(code, 1); + assert!(err.contains("it is a project's dependency"), "{err}"); + let exe = package_copy( + &f, + "venv/lib/python3.12/site-packages/baseRT/_engine", + Some("pip: pip install --upgrade basert"), + "venv/bin/python", + ); + let (code, _, err) = f.run(&exe, &["update", "--yes"]); + assert_eq!(code, 1); + let python = fs::canonicalize(f.path("venv/bin/python")).unwrap(); + assert!( + err.contains(&format!( + "run `{} -m pip install --upgrade basert`", + python.display() + )), + "{err}" + ); + assert_eq!(manager_log(&f), ""); +} + +#[test] +fn superfluid_is_not_replaced_under_a_server_running_from_it() { + let f = Fixture::in_place(); + // A process run from basert's superfluid stands in for its server: a copy + // of the launcher, blocked reading a FIFO named as its release listing. (A + // copy of /bin/sleep is killed at its start on macOS.) + let program = f.root.join("superfluid/libexec/superfluid/sleeper"); + fs::copy(env!("CARGO_BIN_EXE_basert"), &program).unwrap(); + let fifo = f.path("never"); + assert!(Command::new("mkfifo") + .arg(&fifo) + .status() + .unwrap() + .success()); + let mut server = Command::new(&program) + .args(["self", "check", "--record"]) + .env("BASERT_RELEASES", &fifo) + .env("BASERT_STATE_DIR", f.path("state")) + .spawn() + .unwrap(); + std::thread::sleep(std::time::Duration::from_millis(300)); + assert!( + server.try_wait().unwrap().is_none(), + "the stand-in server is up" + ); + let (code, _, err) = f.launch( + &f.root.join("basert"), + &["superfluid", "install", "--force"], + ); + let _ = server.kill(); + let _ = server.wait(); + assert_eq!(code, 1, "{err}"); + assert!( + err.contains(&format!( + "(pid {}): stop it, then run this again", + server.id() + )), + "{err}" + ); + assert_eq!(f.superfluid_release_installed(), "v0.1.3"); +} + +#[test] +fn a_package_copy_installs_superfluid_in_the_data_directory_not_its_folder() { + let f = Fixture::in_place(); + let data = f.data_superfluid(); + for (rel, named, manager) in [ + ( + "brew/Cellar/basert/0.4.3/libexec", + Some("Homebrew: brew upgrade basert"), + "brew/bin/brew", + ), + ( + "node/lib/node_modules/@basecompute/basert-darwin-arm64/engine", + Some("npm: npm install --global @basecompute/basert@latest"), + "node/bin/npm", + ), + ( + "venv/lib/python3.12/site-packages/baseRT/_engine", + Some("pip: pip install --upgrade basert"), + "venv/bin/python", + ), + ("cargo/.cargo/bin", None, "cargo/.cargo/bin/cargo"), + ] { + let _ = fs::remove_dir_all(&data); + let exe = package_copy(&f, rel, named, manager); + let (code, out, err) = f.launch(&exe, &["superfluid", "install"]); + assert_eq!(code, 0, "{rel}: {out}{err}"); + assert!( + out.contains(&format!( + "superfluid v0.1.3 is installed at {}", + data.display() + )), + "{rel}: {out}" + ); + assert!(data.join("bin").join("superfluid").is_file(), "{rel}"); + assert_eq!( + fs::read_to_string(data.join(".release")).unwrap(), + "v0.1.3\n" + ); + assert!( + !exe.parent().unwrap().join("superfluid").exists(), + "{rel}: nothing in the package's folder" + ); + // Found there the next time: nothing is downloaded again. + let (code, out, err) = f.launch(&exe, &["superfluid", "install"]); + assert_eq!(code, 0, "{rel}: {out}{err}"); + assert!(!err.contains("downloading"), "{rel}: {err}"); + let (code, out, err) = f.launch(&exe, &["superfluid", "status"]); + assert_eq!(code, 0, "{rel}: {out}{err}"); + assert!( + out.contains(&format!( + "installed: v0.1.3 ({}, basert's data directory)", + data.display() + )), + "{rel}: {out}" + ); + } +} + +#[test] +fn a_superfluid_beside_a_package_copy_still_runs_and_its_successor_goes_to_the_data_directory() { + let f = Fixture::in_place(); + let exe = package_copy( + &f, + "brew/Cellar/basert/0.4.2/libexec", + Some("Homebrew: brew upgrade basert"), + "brew/bin/brew", + ); + // Where a basert of 0.4.2 put it. + let beside = exe.parent().unwrap().join("superfluid"); + superfluid_at(&beside, "0.1.3", 1); + fs::write(beside.join(".release"), "v0.1.3\n").unwrap(); + let data = f.data_superfluid(); + let model = f.path("m.base"); + fs::write(&model, "").unwrap(); + let model = model.to_str().unwrap(); + // The path as the launcher gives it: macOS keeps the one it was started + // by, Linux the real one. + let in_keg = "/brew/Cellar/basert/0.4.2/libexec/superfluid"; + + // Found and run: nothing downloaded, nothing in the data directory. + let (code, out, err) = f.launch(&exe, &["superfluid", "install"]); + assert_eq!(code, 0, "{out}{err}"); + assert!(out.contains("superfluid v0.1.3 is installed at "), "{out}"); + assert!(out.trim_end().ends_with(in_keg), "{out}"); + let (code, out, err) = f.launch(&exe, &["serve", model, "--print-command"]); + assert_eq!(code, 0, "{out}{err}"); + assert!( + out.contains(&format!("{in_keg}/bin/superfluid serve ")), + "{out}" + ); + let (code, out, err) = f.launch(&exe, &["serve", model]); + assert_eq!(code, 0, "{out}{err}"); + assert!(!err.contains("installing superfluid"), "{err}"); + assert!(!data.exists(), "{}", data.display()); + + // One that cannot serve the engine is left where it is; the release that + // can goes to the data directory, which is run from then on. + fs::remove_dir_all(&beside).unwrap(); + superfluid_at(&beside, "0.1.2", 2); + fs::write(beside.join(".release"), "v0.1.2\n").unwrap(); + let (code, out, err) = f.launch(&exe, &["serve", model]); + assert_eq!(code, 0, "{out}{err}"); + assert!( + err.contains(&format!( + "basert serve: installing superfluid v0.1.3 into {} (", + data.display() + )), + "{err}" + ); + assert_eq!( + fs::read_to_string(data.join(".release")).unwrap(), + "v0.1.3\n" + ); + assert_eq!( + fs::read_to_string(beside.join(".release")).unwrap(), + "v0.1.2\n" + ); + let (code, out, err) = f.launch(&exe, &["serve", model, "--print-command"]); + assert_eq!(code, 0, "{out}{err}"); + // Quoted when it holds a space, as macOS's Application Support does. + let binary = data.join("bin").join("superfluid").display().to_string(); + assert!( + out.contains(&format!(" {binary} serve ")) || out.contains(&format!(" '{binary}' serve ")), + "{out}" + ); + let (code, out, err) = f.launch(&exe, &["serve", model]); + assert_eq!(code, 0, "{out}{err}"); + assert!(!err.contains("installing superfluid"), "{err}"); +} + +#[test] +fn a_start_does_not_replace_the_superfluid_another_server_runs_from() { + let f = Fixture::in_place(); + // Two package copies share basert's data directory: this one's engine + // states ABI 1, and the superfluid there serves ABI 2, for another + // copy's engine, whose server runs from it. + let exe = package_copy( + &f, + "brew/Cellar/basert/0.4.3/libexec", + Some("Homebrew: brew upgrade basert"), + "brew/bin/brew", + ); + let data = f.data_superfluid(); + superfluid_at(&data, "0.1.4", 2); + fs::write(data.join(".release"), "v0.1.4\n").unwrap(); + // A stand-in for that server, as in + // superfluid_is_not_replaced_under_a_server_running_from_it. + let program = data.join("libexec/superfluid/sleeper"); + fs::copy(env!("CARGO_BIN_EXE_basert"), &program).unwrap(); + let fifo = f.path("never"); + assert!(Command::new("mkfifo") + .arg(&fifo) + .status() + .unwrap() + .success()); + let mut server = Command::new(&program) + .args(["self", "check", "--record"]) + .env("BASERT_RELEASES", &fifo) + .env("BASERT_STATE_DIR", f.path("state")) + .spawn() + .unwrap(); + std::thread::sleep(std::time::Duration::from_millis(300)); + assert!( + server.try_wait().unwrap().is_none(), + "the stand-in server is up" + ); + let model = f.path("m.base"); + fs::write(&model, "").unwrap(); + let (code, out, err) = f.launch(&exe, &["serve", model.to_str().unwrap()]); + let _ = server.kill(); + let _ = server.wait(); + assert_eq!(code, 1, "{out}{err}"); + assert!( + err.contains(&format!( + "a server runs from {} (pid {}): stop it, then run this again", + data.display(), + server.id() + )), + "{err}" + ); + assert_eq!( + fs::read_to_string(data.join(".release")).unwrap(), + "v0.1.4\n" + ); +} + +#[test] +fn a_cargo_copy_leaves_a_superfluid_cargo_installed_beside_it_alone() { + let f = Fixture::in_place(); + let exe = package_copy(&f, "cargo/.cargo/bin", None, "cargo/.cargo/bin/cargo"); + // `cargo install superfluid` puts its binary where basert would once have + // made a superfluid/ directory. + let theirs = exe.parent().unwrap().join("superfluid"); + executable(&theirs, "#!/bin/sh\necho superfluid 0.1.5\n"); + let (code, out, err) = f.launch(&exe, &["superfluid", "install"]); + assert_eq!(code, 0, "{out}{err}"); + assert!( + out.contains(&format!( + "superfluid v0.1.3 is installed at {}", + f.data_superfluid().display() + )), + "{out}" + ); + assert_eq!( + fs::read_to_string(&theirs).unwrap(), + "#!/bin/sh\necho superfluid 0.1.5\n" + ); +} + +/// A loopback HTTP server that answers every request with `body` once `go` +/// is sent: downloads held in flight. +fn held_download(body: Vec) -> (String, mpsc::Sender<()>) { + let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!( + "http://{}/superfluid.tar.gz", + listener.local_addr().unwrap() + ); + let (go, wait) = mpsc::channel::<()>(); + let open = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let opened = open.clone(); + std::thread::spawn(move || { + let _ = wait.recv(); + opened.store(true, std::sync::atomic::Ordering::SeqCst); + }); + std::thread::spawn(move || { + for stream in listener.incoming() { + let Ok(mut stream) = stream else { break }; + let (body, open) = (body.clone(), open.clone()); + std::thread::spawn(move || { + let mut request = [0u8; 4096]; + let _ = std::io::Read::read(&mut stream, &mut request); + while !open.load(std::sync::atomic::Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(10)); + } + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = stream.write_all(&body); + }); + } + }); + (url, go) +} + +/// The lines `child` writes to stderr, as they come. +fn stderr_lines(child: &mut Child) -> mpsc::Receiver { + let err = child.stderr.take().unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + for line in BufReader::new(err).lines().map_while(Result::ok) { + if tx.send(line).is_err() { + break; + } + } + }); + rx +} + +/// The first line holding one of `texts`, unless the stream ends (or 30 s +/// pass) first; every line read is kept in `seen`. +fn wait_for( + lines: &mpsc::Receiver, + texts: &[&str], + seen: &mut Vec, +) -> Option { + let deadline = Instant::now() + Duration::from_secs(30); + while let Ok(line) = lines.recv_timeout(deadline.saturating_duration_since(Instant::now())) { + seen.push(line.clone()); + if texts.iter().any(|t| line.contains(t)) { + return Some(line); + } + } + None +} + +/// `command` started with no stdin (a start then downloads without asking) +/// and no proxy, its output piped. +fn start(mut command: Command) -> Child { + command + .env_remove("HTTP_PROXY") + .env_remove("http_proxy") + .env_remove("ALL_PROXY") + .env_remove("all_proxy") + .env("NO_PROXY", "127.0.0.1") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap() +} + +/// A stand-in server run from `dir`: a copy of the launcher there, blocked +/// reading a FIFO named as its release listing (a copy of /bin/sleep is +/// killed at its start on macOS). +fn stand_in_server(f: &Fixture, dir: &Path) -> Child { + let program = dir.join("sleeper"); + fs::copy(env!("CARGO_BIN_EXE_basert"), &program).unwrap(); + let fifo = f.path("never"); + if !fifo.exists() { + assert!(Command::new("mkfifo") + .arg(&fifo) + .status() + .unwrap() + .success()); + } + let mut server = Command::new(&program) + .args(["self", "check", "--record"]) + .env("BASERT_RELEASES", &fifo) + .env("BASERT_STATE_DIR", f.path("state")) + .spawn() + .unwrap(); + std::thread::sleep(Duration::from_millis(300)); + assert!( + server.try_wait().unwrap().is_none(), + "the stand-in server is up" + ); + server +} + +#[test] +fn a_start_waits_for_an_install_into_the_shared_directory_and_runs_it() { + let mut f = Fixture::in_place(); + // The release comes over connections the test holds open. + let tarball = f.superfluid_releases[0]["assets"][0]["browser_download_url"] + .as_str() + .unwrap() + .to_string(); + let (url, go) = held_download(fs::read(&tarball).unwrap()); + f.superfluid_releases[0]["assets"][0]["browser_download_url"] = url.into(); + // Two package copies, which share basert's data directory. + let a = package_copy( + &f, + "brew/Cellar/basert/0.4.3/libexec", + Some("Homebrew: brew upgrade basert"), + "brew/bin/brew", + ); + let b = package_copy( + &f, + "node/lib/node_modules/@basecompute/basert-darwin-arm64/engine", + Some("npm: npm install --global @basecompute/basert@latest"), + "node/bin/npm", + ); + let model = f.path("m.base"); + fs::write(&model, "").unwrap(); + let data = f.data_superfluid(); + + let mut first = start(f.command(&a, &["superfluid", "install"], &[])); + let first_lines = stderr_lines(&mut first); + let mut seen = Vec::new(); + assert!( + wait_for(&first_lines, &["downloading superfluid v0.1.3"], &mut seen).is_some(), + "{seen:?}" + ); + // B starts while A's download is in flight: it waits for A's install + // instead of making one of its own beside it. + let mut second = start(f.command(&b, &["serve", model.to_str().unwrap()], &[])); + let second_lines = stderr_lines(&mut second); + let mut second_seen = Vec::new(); + let said = wait_for( + &second_lines, + &[ + "waiting for another basert installing superfluid into", + "downloading", + ], + &mut second_seen, + ); + go.send(()).unwrap(); + let first = first.wait_with_output().unwrap(); + let second_out = second.wait_with_output().unwrap(); + seen.extend(first_lines.try_iter()); + second_seen.extend(second_lines.try_iter()); + assert!( + said.is_some_and(|l| l.contains("waiting for another basert")), + "{second_seen:?}" + ); + assert!(first.status.success(), "{seen:?}"); + assert!(second_out.status.success(), "{second_seen:?}"); + // A's install serves B's engine too: B runs it, downloading nothing. + assert!( + !second_seen.iter().any(|l| l.contains("downloading")), + "{second_seen:?}" + ); + assert_eq!( + fs::read_to_string(data.join(".release")).unwrap(), + "v0.1.3\n" + ); +} + +#[test] +fn an_install_is_not_swapped_in_under_a_server_started_while_it_downloaded() { + let mut f = Fixture::in_place(); + let tarball = f.superfluid_releases[0]["assets"][0]["browser_download_url"] + .as_str() + .unwrap() + .to_string(); + let (url, go) = held_download(fs::read(&tarball).unwrap()); + f.superfluid_releases[0]["assets"][0]["browser_download_url"] = url.into(); + let a = package_copy( + &f, + "brew/Cellar/basert/0.4.3/libexec", + Some("Homebrew: brew upgrade basert"), + "brew/bin/brew", + ); + let data = f.data_superfluid(); + + let mut install = start(f.command(&a, &["superfluid", "install"], &[])); + let lines = stderr_lines(&mut install); + let mut seen = Vec::new(); + assert!( + wait_for(&lines, &["downloading superfluid v0.1.3"], &mut seen).is_some(), + "{seen:?}" + ); + // Meanwhile a superfluid lands there (another basert's, or one copied by + // hand) and a server starts from it. + superfluid_at(&data, "0.1.4", 2); + fs::write(data.join(".release"), "v0.1.4\n").unwrap(); + let mut server = stand_in_server(&f, &data.join("libexec").join("superfluid")); + go.send(()).unwrap(); + let out = install.wait_with_output().unwrap(); + seen.extend(lines.try_iter()); + let alive = server.try_wait().unwrap().is_none(); + let _ = server.kill(); + let _ = server.wait(); + assert_eq!(out.status.code(), Some(1), "{seen:?}"); + assert!( + seen.iter().any(|l| l.contains(&format!( + "a server runs from {} (pid {})", + data.display(), + server.id() + ))), + "{seen:?}" + ); + assert!(alive, "the server is still running"); + assert_eq!( + fs::read_to_string(data.join(".release")).unwrap(), + "v0.1.4\n" + ); +} + +/// superfluid 0.1.3 at `root` (over what is there) whose server is a +/// program run from elsewhere, cat blocked on the FIFO `$HOLD`, which no look +/// at what runs from the directory finds. +fn holding_superfluid(root: &Path) { + fs::create_dir_all(root.join("bin")).unwrap(); + executable( + &root.join("bin").join("superfluid"), + "#!/bin/sh\n\ + case \"$1\" in\n\ + --version) echo \"superfluid 0.1.3\" ;;\n\ + runtimes) echo '[{\"id\":\"basert\",\"state\":\"ready\",\"ready\":true,\"abi\":1}]' ;;\n\ + serve) exec cat \"$HOLD\" ;;\n\ + esac\n", + ); +} + +/// `launcher serve` of a model, with `env`, once it has become the server (the +/// same process: the cat of [`holding_superfluid`]). +fn serve_held(f: &Fixture, launcher: &Path, env: &[(&str, &Path)]) -> Child { + let hold = f.path("hold"); + if !hold.exists() { + assert!(Command::new("mkfifo") + .arg(&hold) + .status() + .unwrap() + .success()); + } + let model = f.path("m.base"); + fs::write(&model, "").unwrap(); + let mut env = env.to_vec(); + env.push(("HOLD", hold.as_path())); + let mut server = start(f.command(launcher, &["serve", model.to_str().unwrap()], &env)); + let deadline = Instant::now() + Duration::from_secs(30); + loop { + let comm = Command::new("ps") + .args(["-o", "comm=", "-p", &server.id().to_string()]) + .output() + .unwrap(); + if String::from_utf8_lossy(&comm.stdout) + .trim() + .ends_with("cat") + { + return server; + } + if let Some(status) = server.try_wait().unwrap() { + let out = server.wait_with_output().unwrap(); + panic!( + "the server ended ({status}): {}", + String::from_utf8_lossy(&out.stdout) + ); + } + assert!(Instant::now() < deadline, "the server never came up"); + std::thread::sleep(Duration::from_millis(100)); + } +} + +#[test] +fn a_server_basert_serve_started_keeps_its_superfluid_in_place() { + let f = Fixture::in_place(); + let a = package_copy( + &f, + "brew/Cellar/basert/0.4.3/libexec", + Some("Homebrew: brew upgrade basert"), + "brew/bin/brew", + ); + let data = f.data_superfluid(); + superfluid_at(&data, "0.1.3", 1); + fs::write(data.join(".release"), "v0.1.3\n").unwrap(); + holding_superfluid(&data); + let mut server = serve_held(&f, &a, &[]); + + let (code, out, err) = f.launch(&a, &["superfluid", "install", "--force"]); + let _ = server.kill(); + let _ = server.wait(); + assert_eq!(code, 1, "{out}{err}"); + assert!( + err.contains(&format!("a server runs from {}", data.display())), + "{err}" + ); + assert!(err.contains("one basert serve started"), "{err}"); + let said = Command::new(data.join("bin").join("superfluid")) + .arg("--version") + .output() + .unwrap(); + assert_eq!( + String::from_utf8_lossy(&said.stdout).trim(), + "superfluid 0.1.3" + ); +} + +#[test] +fn status_and_the_printed_command_name_the_superfluid_a_start_runs() { + let f = Fixture::in_place(); + let exe = package_copy( + &f, + "brew/Cellar/basert/0.4.2/libexec", + Some("Homebrew: brew upgrade basert"), + "brew/bin/brew", + ); + // The data directory's serves ABI 2, for another basert; the one a + // basert of 0.4.2 put beside this one serves its ABI 1 engine. + let data = f.data_superfluid(); + superfluid_at(&data, "0.1.4", 2); + fs::write(data.join(".release"), "v0.1.4\n").unwrap(); + let beside = exe.parent().unwrap().join("superfluid"); + superfluid_at(&beside, "0.1.3", 1); + fs::write(beside.join(".release"), "v0.1.3\n").unwrap(); + let in_keg = "/brew/Cellar/basert/0.4.2/libexec/superfluid"; + let model = f.path("m.base"); + fs::write(&model, "").unwrap(); + let model = model.to_str().unwrap(); + + let (code, out, err) = f.launch(&exe, &["superfluid", "status"]); + assert_eq!(code, 0, "{out}{err}"); + assert!(out.contains("installed: v0.1.3 ("), "{out}"); + assert!(out.contains(in_keg), "{out}"); + assert!(out.contains("engine: ready to serve"), "{out}"); + let (code, out, err) = f.launch(&exe, &["serve", model, "--print-command"]); + assert_eq!(code, 0, "{out}{err}"); + assert!( + out.contains(&format!("{in_keg}/bin/superfluid serve ")), + "{out}" + ); + let (code, out, err) = f.launch(&exe, &["serve", "--version"]); + assert_eq!(code, 0, "{out}{err}"); + assert!(out.contains("superfluid v0.1.3 ("), "{out}"); + assert!(out.contains(in_keg), "{out}"); + // And a start runs it, as before. + let (code, out, err) = f.launch(&exe, &["serve", model]); + assert_eq!(code, 0, "{out}{err}"); + assert!(!err.contains("installing superfluid"), "{err}"); +} + +/// A release bundle of this version holding the launcher under test, for +/// `self install --from`: `/`, whose engine states ABI 1. +fn real_bundle(f: &Fixture, name: &str) -> PathBuf { + let dir = f.path(name); + fs::create_dir_all(&dir).unwrap(); + fs::copy(env!("CARGO_BIN_EXE_basert"), dir.join("basert")).unwrap(); + engine_files(&dir, HERE, 1); + dir +} + +/// The fixture's install moved into the versions layout by `self install`, +/// with this version (the launcher under test) in use. +fn managed(f: &Fixture) { + let bundle = real_bundle(f, "bundle-here"); + let (code, out, err) = f.run( + &f.root.join("basert"), + &[ + "install", + "--from", + bundle.to_str().unwrap(), + "--root", + f.root.to_str().unwrap(), + ], + ); + assert_eq!(code, 0, "{out}{err}"); + assert_eq!(f.current().as_deref(), Some(HERE)); +} + +#[test] +fn a_flat_install_is_not_moved_under_a_server_running_on_it() { + let mut f = Fixture::in_place(); + let next = next_version(); + f.release(&next, 1); + holding_superfluid(&f.root.join("superfluid")); + let mut server = serve_held(&f, &f.root.join("basert"), &[]); + // `self update`, and the install script's `self install`. + let (update, _, update_err) = f.run(&f.root.join("basert"), &["update"]); + let bundle = real_bundle(&f, "bundle"); + let (install, _, install_err) = f.run( + &f.root.join("basert"), + &[ + "install", + "--from", + bundle.to_str().unwrap(), + "--root", + f.root.to_str().unwrap(), + ], + ); + let alive = server.try_wait().unwrap().is_none(); + let _ = server.kill(); + let _ = server.wait(); + // `self update` names the root by its real path, `self install` as given. + let said = |root: &Path| { + format!( + "a server runs on the engine in {} (pid {}), whose files this would replace: stop it, then run this again", + root.display(), + server.id() + ) + }; + assert_eq!(update, 1, "{update_err}"); + assert!( + update_err.contains(&said(&fs::canonicalize(&f.root).unwrap())), + "{update_err}" + ); + assert_eq!(install, 1, "{install_err}"); + assert!(install_err.contains(&said(&f.root)), "{install_err}"); + assert!(alive); + // Still the flat copy, the engine's files in place. + for name in ["basert", &format!("libbaseRT.{HERE}.dylib")] { + assert!( + fs::symlink_metadata(f.root.join(name)).unwrap().is_file(), + "{name}" + ); + } + assert!(f.current().is_none()); + assert!(!f.version_dir(&next).exists()); +} + +#[test] +fn reinstalling_the_version_a_server_runs_on_keeps_it_and_its_record() { + let f = Fixture::in_place(); + managed(&f); + holding_superfluid(&f.root.join("superfluid")); + let marker = f.version_dir(HERE).join("marker"); + fs::write(&marker, "the copy the server runs on").unwrap(); + let mut server = serve_held(&f, &f.root.join("basert"), &[]); + let record = f + .version_dir(HERE) + .join(".serving") + .join(server.id().to_string()); + assert!(record.is_file(), "{}", record.display()); + + let again = real_bundle(&f, "bundle-again"); + let (code, out, err) = f.run( + &f.root.join("basert"), + &[ + "install", + "--from", + again.to_str().unwrap(), + "--root", + f.root.to_str().unwrap(), + ], + ); + let kept = (marker.is_file(), record.is_file()); + let _ = server.kill(); + let _ = server.wait(); + assert_eq!(code, 0, "{out}{err}"); + assert!( + err.contains(&format!( + "basert {HERE} in {} stays as it is: a server runs on it (pid {})", + f.version_dir(HERE).display(), + server.id() + )), + "{err}" + ); + assert_eq!(kept, (true, true)); + assert_eq!(f.current().as_deref(), Some(HERE)); +} + +#[test] +fn a_server_on_the_engine_basert_lib_names_keeps_its_version() { + let mut f = Fixture::in_place(); + managed(&f); + let next = next_version(); + f.release(&next, 1); + f.release("0.0.1", 1); + holding_superfluid(&f.root.join("superfluid")); + let engine = f.version_dir(HERE); + let mut server = serve_held( + &f, + &f.root.join("basert"), + &[("BASERT_LIB", engine.as_path())], + ); + // Two switches away: it is neither the version in use nor the one before. + let launcher = engine.join("basert"); + let (first, out1, err1) = f.run(&launcher, &["update", "--version", &next]); + let (second, out2, err2) = f.run(&launcher, &["update", "--version", "0.0.1"]); + let kept = engine.join("basert").is_file(); + let _ = server.kill(); + let _ = server.wait(); + assert_eq!(first, 0, "{out1}{err1}"); + assert_eq!(second, 0, "{out2}{err2}"); + assert_eq!(f.current().as_deref(), Some("0.0.1")); + assert!(kept, "{}", engine.display()); +} diff --git a/base-convert/crates/base-hub/catalog.json b/base-convert/crates/base-hub/catalog.json index a813239..f24e8fe 100644 --- a/base-convert/crates/base-hub/catalog.json +++ b/base-convert/crates/base-hub/catalog.json @@ -1,6 +1,6 @@ { "schema": 1, - "updated": "2026-09-24", + "updated": "2026-10-30", "models": [ { "id": "basecompute/Llama-3.1-8B-Instruct", @@ -716,8 +716,10 @@ "revision": "main", "arch": "gemma4", "quant": "default-q4", - "size": 2941140992, - "sha256": "785c3527ab6d33e4d2a1976e63ecc2bf13ceb0cab5b00ce375fd9cec2a971bc0" + "size": 3095953408, + "sha256": "9f1af96cfa1e4b1e2402a5493266be78b4f1330abd9d301e0f90626ee3c0155d", + "speculator": "gemma4-mtp", + "speculator_version": "1" }, { "id": "basecompute/gemma-4-E2B-it", @@ -726,8 +728,10 @@ "revision": "main", "arch": "gemma4", "quant": "default-q8", - "size": 5334106112, - "sha256": "3e9a9c8629756b2513cf2de35e6a3c18604374085b999b7841c128d542af52d9" + "size": 5488918528, + "sha256": "511c4e05778de12646b622339286577223feed46577b7961b8a8cc26b961e3d5", + "speculator": "gemma4-mtp", + "speculator_version": "1" }, { "id": "basecompute/gemma-4-E2B-it", @@ -736,9 +740,11 @@ "revision": "main", "arch": "gemma4", "quant": "cuda-q4mix", - "size": 4298522624, - "sha256": "47e5da7e8d4d339bacdaf82b0550b5de53506fcbf1effd52e0fe8b680b68e214", - "backend": "cuda" + "size": 4453335040, + "sha256": "bef4c080b67e7dd64a55e44e109f58f32bb3afacc3f866b9decfa7b1af1730f7", + "backend": "cuda", + "speculator": "gemma4-mtp", + "speculator_version": "1" }, { "id": "basecompute/gemma-4-E2B-it", @@ -747,9 +753,11 @@ "revision": "main", "arch": "gemma4", "quant": "cuda-q8", - "size": 5350326272, - "sha256": "115c4ae330c2b4acd9dfc586a3ebe6a2080efb9de10f893f8d42eaf960a8e5d4", - "backend": "cuda" + "size": 5505138688, + "sha256": "62ed6171de32f6ab850c2e0d4d5b008e89acd85d546ba5d3dfead7e0e8ea3ac2", + "backend": "cuda", + "speculator": "gemma4-mtp", + "speculator_version": "1" }, { "id": "basecompute/gemma-4-E4B-it", @@ -758,8 +766,10 @@ "revision": "main", "arch": "gemma4", "quant": "default-q4", - "size": 4558422016, - "sha256": "68348e8a9d4d0a00f9fe9e3ea80835f9d45d2165e1fc407f68366bf72b64bb8f" + "size": 4714807296, + "sha256": "8b9ed714236cab79117bef245038085125a22bc94a827022be14e61af6ba87b2", + "speculator": "gemma4-mtp", + "speculator_version": "1" }, { "id": "basecompute/gemma-4-E4B-it", @@ -768,8 +778,10 @@ "revision": "main", "arch": "gemma4", "quant": "default-q8", - "size": 8297676800, - "sha256": "c293bd1e3479b293fab353783af6d96dc285d3b07b688034eb544e36c99c9ef7" + "size": 8454062080, + "sha256": "7765fb9d5ec1430df5e03e270b082e41bcc3aa17d0e7b47c5b0075b3c51380dc", + "speculator": "gemma4-mtp", + "speculator_version": "1" }, { "id": "basecompute/nomic-embed-text-v1.5", diff --git a/base-convert/crates/base-hub/src/catalog.rs b/base-convert/crates/base-hub/src/catalog.rs index 11c86e4..2af565a 100644 --- a/base-convert/crates/base-hub/src/catalog.rs +++ b/base-convert/crates/base-hub/src/catalog.rs @@ -84,15 +84,22 @@ pub struct CatalogEntry { #[serde(default, skip_serializing_if = "Option::is_none")] pub backend: Option, /// Speculative-decoding role. On a target bundle: the in-bundle - /// speculator it carries (`"mtp"` — `basertd --speculate mtp-head` - /// works out of the box). On a drafter sidecar row: its kind + /// speculator it carries (`"mtp"` — `superfluid serve --speculate + /// mtp-head` works out of the box). On a drafter sidecar row: its kind /// (`"dflash"` / `"dspark"` / `"eagle3"`), served with /// `--speculate :`. #[serde(default, skip_serializing_if = "Option::is_none")] pub speculator: Option, + /// Target rows with an in-bundle head: the head's version, when + /// `basert add-speculator --head-version` recorded one + /// (`speculator.config.head_version`). A client serving with + /// `--speculate` pulls the file again when its own head's version is + /// not this one. Absent: unversioned, only a missing head is stale. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub speculator_version: Option, /// Sidecar rows only: the catalog id of the target the drafter was /// trained for (it shares that target's embedding / lm_head and taps - /// its hidden states). `basertd --speculate auto` pairs an installed + /// its hidden states). superfluid's `--speculate auto` pairs an installed /// drafter with its target through this. #[serde(default, skip_serializing_if = "Option::is_none")] pub speculator_for: Option, @@ -146,6 +153,21 @@ impl Catalog { } } + /// The hosted catalog fetched past the cache, and written to it: for a + /// caller with reason to think the cache is behind the Hub — a pull whose + /// bytes fail the cached row's sha256, when a bundle's row and file changed + /// together and the cache, under its TTL, has not looked since. None + /// offline (`BASERT_CATALOG_OFFLINE`) or when the fetch fails; the caller + /// then keeps what it had. + pub fn load_fresh(cache_dir: &Path) -> Option { + if std::env::var_os("BASERT_CATALOG_OFFLINE").is_some() { + return None; + } + let c = Self::fetch(&catalog_url())?; + let _ = Self::write_cache(&cache_dir.join(CACHE_FILE), &c); + Some(c) + } + /// Parse the cache regardless of age, then fall back to the bundled copy, /// then to an empty catalog (load never fails). fn cache_or_bundled(cache: &Path) -> Self { diff --git a/base-convert/crates/base-hub/src/gen.rs b/base-convert/crates/base-hub/src/gen.rs index 8d93a7c..957f2a7 100644 --- a/base-convert/crates/base-hub/src/gen.rs +++ b/base-convert/crates/base-hub/src/gen.rs @@ -141,8 +141,9 @@ pub fn entry_from_header( backend, // The selectable strategy kind: a target's in-bundle head says so in // `speculator.config.kind` ("mtp"; its `arch` is model-specific, e.g. - // qwen35-mtp), a sidecar IS the drafter, so its own architecture is - // the kind (dflash / dspark / eagle3). + // qwen35-mtp; "gemma4-mtp" for Gemma 4's separate head embedded by + // `basert add-speculator`), a sidecar IS the drafter, so its own + // architecture is the kind (dflash / dspark / eagle3). speculator: header .speculator .as_ref() @@ -156,6 +157,12 @@ pub fn entry_from_header( "dflash" | "dspark" | "eagle3" => Some(header.arch.clone()), _ => None, }), + speculator_version: header.speculator.as_ref().and_then(|s| { + s.config + .get("head_version") + .and_then(|v| v.as_str()) + .map(str::to_string) + }), speculator_for: None, speculator_rank: None, }) diff --git a/base-convert/crates/base-hub/src/registry.rs b/base-convert/crates/base-hub/src/registry.rs index db31ebf..8f0d884 100644 --- a/base-convert/crates/base-hub/src/registry.rs +++ b/base-convert/crates/base-hub/src/registry.rs @@ -492,6 +492,15 @@ impl MergedRegistry { Ok(Self::new(root, catalog)) } + /// Build from the cache root + the hosted catalog fetched past the cache + /// ([`Catalog::load_fresh`]): what a pull turns to when the cached rows + /// look behind the Hub. None when it cannot be fetched. + pub fn load_fresh() -> Option { + let root = cache::models_dir().ok()?; + let catalog = Catalog::load_fresh(&root)?; + Some(Self::new(root, CatalogRegistry::from_catalog(catalog))) + } + /// Resolve an id to a concrete [`ModelRef`]. /// /// `want_quant` is the quant the caller is after (`q4`, `q8`, …); when set, diff --git a/base-convert/crates/base-hub/src/scan.rs b/base-convert/crates/base-hub/src/scan.rs index 021f468..e7f836d 100644 --- a/base-convert/crates/base-hub/src/scan.rs +++ b/base-convert/crates/base-hub/src/scan.rs @@ -752,6 +752,7 @@ mod tests { parts_sha256: None, backend: None, speculator: None, + speculator_version: None, speculator_for: None, speculator_rank: None, }], @@ -820,6 +821,7 @@ mod tests { parts_sha256: None, backend: None, speculator: None, + speculator_version: None, speculator_for: None, speculator_rank: None, }, @@ -836,6 +838,7 @@ mod tests { parts_sha256: None, backend: None, speculator: None, + speculator_version: None, speculator_for: None, speculator_rank: None, }, @@ -874,6 +877,7 @@ mod tests { parts_sha256: None, backend: None, speculator: Some("dflash".into()), + speculator_version: None, speculator_for: Some("basecompute/Qwen3-4B".into()), speculator_rank: Some(2), }, @@ -890,6 +894,7 @@ mod tests { parts_sha256: None, backend: None, speculator: None, + speculator_version: None, speculator_for: None, speculator_rank: None, }, @@ -1223,6 +1228,7 @@ mod tests { parts_sha256: None, backend: None, speculator: None, + speculator_version: None, speculator_for: None, speculator_rank: None, }], diff --git a/bindings/node/package.json b/bindings/node/package.json index 6e75d94..81847ab 100644 --- a/bindings/node/package.json +++ b/bindings/node/package.json @@ -1,6 +1,6 @@ { "name": "@baseRT/node", - "version": "0.4.2", + "version": "0.4.3", "private": true, "description": "Node.js bindings for BaseRT — LLM inference engine for Apple Silicon (Metal)", "main": "dist/index.js", diff --git a/bindings/python/baseRT/__init__.py b/bindings/python/baseRT/__init__.py index 92fb38e..ff1d180 100644 --- a/bindings/python/baseRT/__init__.py +++ b/bindings/python/baseRT/__init__.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union -__version__ = "0.4.2" +__version__ = "0.4.3" # --------------------------------------------------------------------------- # Library loading diff --git a/bindings/python/setup.py b/bindings/python/setup.py index e270336..aa83157 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -6,7 +6,7 @@ setup( name="baseRT", - version="0.4.2", + version="0.4.3", description="Python bindings for the BaseRT LLM inference engine (Apple Silicon / Metal)", long_description=long_description, long_description_content_type="text/markdown", diff --git a/bindings/rust/baseRT-sys/Cargo.toml b/bindings/rust/baseRT-sys/Cargo.toml index b132239..e370e74 100644 --- a/bindings/rust/baseRT-sys/Cargo.toml +++ b/bindings/rust/baseRT-sys/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT-sys" -version = "0.4.2" +version = "0.4.3" edition = "2021" description = "Raw FFI bindings for the BaseRT LLM inference engine" license = "Apache-2.0" diff --git a/bindings/rust/baseRT/Cargo.toml b/bindings/rust/baseRT/Cargo.toml index 3cd2300..bfe13af 100644 --- a/bindings/rust/baseRT/Cargo.toml +++ b/bindings/rust/baseRT/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "baseRT" -version = "0.4.2" +version = "0.4.3" edition = "2021" description = "Safe Rust bindings for the BaseRT LLM inference engine (Apple Silicon)" license = "Apache-2.0" diff --git a/bindings/swift/Sources/CBaseRT/include/baseRT.h b/bindings/swift/Sources/CBaseRT/include/baseRT.h index cc5bec9..44ccd85 100644 --- a/bindings/swift/Sources/CBaseRT/include/baseRT.h +++ b/bindings/swift/Sources/CBaseRT/include/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 2 +#define BASERT_VERSION_PATCH 3 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. diff --git a/include/baseRT/baseRT.h b/include/baseRT/baseRT.h index cc5bec9..44ccd85 100644 --- a/include/baseRT/baseRT.h +++ b/include/baseRT/baseRT.h @@ -64,7 +64,7 @@ extern "C" { #define BASERT_VERSION_MAJOR 0 #define BASERT_VERSION_MINOR 4 -#define BASERT_VERSION_PATCH 2 +#define BASERT_VERSION_PATCH 3 /// Compile-time version, packed as `(MAJOR<<16) | (MINOR<<8) | PATCH`. /// Useful for `#if BASERT_VERSION >= 0x000200` feature checks. diff --git a/include/baseRT/baseRT_tick.h b/include/baseRT/baseRT_tick.h index 9a0e73b..a2b26e7 100644 --- a/include/baseRT/baseRT_tick.h +++ b/include/baseRT/baseRT_tick.h @@ -1,4 +1,6 @@ -// baseRT_tick.h — the basertd engine ABI additions (basertd §10 items 1–3, 8). +// baseRT_tick.h — the tick engine ABI superfluid drives (designed for the +// retired basertd daemon: docs/design/basertd.md §10 items 1–3, 8; "basertd §N" +// below cites that document). // // Normative reference: docs/design/basertd-tick-schema.md (draft 7). // This header is the in-process ABI between a Rust engine-worker and the @@ -9,7 +11,7 @@ // Layout rules (normative): // - Every struct is 8-byte aligned with EXPLICIT padding; no field relies // on implicit compiler padding. Layouts are identical across the -// compilers we target; basertd-abi's layout probe test enforces parity +// compilers we target; superfluid-abi's layout probe test enforces parity // with the Rust #[repr(C)] mirrors. // - Top-level structs carry `struct_size` and grow append-only. // - Every record array is a BaseRTArray; decoding follows the four @@ -563,7 +565,7 @@ baseRT_status baseRT_tick(baseRT_model_t bundle, const BaseRTTickPlan *plan, con // --------------------------------------------------------------------------- // // The worker owns the shm mappings; the engine is handed base pointers. -// Ring memory layout (normative; basertd-shm's `SharedRing` is the +// Ring memory layout (normative; superfluid-shm's `SharedRing` is the // reference implementation and the interop test): // // [64-byte ring header] magic u64 = BASERT_RING_MAGIC ("BRTRING1" LE) diff --git a/packaging/npm/build-packages.mjs b/packaging/npm/build-packages.mjs index c7e7adc..f4c1692 100644 --- a/packaging/npm/build-packages.mjs +++ b/packaging/npm/build-packages.mjs @@ -101,7 +101,11 @@ try { throw new Error(`missing ${bundle} (bindings/node is at ${version}; has release v${version} been published?)`); } const dir = path.join(stage, p.name.split("/")[1]); - run(path.join(root, "packaging/stage-engine.sh"), [bundle, path.join(dir, "engine")]); + run(path.join(root, "packaging/stage-engine.sh"), [ + bundle, + path.join(dir, "engine"), + `npm: npm install --global ${SCOPE}/basert@latest`, + ]); fs.copyFileSync(path.join(dir, "engine/LICENSE"), path.join(dir, "LICENSE")); fs.writeFileSync( path.join(dir, "README.md"), diff --git a/packaging/python/build-wheel.sh b/packaging/python/build-wheel.sh index 8ea492d..a2e50c8 100755 --- a/packaging/python/build-wheel.sh +++ b/packaging/python/build-wheel.sh @@ -36,7 +36,7 @@ cp "$root"/bindings/python/baseRT/*.py "$stage/baseRT/" cp "$here/_cli.py" "$stage/baseRT/_cli.py" cp "$here/pyproject.toml" "$here/README.md" "$stage/" cp "$root/LICENSE" "$root/NOTICE" "$stage/" -"$root/packaging/stage-engine.sh" "$bundle" "$stage/baseRT/_engine" +"$root/packaging/stage-engine.sh" "$bundle" "$stage/baseRT/_engine" "pip: pip install --upgrade basert" tag=$(python3 "$here/platform_tag.py" "$stage/baseRT/_engine") diff --git a/packaging/stage-engine.sh b/packaging/stage-engine.sh index 467b449..ef7d309 100755 --- a/packaging/stage-engine.sh +++ b/packaging/stage-engine.sh @@ -3,16 +3,21 @@ # and the npm engine packages ship: every runtime file side by side, exactly as # install.sh lays out ~/.basert, minus the C headers. # -# packaging/stage-engine.sh basert-engine-macos-arm64-0.4.1.tar.gz out/engine +# packaging/stage-engine.sh basert-engine-macos-arm64-0.4.1.tar.gz out/engine [': '] +# +# The third argument, when given, is written beside the launcher as +# `installed-by`: `basert self update` then leaves the copy to that package +# manager and names the command, and the notice of a newer release names it too. # # Keeping the release layout matters: the `basert` launcher finds the # `basert-*` runtime tools next to itself, and those find their kernel files # (baseRT*.metallib / baseRT_cuda_kernels.fatbin) the same way. set -eu -[ $# -eq 2 ] || { echo "usage: $0 " >&2; exit 2; } +[ $# -eq 2 ] || [ $# -eq 3 ] || { echo "usage: $0 [': ']" >&2; exit 2; } bundle=$1 dest=$2 +installed_by=${3:-} mkdir -p "$dest" tar -xzf "$bundle" -C "$dest" @@ -64,3 +69,7 @@ esac python3 "$(dirname "$0")/check_engine_links.py" "$dest" [ -x "$dest/basert" ] || { echo "bundle has no executable basert launcher" >&2; exit 1; } +if [ -n "$installed_by" ]; then + case "$installed_by" in *": "*) ;; *) echo "installed-by must read ': ', not '$installed_by'" >&2; exit 2 ;; esac + printf '%s\n' "$installed_by" > "$dest/installed-by" +fi diff --git a/packaging/update-formula.sh b/packaging/update-formula.sh index 417fbe7..8e8a62a 100755 --- a/packaging/update-formula.sh +++ b/packaging/update-formula.sh @@ -63,6 +63,8 @@ class Basert < Formula # basert-* runtime tools next to itself, and they find their kernel files # the same way. bin/basert is a small script that runs libexec/basert. libexec.install Dir["*"] + # basert self update leaves this copy to Homebrew and names the command. + (libexec/"installed-by").write "Homebrew: brew upgrade basert\n" bin.write_exec_script libexec/"basert" # For the Rust / Python / Node bindings and C API users. lib.install_symlink Dir[libexec/"libbaseRT*"] diff --git a/tests/scripts/test_packaging.py b/tests/scripts/test_packaging.py index 4a840b2..0b83ccc 100644 --- a/tests/scripts/test_packaging.py +++ b/tests/scripts/test_packaging.py @@ -80,11 +80,13 @@ class StageEngineTests(unittest.TestCase): def setUp(self) -> None: self.tmp = Path(tempfile.mkdtemp()) - def stage(self, files: dict[str, bytes], links: dict[str, str]) -> tuple[subprocess.CompletedProcess, Path]: + def stage( + self, files: dict[str, bytes], links: dict[str, str], *extra: str + ) -> tuple[subprocess.CompletedProcess, Path]: tarball = self.tmp / "bundle.tar.gz" bundle(tarball, files, links) dest = self.tmp / "engine" - run = subprocess.run(["sh", str(STAGE), str(tarball), str(dest)], capture_output=True, text=True) + run = subprocess.run(["sh", str(STAGE), str(tarball), str(dest), *extra], capture_output=True, text=True) return run, dest def libs(self, dest: Path) -> dict[str, bytes]: @@ -114,6 +116,22 @@ def test_the_macos_chain_collapses_to_libbaseRT_dylib(self) -> None: self.assertEqual(self.libs(dest), {"libbaseRT.dylib": b"release"}) self.assertFalse((dest / "include").exists()) + def test_the_package_manager_that_installs_it_is_named_beside_the_launcher(self) -> None: + # `basert self update` reads it to leave the copy to that manager. + files = {"libbaseRT.0.4.1.dylib": b"release", "basert": macho([])} + links = {"libbaseRT.dylib": "libbaseRT.0.4.1.dylib"} + run, dest = self.stage(files, links, "pip: pip install --upgrade basert") + self.assertEqual(run.returncode, 0, run.stderr) + self.assertEqual((dest / "installed-by").read_text(), "pip: pip install --upgrade basert\n") + shutil.rmtree(dest) + run, dest = self.stage(files, links) + self.assertEqual(run.returncode, 0, run.stderr) + self.assertFalse((dest / "installed-by").exists()) + shutil.rmtree(dest) + run, _ = self.stage(files, links, "pip install --upgrade basert") + self.assertEqual(run.returncode, 2) + self.assertIn(": ", run.stderr) + def test_the_library_also_ships_under_the_name_it_records(self) -> None: # A program linked against the library asks for its install name # (SONAME on Linux) at startup, so that name is a second real file. @@ -369,6 +387,10 @@ def test_the_bindings_compile_on_a_machine_they_do_not_install_on(self) -> None: self.assertEqual((main["os"], main["cpu"]), (["darwin", "linux"], ["arm64"])) # Only the engine package this release publishes. self.assertEqual(main["optionalDependencies"], {"@basecompute/basert-darwin-arm64": v}) + # The engine names npm as what updates it, for `basert self update`. + with tarfile.open(out / f"basecompute-basert-darwin-arm64-{v}.tgz") as tar: + said = tar.extractfile("package/engine/installed-by").read().decode() + self.assertEqual(said, "npm: npm install --global @basecompute/basert@latest\n") # The bindings' own folder is left as it was. source = json.loads((self.root / "bindings" / "node" / "package.json").read_text()) self.assertEqual(source["os"], self.FOREIGN["os"])