From 449c4d4fb1be8b8b46dca266860ad9713d3e7f3d Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 12:39:15 +0200 Subject: [PATCH 01/46] fix(file): keep the image layout when writing a partition back write_partition ran `fallocate -d` on the whole image, which turned every zero-filled block into a hole. A bmap created afterwards with `-b` then skipped those blocks, e.g. ext4 journals that the image deliberately writes as zeros, so a bmap-based flash left the old journal data on the device. Write only the blocks that differ from the image instead. Unchanged blocks keep their allocation, and a block that changed to all zeros is now written too (`dd conv=sparse` skipped it and kept the old content). Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 2 +- Cargo.toml | 2 +- Dockerfile | 1 - project-context.md | 2 +- src/file/functions.rs | 68 ++++++++++++++++++++++++++++---------- tests/integration_tests.rs | 54 +++++++++++++++++++++++++++++- 6 files changed, 106 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d4efd9a..8b7d028d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1789,7 +1789,7 @@ dependencies = [ [[package]] name = "omnect-cli" -version = "0.28.2" +version = "0.28.3" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index c2c160d4..f2f9ebf2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0" name = "omnect-cli" readme = "README.md" repository = "https://github.com/omnect/omnect-cli" -version = "0.28.2" +version = "0.28.3" [dependencies] actix-web = "4.11" diff --git a/Dockerfile b/Dockerfile index 989ae539..8eb12929 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,6 @@ RUN < Result<()> { - let mut dd = Command::new("dd"); - dd.arg(format!("if={partition_file}")) - .arg(format!("of={image_file}")) - .arg("bs=512") - .arg(format!("seek={}", partition_info.start)) - .arg(format!("count={}", partition_info.count)) - .arg("conv=notrunc,sparse") - .arg("status=none"); - exec_cmd!(dd); - - let mut fallocate = Command::new("fallocate"); - fallocate.arg("-d").arg(image_file); - exec_cmd!(fallocate); - - let mut sync = Command::new("sync"); - exec_cmd!(sync); + let partition = fs::File::open(partition_file) + .context(format!("write_partition: cannot open {partition_file}"))?; + let image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(image_file) + .context(format!("write_partition: cannot open {image_file}"))?; + + let image_offset = partition_info.start * SECTOR_SIZE; + let len = partition + .metadata() + .context("write_partition: cannot get partition file size")? + .len() + .min(partition_info.count * SECTOR_SIZE); + + // Only write blocks that differ: unchanged blocks keep their allocation, so a bmap + // created from the result still maps every block the source image mapped. + let mut src = vec![0u8; COPY_CHUNK_SIZE]; + let mut dst = vec![0u8; COPY_CHUNK_SIZE]; + let mut pos = 0; + while pos < len { + let n = usize::try_from((len - pos).min(COPY_CHUNK_SIZE as u64))?; + partition + .read_exact_at(&mut src[..n], pos) + .context("write_partition: cannot read partition file")?; + image + .read_exact_at(&mut dst[..n], image_offset + pos) + .context("write_partition: cannot read image")?; + + let blocks = src[..n] + .chunks(COMPARE_BLOCK_SIZE) + .zip(dst[..n].chunks(COMPARE_BLOCK_SIZE)); + for (i, (s, d)) in blocks.enumerate() { + if s != d { + image + .write_all_at(s, image_offset + pos + (i * COMPARE_BLOCK_SIZE) as u64) + .context("write_partition: cannot write image")?; + } + } + pos += n as u64; + } - Ok(()) + image + .sync_all() + .context("write_partition: cannot sync image") } pub fn generate_bmap_file(image_file: &str) -> Result<()> { diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index ac3e7167..e51b00c0 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -5,7 +5,7 @@ use common::Testrunner; use httpmock::prelude::*; use omnect_cli::ssh; use predicates::prelude::*; -use std::{fs::create_dir_all, path::PathBuf}; +use std::{fs::create_dir_all, os::unix::fs::MetadataExt, path::PathBuf}; use stdext::function_name; #[test] @@ -895,6 +895,58 @@ fn check_file_copy(tr: Testrunner, partition: &str) { assert!(file_diff::diff(in_file4, out_file4)); } +// unit of `st_blocks` +const STAT_BLOCK_SIZE: u64 = 512; + +fn bmap_value(bmap: &str, tag: &str) -> u64 { + // the header comment of a bmap file mentions the tags too; only the closing tag is unique + let open_tag = format!("<{tag}>"); + let end = bmap + .find(&format!("")) + .expect("closing tag not found in bmap"); + let start = bmap[..end] + .rfind(&open_tag) + .expect("opening tag not found in bmap") + + open_tag.len(); + bmap[start..end] + .trim() + .parse() + .expect("tag value is no number") +} + +#[test] +fn check_file_copy_keeps_image_layout() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + let meta = image_path.metadata().expect("image metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE >= meta.len(), + "test image must be fully allocated" + ); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .assert() + .success(); + + // zero blocks of the input must stay mapped, e.g. zeroed ext4 journals + let bmap = std::fs::read_to_string(&bmap_path).expect("read bmap"); + assert_eq!( + bmap_value(&bmap, "MappedBlocksCount"), + bmap_value(&bmap, "BlocksCount") + ); +} + #[test] fn check_bmap_generation_wic() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); From 5186d4b34f36de75cfe5544fb8c8e95190746087 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:10:51 +0200 Subject: [PATCH 02/46] chore: bump dependencies including major versions - all direct dependencies to their latest version, omnect-crypto to 0.5.0 (create_cert_and_key no longer takes the unused extensions argument; the extensions of a certificate without CSR are the same) - actix-web without default features: the OAuth callback server only serves plain HTTP on localhost, and the http2 feature pulled in h2 0.3 (RUSTSEC-2026-0258, no fixed 0.3 release) - flate2 selects its rust_backend itself instead of relying on actix-web's compression features to enable one Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 1307 ++++++++++++++++++++++------------------------------ Cargo.toml | 26 +- src/lib.rs | 2 +- 3 files changed, 571 insertions(+), 764 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8b7d028d..a6f3a345 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "actix-codec" -version = "0.5.2" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7b0a21988c1bf877cf4759ef5ddaac04c1c9fe808c9142ecb78ba97d97a28a" +checksum = "4c13df95297bcf9014dc89162b0cc69431e192e34e3b419612fc124cfcd45dbf" dependencies = [ "bitflags", "bytes", @@ -21,51 +21,43 @@ dependencies = [ [[package]] name = "actix-http" -version = "3.12.1" +version = "3.18.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93acb4a42f64936f9b8cae4a433b237599dd6eb6ed06124eb67132ef8cc90662" +checksum = "e2ab7d72c0bb26f10df574ea560177b4de2f27ee8f421b2c1e9b75628bd9db23" dependencies = [ "actix-codec", - "actix-rt", "actix-service", "actix-utils", - "base64", "bitflags", - "brotli", "bytes", "bytestring", "derive_more", "encoding_rs", - "flate2", "foldhash", "futures-core", - "h2 0.3.27", "http 0.2.12", "httparse", "httpdate", "itoa", "language-tags", - "local-channel", "mime", "percent-encoding", "pin-project-lite", - "rand 0.10.1", - "sha1 0.11.0", "smallvec", "tokio", "tokio-util", "tracing", - "zstd", ] [[package]] name = "actix-macros" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01ed3140b2f8d422c68afa1ed2e85d996ea619c988ac834d255db32138655cb" +checksum = "367f814ad4afbac74f07df5001214da65f65e185c90ef56c4dd8df23f8695b9b" dependencies = [ + "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -77,7 +69,6 @@ dependencies = [ "bytestring", "cfg-if", "http 0.2.12", - "regex", "regex-lite", "serde", "tracing", @@ -85,9 +76,9 @@ dependencies = [ [[package]] name = "actix-rt" -version = "2.11.0" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92589714878ca59a7626ea19734f0e07a6a875197eec751bb5d3f99e64998c63" +checksum = "e5f794807f82bbd36430c12cd600c73bbab0f52fdde4f0ed49978df113f4807f" dependencies = [ "futures-core", "tokio", @@ -95,17 +86,16 @@ dependencies = [ [[package]] name = "actix-server" -version = "2.6.0" +version = "2.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a65064ea4a457eaf07f2fba30b4c695bf43b721790e9530d26cb6f9019ff7502" +checksum = "8db2cf6f034e227c0c79825fbf11f3cad48569109515f62542b84d8abedb9376" dependencies = [ "actix-rt", "actix-service", - "actix-utils", "futures-core", "futures-util", "mio", - "socket2 0.5.10", + "socket2", "tokio", "tracing", ] @@ -122,9 +112,9 @@ dependencies = [ [[package]] name = "actix-utils" -version = "3.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88a1dcdff1466e3c2488e1cb5c36a71822750ad43839937f85d2f4d9f8b705d8" +checksum = "0128396dd7313f697ad05b21b1a7be7d4cbb81888704f55996e4a27db196bb4d" dependencies = [ "local-waker", "pin-project-lite", @@ -132,9 +122,9 @@ dependencies = [ [[package]] name = "actix-web" -version = "4.13.0" +version = "4.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff87453bc3b56e9b2b23c1cc0b1be8797184accf51d2abe0f8a33ec275d316bf" +checksum = "bbacab3593b6b4f7be815076fc52d60a83c873426824675417e2abdd229e2e36" dependencies = [ "actix-codec", "actix-http", @@ -148,7 +138,6 @@ dependencies = [ "bytes", "bytestring", "cfg-if", - "cookie", "derive_more", "encoding_rs", "foldhash", @@ -161,13 +150,12 @@ dependencies = [ "mime", "once_cell", "pin-project-lite", - "regex", "regex-lite", "serde", "serde_json", "serde_urlencoded", "smallvec", - "socket2 0.6.4", + "socket2", "time", "tracing", "url", @@ -175,14 +163,14 @@ dependencies = [ [[package]] name = "actix-web-codegen" -version = "4.3.0" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f591380e2e68490b5dfaf1dd1aa0ebe78d84ba7067078512b4ea6e4492d622b8" +checksum = "b96b09c4878563f8ab4a5fd0c59f9f0d6e0e9f60eb9b748526a0b9604fd89c50" dependencies = [ "actix-router", "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -193,33 +181,18 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aho-corasick" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" dependencies = [ "memchr", ] -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e76a019e91224d279006ff972f1e984179a6e9feb050adba6ce8274aef23195" -dependencies = [ - "alloc-no-stdlib", -] - [[package]] name = "android_system_properties" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +checksum = "ae221649c9976a6f6c56ae1facf410f3ddb33cc661c4b7b61020a912d4237fbc" dependencies = [ "libc", ] @@ -232,9 +205,48 @@ checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "asn1-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] [[package]] name = "assert-json-diff" @@ -284,13 +296,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -311,6 +323,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bincode" version = "2.0.1" @@ -333,15 +351,15 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.13.0" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bitvec" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +checksum = "ddcec3d12c579d40898fe0a9a358a803c23e9c52ca3c425707f81c9436211837" dependencies = [ "funty", "radium", @@ -367,36 +385,15 @@ dependencies = [ "hybrid-array", ] -[[package]] -name = "brotli" -version = "8.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc91aac060a7a1e25823bdccbfb6af1875b88f17c6daac97894eed8207166b3" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "5.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a32acac15fe1967bc3986b2a6347dffc965602354ea6f450ad07e8bfd253583" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - [[package]] name = "bstr" -version = "1.12.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" dependencies = [ "memchr", "regex-automata", - "serde", + "serde_core", ] [[package]] @@ -405,11 +402,17 @@ version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + [[package]] name = "bytes" -version = "1.11.1" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" dependencies = [ "serde", ] @@ -434,32 +437,19 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.64" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f" +checksum = "f360145194ee8e21db5ee7f3fcd4fe52210864c75c985dae33218202c8bbe040" dependencies = [ "find-msvc-tools", - "jobserver", - "libc", "shlex", ] [[package]] name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "chacha20" -version = "0.10.0" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "rand_core 0.10.1", -] +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "chrono" @@ -477,9 +467,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.1" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -487,9 +477,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.0" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstyle", "clap_lex", @@ -497,21 +487,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.1" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "cmov" @@ -519,21 +509,6 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "const-oid" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" - [[package]] name = "convert_case" version = "0.10.0" @@ -543,17 +518,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "cookie" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" -dependencies = [ - "percent-encoding", - "time", - "version_check", -] - [[package]] name = "core-foundation" version = "0.10.1" @@ -570,6 +534,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -581,9 +551,9 @@ dependencies = [ [[package]] name = "cpufeatures" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" dependencies = [ "libc", ] @@ -605,18 +575,18 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-utils" -version = "0.8.21" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crypto-common" @@ -648,79 +618,98 @@ dependencies = [ [[package]] name = "darling" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ - "darling_core 0.20.11", - "darling_macro 0.20.11", + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core 0.21.3", - "darling_macro 0.21.3", + "darling_core 0.23.0", + "darling_macro 0.23.0", ] [[package]] name = "darling_core" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" dependencies = [ "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.119", ] [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", "strsim", - "syn", + "syn 2.0.119", ] [[package]] name = "darling_macro" -version = "0.20.11" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ - "darling_core 0.20.11", + "darling_core 0.21.3", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core 0.21.3", + "darling_core 0.23.0", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "data-encoding" -version = "2.11.0" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "der" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" + +[[package]] +name = "der-parser" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] [[package]] name = "deranged" @@ -747,7 +736,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.119", "unicode-xid", ] @@ -774,7 +763,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", - "const-oid", "crypto-common 0.2.2", "ctutils", ] @@ -802,38 +790,44 @@ dependencies = [ [[package]] name = "displaydoc" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "encoding_rs" -version = "0.8.35" +version = "0.8.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +checksum = "8e985e0451871ad22fb8d2b6b076e2028a502a0d3950998c2c5c0a4f9b5d9679" dependencies = [ "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] name = "env_filter" -version = "1.0.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32e90c2accc4b07a8456ea0debdc2e7587bdd890680d71173a15d4ae604f6eef" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" dependencies = [ "log", + "regex", ] [[package]] name = "env_logger" -version = "0.11.10" +version = "0.11.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0621c04f2196ac3f488dd583365b9c09be011a4ab8b9f37248ffcc8f6198b56a" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" dependencies = [ "env_filter", "log", @@ -857,11 +851,10 @@ dependencies = [ [[package]] name = "event-listener" -version = "5.4.1" +version = "5.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" dependencies = [ - "concurrent-queue", "parking", "pin-project-lite", ] @@ -878,9 +871,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "file_diff" @@ -911,15 +904,15 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.9" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "flate2" -version = "1.1.9" +version = "1.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" dependencies = [ "crc32fast", "miniz_oxide", @@ -942,9 +935,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "foreign-types" @@ -978,41 +971,41 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures-channel" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" dependencies = [ "futures-core", ] [[package]] name = "futures-core" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" [[package]] name = "futures-macro" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "futures-sink" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "futures-task" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" [[package]] name = "futures-timer" @@ -1022,9 +1015,9 @@ checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968" [[package]] name = "futures-util" -version = "0.3.32" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" dependencies = [ "futures-core", "futures-macro", @@ -1058,28 +1051,13 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "libc", - "r-efi 5.3.0", - "wasip2", -] - -[[package]] -name = "getrandom" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi 6.0.0", - "rand_core 0.10.1", - "wasip2", - "wasip3", + "r-efi", ] [[package]] @@ -1089,41 +1067,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13bd684d0cdb92e5993c63ef59bb8d1e959ff32ae131e5614fc143612f8544cc" dependencies = [ "crc", - "thiserror 2.0.18", + "thiserror 2.0.21", "wincode", ] [[package]] name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 0.2.12", - "indexmap", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "h2" -version = "0.4.15" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", "fnv", "futures-core", "futures-sink", - "http 1.4.2", + "http 1.5.0", "indexmap", "slab", "tokio", @@ -1131,15 +1090,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hashbrown" -version = "0.15.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" -dependencies = [ - "foldhash", -] - [[package]] name = "hashbrown" version = "0.17.1" @@ -1148,17 +1098,17 @@ checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" [[package]] name = "headers" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" +checksum = "fc2700e3fe555c3310aa7286cac6167449f2c87e0eb58769c9208a1c58a1d106" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "headers-core", - "http 1.4.2", + "http 1.5.0", "httpdate", "mime", - "sha1 0.10.6", + "sha1", ] [[package]] @@ -1167,7 +1117,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http 1.4.2", + "http 1.5.0", ] [[package]] @@ -1178,9 +1128,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" [[package]] name = "hmac" @@ -1204,9 +1154,9 @@ dependencies = [ [[package]] name = "http" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" dependencies = [ "bytes", "itoa", @@ -1214,23 +1164,23 @@ dependencies = [ [[package]] name = "http-body" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" dependencies = [ "bytes", - "http 1.4.2", + "http 1.5.0", ] [[package]] name = "http-body-util" -version = "0.1.3" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +checksum = "23169fe34a5fbcdd3f3862e78fb9b6fccd5f02a6dc6f732547005d45631ce71c" dependencies = [ "bytes", "futures-core", - "http 1.4.2", + "http 1.5.0", "http-body", "pin-project-lite", ] @@ -1256,14 +1206,14 @@ dependencies = [ "assert-json-diff", "async-object-pool", "async-trait", - "base64", + "base64 0.22.1", "bytes", "crossbeam-utils", "form_urlencoded", "futures-timer", "futures-util", "headers", - "http 1.4.2", + "http 1.5.0", "http-body-util", "hyper", "hyper-util", @@ -1275,7 +1225,7 @@ dependencies = [ "similar", "stringmetrics", "tabwriter", - "thiserror 2.0.18", + "thiserror 2.0.21", "tokio", "tracing", "url", @@ -1283,25 +1233,25 @@ dependencies = [ [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ "typenum", ] [[package]] name = "hyper" -version = "1.10.1" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55281c53a1894c864990125767da440a4e630446785086f52523b20033b74498" +checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" dependencies = [ "atomic-waker", "bytes", "futures-channel", "futures-core", - "h2 0.4.15", - "http 1.4.2", + "h2", + "http 1.5.0", "http-body", "httparse", "httpdate", @@ -1330,22 +1280,23 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-channel", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body", + "httparse", "hyper", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.4", + "socket2", "tokio", "tower-service", "tracing", @@ -1377,9 +1328,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" dependencies = [ "displaydoc", "potential_utf", @@ -1391,9 +1342,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" dependencies = [ "displaydoc", "litemap", @@ -1404,9 +1355,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -1418,16 +1369,17 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" [[package]] name = "icu_properties" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" dependencies = [ + "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", @@ -1438,15 +1390,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" [[package]] name = "icu_provider" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" dependencies = [ "displaydoc", "icu_locale_core", @@ -1457,12 +1409,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "id-arena" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" - [[package]] name = "ident_case" version = "1.0.1" @@ -1492,27 +1438,25 @@ dependencies = [ [[package]] name = "impl-more" -version = "0.1.9" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a5a9a0ff0086c7a148acb942baaabeadf9504d10400b5a05645853729b9cd2" +checksum = "30c0cddce6b7505483307994f60d97c4b156020e1504d0cf1f05a21b1b325e64" [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown 0.17.1", - "serde", - "serde_core", + "hashbrown", ] [[package]] name = "ipnet" -version = "2.12.0" +version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" +checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" [[package]] name = "is-docker" @@ -1539,21 +1483,11 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" -[[package]] -name = "jobserver" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" -dependencies = [ - "getrandom 0.3.4", - "libc", -] - [[package]] name = "js-sys" -version = "0.3.102" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -1576,10 +1510,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" [[package]] -name = "leb128fmt" -version = "0.1.0" +name = "lazy_static" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libbz2-rs-sys" @@ -1589,9 +1523,9 @@ checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libfs" @@ -1604,15 +1538,15 @@ dependencies = [ "linux-raw-sys", "log", "rustix", - "thiserror 2.0.18", + "thiserror 2.0.21", "xattr", ] [[package]] name = "libredox" -version = "0.1.17" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02ab6bace2054fb888a3c16f990117b579d14a3088e472d63c6011fa185c9d3" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ "libc", ] @@ -1625,20 +1559,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "litemap" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" - -[[package]] -name = "local-channel" -version = "0.1.5" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6cbc85e69b8df4b8bb8b89ec634e7189099cea8927a276b7384ce5488e53ec8" -dependencies = [ - "futures-core", - "futures-sink", - "local-waker", -] +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" [[package]] name = "local-waker" @@ -1657,9 +1580,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.32" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lzma-sys" @@ -1682,14 +1605,14 @@ dependencies = [ "bitvec", "serde", "serde-big-array", - "thiserror 2.0.18", + "thiserror 2.0.21", ] [[package]] name = "memchr" -version = "2.8.2" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "mime" @@ -1697,11 +1620,17 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" -version = "0.8.9" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", "simd-adler32", @@ -1709,9 +1638,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.1" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02bd0af71c67b473010cbbc60715ee815645a4dc942899111f494b4b737d6fda" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" dependencies = [ "libc", "log", @@ -1719,6 +1648,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "native-tls" version = "0.2.18" @@ -1736,18 +1671,47 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1773,11 +1737,11 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "getrandom 0.2.17", - "http 1.4.2", - "rand 0.8.6", + "http 1.5.0", + "rand", "reqwest 0.12.28", "serde", "serde_json", @@ -1787,6 +1751,15 @@ dependencies = [ "url", ] +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs", +] + [[package]] name = "omnect-cli" version = "0.28.3" @@ -1795,7 +1768,7 @@ dependencies = [ "anyhow", "assert-json-diff", "assert_cmd", - "base64", + "base64 0.23.1", "bzip2", "clap", "data-encoding", @@ -1817,7 +1790,7 @@ dependencies = [ "open", "predicates", "regex", - "reqwest 0.13.4", + "reqwest 0.13.5", "ring", "serde", "serde_json", @@ -1839,13 +1812,17 @@ dependencies = [ [[package]] name = "omnect-crypto" -version = "0.4.0" -source = "git+https://github.com/omnect/omnect-crypto.git?tag=0.4.0#2cf5fb05c514c9a8d3d7c4f2cdd4d4eeaaeef485" +version = "0.5.0" +source = "git+https://github.com/omnect/omnect-crypto.git?tag=0.5.0#128777f9946d9a6d31546da36fd500a700ecdee9" dependencies = [ "anyhow", + "byteorder", + "der", + "der-parser", "log", "openssl", "openssl-sys", + "x509-parser", ] [[package]] @@ -1856,13 +1833,12 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "open" -version = "5.3.5" +version = "5.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbaa89d2ddc8473c78a3adf69eea8cffa28c483b8e02a971ef31527cd0fc92c" +checksum = "aa576c76302b7b808eecc68061e67336c47833ef9d22caa74dda10fa9675eebc" dependencies = [ "is-wsl", "libc", - "pathdiff", ] [[package]] @@ -1887,7 +1863,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -1952,12 +1928,6 @@ dependencies = [ "smallvec", ] -[[package]] -name = "pathdiff" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" - [[package]] name = "percent-encoding" version = "2.3.2" @@ -1972,15 +1942,15 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pkg-config" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "potential_utf" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" dependencies = [ "zerovec", ] @@ -2031,61 +2001,45 @@ dependencies = [ ] [[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - -[[package]] -name = "proc-macro-error-attr2" -version = "2.0.0" +name = "proc-macro-error-attr3" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +checksum = "9e564d14133360e1ae169ffde5da25881b5fa47261665b8e5713c212c27799da" dependencies = [ "proc-macro2", "quote", ] [[package]] -name = "proc-macro-error2" -version = "2.0.1" +name = "proc-macro-error3" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +checksum = "8f0d4471b3436c22106b21913b1dda531558918ae9b7ec55d58aa84b43552233" dependencies = [ - "proc-macro-error-attr2", + "proc-macro-error-attr3", "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "proc-macro2" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.45" +version = "1.0.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" dependencies = [ "proc-macro2", ] -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - [[package]] name = "r-efi" version = "6.0.0" @@ -2100,24 +2054,13 @@ checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" [[package]] name = "rand" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" dependencies = [ "libc", "rand_chacha", - "rand_core 0.6.4", -] - -[[package]] -name = "rand" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" -dependencies = [ - "chacha20", - "getrandom 0.4.2", - "rand_core 0.10.1", + "rand_core", ] [[package]] @@ -2127,7 +2070,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.4", + "rand_core", ] [[package]] @@ -2139,12 +2082,6 @@ dependencies = [ "getrandom 0.2.17", ] -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - [[package]] name = "redox_syscall" version = "0.5.18" @@ -2156,20 +2093,19 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +checksum = "60dc65c0ff1a7ae1294b0c67b9f14baf70b644404010370171787bfac1038fc0" dependencies = [ - "getrandom 0.2.17", "libredox", - "thiserror 2.0.18", + "thiserror 2.0.21", ] [[package]] name = "regex" -version = "1.12.4" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" dependencies = [ "aho-corasick", "memchr", @@ -2179,9 +2115,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.14" +version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" dependencies = [ "aho-corasick", "memchr", @@ -2206,10 +2142,10 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", - "http 1.4.2", + "http 1.5.0", "http-body", "http-body-util", "hyper", @@ -2238,14 +2174,14 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029" dependencies = [ - "base64", + "base64 0.23.1", "bytes", "futures-core", - "http 1.4.2", + "http 1.5.0", "http-body", "http-body-util", "hyper", @@ -2294,11 +2230,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ "bitflags", "errno", @@ -2309,18 +2254,18 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" dependencies = [ "zeroize", ] [[package]] name = "rustversion" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" [[package]] name = "ryu" @@ -2374,9 +2319,9 @@ checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" dependencies = [ "serde_core", "serde_derive", @@ -2393,29 +2338,29 @@ dependencies = [ [[package]] name = "serde_core" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.228" +version = "1.0.229" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "serde_json" -version = "1.0.150" +version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ "itoa", "memchr", @@ -2468,26 +2413,15 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" dependencies = [ "cfg-if", "cpufeatures 0.2.17", "digest 0.10.7", ] -[[package]] -name = "sha1" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "digest 0.11.3", -] - [[package]] name = "sha2" version = "0.10.9" @@ -2506,7 +2440,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" dependencies = [ "cfg-if", - "cpufeatures 0.3.0", + "cpufeatures 0.3.1", "digest 0.11.3", ] @@ -2528,9 +2462,15 @@ dependencies = [ [[package]] name = "simd-adler32" -version = "0.3.9" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "simdutf8" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "similar" @@ -2546,25 +2486,15 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" [[package]] name = "socket2" -version = "0.5.10" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "socket2" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", "windows-sys 0.61.2", @@ -2609,14 +2539,36 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.119", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", ] [[package]] name = "syn" -version = "2.0.117" +version = "2.0.119" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -2634,13 +2586,25 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.2" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + +[[package]] +name = "synstructure" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -2676,7 +2640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.2", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2699,11 +2663,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.18", + "thiserror-impl 2.0.21", ] [[package]] @@ -2714,25 +2678,25 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "time" -version = "0.3.49" +version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711a53c2d47bbd818258c498c8dbfe186a2526c631495cfe7e078567f86b8469" +checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" dependencies = [ "deranged", "num-conv", @@ -2750,9 +2714,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.29" +version = "0.2.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c652a3727a9cbb9a02f707f530b618ce00d0ccd762009c8c23bd191df3c17d" +checksum = "7e689342a48d2ea927c87ea50cabf8594854bf940e9310208848d680d668ed85" dependencies = [ "num-conv", "time-core", @@ -2760,9 +2724,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" dependencies = [ "displaydoc", "zerovec", @@ -2770,9 +2734,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.52.3" +version = "1.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" dependencies = [ "bytes", "libc", @@ -2780,20 +2744,20 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.4", + "socket2", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.7.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] @@ -2808,22 +2772,23 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.18" +version = "0.7.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" dependencies = [ "bytes", "futures-core", "futures-sink", + "libc", "pin-project-lite", "tokio", ] [[package]] name = "toml" -version = "1.1.2+spec-1.1.0" +version = "1.1.6+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81f3d15e84cbcd896376e6730314d59fb5a87f31e4b038454184435cd57defee" +checksum = "920602543f0911ab71da12c50d59701da54c196d1a2bf5cb4b75667f137a406a" dependencies = [ "serde_core", "serde_spanned", @@ -2843,9 +2808,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.2+spec-1.1.0" +version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ "winnow", ] @@ -2874,7 +2839,7 @@ dependencies = [ "bitflags", "bytes", "futures-util", - "http 1.4.2", + "http 1.5.0", "http-body", "pin-project-lite", "tower", @@ -2915,7 +2880,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -2941,9 +2906,9 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "unicode-segmentation" @@ -2996,21 +2961,20 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.23.3" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144d6b123cef80b301b8f72a9e2ca4370ddec21950d0a103dd22c437006d2db7" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ - "getrandom 0.4.2", + "getrandom 0.4.3", ] [[package]] name = "validator" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43fb22e1a008ece370ce08a3e9e4447a910e92621bb49b85d6e48a45397e7cfa" +checksum = "c3d68c6633c483df6780cc5277a417c7c2d1bceee2649d06c8ab6b0fd2dd3c81" dependencies = [ "idna", - "once_cell", "regex", "serde", "serde_derive", @@ -3021,16 +2985,15 @@ dependencies = [ [[package]] name = "validator_derive" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7df16e474ef958526d1205f6dda359fdfab79d9aa6d54bafcb92dcd07673dca" +checksum = "240e4b81c20a1d6d50d1d7265c658dfbd204e8b9ac4d80f3c931f39462196335" dependencies = [ - "darling 0.20.11", - "once_cell", - "proc-macro-error2", + "darling 0.23.0", + "proc-macro-error3", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3075,29 +3038,11 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen 0.57.1", -] - -[[package]] -name = "wasip3" -version = "0.4.0+wasi-0.3.0-rc-2026-01-06" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" -dependencies = [ - "wit-bindgen 0.51.0", -] - [[package]] name = "wasm-bindgen" -version = "0.2.125" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -3108,19 +3053,20 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.75" +version = "0.4.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +checksum = "3cbab34de2d982e9b48e18d216d04c4a6f641066ff19ffb699980f591ee3610e" dependencies = [ "js-sys", + "tokio", "wasm-bindgen", ] [[package]] name = "wasm-bindgen-macro" -version = "0.2.125" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3128,65 +3074,31 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.125" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 3.0.6", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.125" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] -[[package]] -name = "wasm-encoder" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" -dependencies = [ - "leb128fmt", - "wasmparser", -] - -[[package]] -name = "wasm-metadata" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" -dependencies = [ - "anyhow", - "indexmap", - "wasm-encoder", - "wasmparser", -] - -[[package]] -name = "wasmparser" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" -dependencies = [ - "bitflags", - "hashbrown 0.15.5", - "indexmap", - "semver", -] - [[package]] name = "web-sys" -version = "0.3.102" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -3200,7 +3112,7 @@ checksum = "d5cec722a3274e47d1524cbe2cea762f2c19d615bd9d73ada21db9066349d57e" dependencies = [ "proc-macro2", "quote", - "thiserror 2.0.18", + "thiserror 2.0.21", "wincode-derive", ] @@ -3213,7 +3125,7 @@ dependencies = [ "darling 0.21.3", "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3237,7 +3149,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3248,7 +3160,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3359,109 +3271,15 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0592e1c9d151f854e6fd382574c3a0855250e1d9b2f99d9281c6e6391af352f1" - -[[package]] -name = "wit-bindgen" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" -dependencies = [ - "wit-bindgen-rust-macro", -] - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "wit-bindgen-core" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" -dependencies = [ - "anyhow", - "heck", - "wit-parser", -] - -[[package]] -name = "wit-bindgen-rust" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" -dependencies = [ - "anyhow", - "heck", - "indexmap", - "prettyplease", - "syn", - "wasm-metadata", - "wit-bindgen-core", - "wit-component", -] - -[[package]] -name = "wit-bindgen-rust-macro" -version = "0.51.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" -dependencies = [ - "anyhow", - "prettyplease", - "proc-macro2", - "quote", - "syn", - "wit-bindgen-core", - "wit-bindgen-rust", -] - -[[package]] -name = "wit-component" -version = "0.244.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" -dependencies = [ - "anyhow", - "bitflags", - "indexmap", - "log", - "serde", - "serde_derive", - "serde_json", - "wasm-encoder", - "wasm-metadata", - "wasmparser", - "wit-parser", -] - -[[package]] -name = "wit-parser" -version = "0.244.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" -dependencies = [ - "anyhow", - "id-arena", - "indexmap", - "log", - "semver", - "serde", - "serde_derive", - "serde_json", - "unicode-xid", - "wasmparser", -] +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" [[package]] name = "writeable" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" [[package]] name = "wyz" @@ -3472,6 +3290,23 @@ dependencies = [ "tap", ] +[[package]] +name = "x509-parser" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" +dependencies = [ + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "xattr" version = "1.6.1" @@ -3504,34 +3339,34 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 3.0.6", + "synstructure 0.14.0", ] [[package]] name = "zerocopy" -version = "0.8.52" +version = "0.8.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +checksum = "6df92bf3d9227be3d53173901ddbffac2babc27ae50f397776ffd6dc33f800cb" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.52" +version = "0.8.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +checksum = "ac4f328cf2f05d084e496c3e9c3f33ed0a183656a16e1fcec4d464d8373aec82" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.119", ] [[package]] @@ -3545,14 +3380,14 @@ dependencies = [ [[package]] name = "zerofrom-derive" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn", - "synstructure", + "syn 3.0.6", + "synstructure 0.14.0", ] [[package]] @@ -3563,9 +3398,9 @@ checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" [[package]] name = "zerotrie" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" dependencies = [ "displaydoc", "yoke", @@ -3574,9 +3409,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" dependencies = [ "yoke", "zerofrom", @@ -3585,45 +3420,17 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.3" +version = "0.11.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 3.0.6", ] [[package]] name = "zmij" -version = "1.0.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" - -[[package]] -name = "zstd" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" -dependencies = [ - "zstd-sys", -] - -[[package]] -name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" -dependencies = [ - "cc", - "pkg-config", -] +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/Cargo.toml b/Cargo.toml index f2f9ebf2..e1cccc79 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,11 +10,11 @@ repository = "https://github.com/omnect/omnect-cli" version = "0.28.3" [dependencies] -actix-web = "4.11" +actix-web = { version = "4.15", default-features = false, features = ["macros"] } anyhow = "1" -base64 = { version = "0.22", default-features = false } +base64 = { version = "0.23", default-features = false } bzip2 = { version = "0.6", default-features = true } -clap = { version = "4.5", default-features = false, features = [ +clap = { version = "4.6", default-features = false, features = [ "derive", "help", "std", @@ -25,11 +25,11 @@ env_logger = { version = "0.11", default-features = false } filemagic = { version = "0.13", default-features = false, features = [ "pkg-config", ] } -flate2 = { version = "1.1", default-features = false } +flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } gptman = { version = "3.1", default-features = false } hmac = { version = "0.13", default-features = false } mbrman = { version = "0.6", default-features = false } -omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.4.0" } +omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" } keyring-core = { version = "1.0", default-features = false } libfs = { version = "0.9", default-features = false } log = { version = "0.4", default-features = false } @@ -38,8 +38,8 @@ oauth2 = { version = "5.0", default-features = false, features = [ "reqwest", "native-tls", ] } -open = { version = "5.3", default-features = false } -regex = { version = "1.11", default-features = false } +open = { version = "5.4", default-features = false } +regex = { version = "1.13", default-features = false } reqwest = { version = "0.13", default-features = false, features = [ "json", "native-tls", @@ -51,28 +51,28 @@ sha2 = { version = "0.11", default-features = false } stdext = { version = "0.3", default-features = false } strum = { version = "0.28", default-features = false } strum_macros = { version = "0.28", default-features = false } -tempfile = { version = "3.20", default-features = false } +tempfile = { version = "3.27", default-features = false } time = { version = "0.3", default-features = false } tokio = { version = "1", default-features = false, features = [ "macros", "fs", "rt-multi-thread", ] } -toml = { version = "1.0", default-features = false, features = [ +toml = { version = "1.1", default-features = false, features = [ "parse", "serde", ] } -uuid = { version = "1.17", default-features = false, features = ["v4"] } +uuid = { version = "1.26", default-features = false, features = ["v4"] } url = { version = "2.5", default-features = false } -validator = { version = "0.20", default-features = false, features = [ +validator = { version = "0.21", default-features = false, features = [ "derive", ] } xz2 = { version = "0.1", default-features = false } [dev-dependencies] -assert_cmd = "2.0" +assert_cmd = "2.2" assert-json-diff = "2.0" -data-encoding = "2.9" +data-encoding = "2.11" file_diff = "1.0" httpmock = "0.8" predicates = "3.1" diff --git a/src/lib.rs b/src/lib.rs index 11eeaa62..343f912e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -175,7 +175,7 @@ fn create_image_cert(image: &Path, cert_opts: CertificateOptions) -> Result Date: Mon, 28 Sep 2026 13:12:50 +0200 Subject: [PATCH 03/46] style(device_update): drop redundant borrows in info! arguments Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/device_update/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/device_update/mod.rs b/src/device_update/mod.rs index 8e2f6da3..761aa5c0 100644 --- a/src/device_update/mod.rs +++ b/src/device_update/mod.rs @@ -292,7 +292,7 @@ pub async fn import_update( debug!("import update: {import_update}"); let import_update_response = adu_client.import_update(instance_id, import_update).await?; - info!("Result of import update: {:?}", &import_update_response); + info!("Result of import update: {:?}", import_update_response); Ok(()) } @@ -321,7 +321,7 @@ pub async fn remove_update( let remove_update_response = adu_client .delete_update(instance_id, provider, name, version) .await?; - info!("Result of remove update: {:?}", &remove_update_response); + info!("Result of remove update: {:?}", remove_update_response); Ok(()) } From e9163a560b190610973acbcd3902f41a046001d1 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:01:01 +0200 Subject: [PATCH 04/46] fix(file): keep holes when copying the image libfs was built with `default-features = false`, which also drops its `use_linux` feature. Without it libfs copies byte by byte, so every image omnect-cli works on came back fully allocated, and a bmap created with `-b` mapped the whole image. The `fallocate -d` that write_partition used to run hid this. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.toml | 2 +- tests/integration_tests.rs | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index e1cccc79..6451b69d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,7 @@ hmac = { version = "0.13", default-features = false } mbrman = { version = "0.6", default-features = false } omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" } keyring-core = { version = "1.0", default-features = false } -libfs = { version = "0.9", default-features = false } +libfs = { version = "0.9", default-features = false, features = ["use_linux"] } log = { version = "0.4", default-features = false } num_cpus = { version = "1.17", default-features = false } oauth2 = { version = "5.0", default-features = false, features = [ diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index e51b00c0..4052768b 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -947,6 +947,40 @@ fn check_file_copy_keeps_image_layout() { ); } +#[test] +fn check_file_copy_keeps_holes() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + let status = std::process::Command::new("cp") + .arg("--sparse=always") + .arg(&image_path) + .arg(&sparse_path) + .status() + .expect("run cp"); + assert!(status.success(), "sparse copy of the test image failed"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .arg("-b") + .assert() + .success(); + + let meta = sparse_path.metadata().expect("image metadata"); + assert!(meta.blocks() * STAT_BLOCK_SIZE < meta.len()); + let bmap = std::fs::read_to_string(&bmap_path).expect("read bmap"); + assert!(bmap_value(&bmap, "MappedBlocksCount") < bmap_value(&bmap, "BlocksCount")); +} + #[test] fn check_bmap_generation_wic() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); From e2443b265d70789ae496d5b05d6c048f16f7d6ea Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 16:01:02 +0200 Subject: [PATCH 05/46] fix(file): map the ext4 journals in a generated bmap A fresh ext4 journal is all zeros, and a sparse image, e.g. one that xz decompressed into a file, holds it as holes. A bmap then skips the journal, so a bmap-based flash leaves the journal of the previous installation on the device, and after a power cut its replay can apply an old transaction to the new filesystem. Before creating the bmap, rewrite the journal blocks of every ext4 partition with their own content. Holes become allocated zero blocks and all other blocks stay unchanged. debugfs lists the journal blocks, so the package now depends on e2fsprogs. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.toml | 2 +- src/file/functions.rs | 226 ++++++++++++++++++++++++++++++++++++++++++ src/file/partition.rs | 36 +++++++ 3 files changed, 263 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 6451b69d..b9efc7d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,5 +81,5 @@ tar = "0.4" # metadata for building with cargo-deb (https://crates.io/crates/cargo-deb) [package.metadata.deb] -depends = "bmap-tools, e2tools, keychain, libc6 (>= 2.34), libmagic1, libssl3 (>= 3.0.0), mtools" +depends = "bmap-tools, e2fsprogs, e2tools, keychain, libc6 (>= 2.34), libmagic1, libssl3 (>= 3.0.0), mtools" revision = "" diff --git a/src/file/functions.rs b/src/file/functions.rs index 49a486c7..800ff5a4 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -14,6 +14,16 @@ const SECTOR_SIZE: u64 = 512; const COMPARE_BLOCK_SIZE: usize = 4096; const COPY_CHUNK_SIZE: usize = 1024 * 1024; +const EXT4_SUPERBLOCK_OFFSET: u64 = 1024; +const EXT4_SUPERBLOCK_SIZE: usize = 1024; +const EXT4_S_LOG_BLOCK_SIZE: usize = 0x18; +const EXT4_S_MAGIC: usize = 0x38; +const EXT4_S_FEATURE_COMPAT: usize = 0x5c; +const EXT4_S_JOURNAL_INUM: usize = 0xe0; +const EXT4_MAGIC: u16 = 0xef53; +const EXT4_FEATURE_COMPAT_HAS_JOURNAL: u32 = 0x4; +const EXT4_MIN_BLOCK_SIZE: u64 = 1024; + #[derive(clap::ValueEnum, Debug, Clone, Eq, Hash, PartialEq)] #[clap(rename_all = "verbatim")] #[allow(non_camel_case_types)] @@ -485,7 +495,113 @@ fn write_partition( .context("write_partition: cannot sync image") } +struct Ext4Journal { + inode: u32, + block_size: u64, +} + +fn read_ext4_journal(image: &fs::File, offset: u64) -> Result> { + let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; + image + .read_exact_at(&mut sb, offset + EXT4_SUPERBLOCK_OFFSET) + .context("read_ext4_journal: cannot read superblock")?; + let le16 = |o: usize| u16::from_le_bytes([sb[o], sb[o + 1]]); + let le32 = |o: usize| u32::from_le_bytes([sb[o], sb[o + 1], sb[o + 2], sb[o + 3]]); + + if le16(EXT4_S_MAGIC) != EXT4_MAGIC + || le32(EXT4_S_FEATURE_COMPAT) & EXT4_FEATURE_COMPAT_HAS_JOURNAL == 0 + { + return Ok(None); + } + + // an external journal has no inode in this filesystem + let inode = le32(EXT4_S_JOURNAL_INUM); + if inode == 0 { + return Ok(None); + } + + let block_size = EXT4_MIN_BLOCK_SIZE + .checked_shl(le32(EXT4_S_LOG_BLOCK_SIZE)) + .context("read_ext4_journal: invalid block size")?; + + Ok(Some(Ext4Journal { inode, block_size })) +} + +fn ext4_journal_blocks(image_file: &str, offset: u64, inode: u32) -> Result> { + let mut debugfs = Command::new("debugfs"); + debugfs + .arg("-R") + .arg(format!("blocks <{inode}>")) + .arg(format!("{image_file}?offset={offset}")); + let output = debugfs + .output() + .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; + anyhow::ensure!( + output.status.success(), + "ext4_journal_blocks: {debugfs:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let blocks = String::from_utf8(output.stdout) + .context("ext4_journal_blocks: invalid debugfs output")? + .split_whitespace() + .map(|b| { + b.parse::() + .context(format!("ext4_journal_blocks: invalid block number {b}")) + }) + .collect::>>()?; + anyhow::ensure!( + !blocks.is_empty(), + "ext4_journal_blocks: no journal blocks at offset {offset}" + ); + + Ok(blocks) +} + +/// Rewrites the journal blocks of every ext4 partition with their own content. Blocks that +/// are holes become allocated, so a bmap maps the whole journal and a bmap-based flash +/// overwrites the journal an earlier installation left on the device. +fn allocate_ext4_journals(image_file: &str) -> Result<()> { + use crate::file::partition::get_partitions; + + let image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(image_file) + .context(format!("allocate_ext4_journals: cannot open {image_file}"))?; + + for partition in get_partitions(image_file)? { + let offset = partition.start * SECTOR_SIZE; + let Some(journal) = read_ext4_journal(&image, offset)? else { + continue; + }; + let blocks = ext4_journal_blocks(image_file, offset, journal.inode)?; + debug!( + "allocate_ext4_journals: partition {}: {} journal blocks", + partition.num, + blocks.len() + ); + + let mut buf = vec![0u8; usize::try_from(journal.block_size)?]; + for block in blocks { + let pos = offset + block * journal.block_size; + image + .read_exact_at(&mut buf, pos) + .context("allocate_ext4_journals: cannot read journal block")?; + image + .write_all_at(&buf, pos) + .context("allocate_ext4_journals: cannot write journal block")?; + } + } + + image + .sync_all() + .context("allocate_ext4_journals: cannot sync image") +} + pub fn generate_bmap_file(image_file: &str) -> Result<()> { + allocate_ext4_journals(image_file)?; + let mut bmaptool = Command::new("bmaptool"); bmaptool .arg("create") @@ -496,3 +612,113 @@ pub fn generate_bmap_file(image_file: &str) -> Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + const PART_START_LBA: u32 = 2048; + const PART_SECTORS: u32 = 16384; + const FS_BLOCK_SIZE: u64 = 1024; + const MBR_DISK_SIGNATURE: [u8; 4] = [1, 2, 3, 4]; + const MBR_TYPE_LINUX: u8 = 0x83; + const BMAP_BLOCK_SIZE: u64 = 4096; + + fn run(cmd: &mut Command) { + let status = cmd.status().expect("run command"); + assert!(status.success(), "{cmd:?} failed"); + } + + fn create_image_with_ext4_journal(image: &Path) { + let raw = image.with_extension("raw"); + let mut f = fs::File::create(&raw).expect("create image"); + f.set_len(u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let mut mbr = mbrman::MBR::new_from(&mut f, SECTOR_SIZE as u32, MBR_DISK_SIGNATURE) + .expect("create mbr"); + mbr[1] = mbrman::MBRPartitionEntry { + boot: mbrman::BOOT_INACTIVE, + first_chs: mbrman::CHS::empty(), + sys: MBR_TYPE_LINUX, + last_chs: mbrman::CHS::empty(), + starting_lba: PART_START_LBA, + sectors: PART_SECTORS, + }; + mbr.write_into(&mut f).expect("write mbr"); + + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; + run(Command::new("mkfs.ext4") + .arg("-q") + .arg("-F") + .arg("-b") + .arg(FS_BLOCK_SIZE.to_string()) + .arg("-J") + .arg("size=1") + .arg("-E") + .arg(format!("offset={offset}")) + .arg(&raw) + .arg(fs_blocks.to_string())); + // mke2fs zeroes the journal as unwritten extents, which a bmap maps; a sparse copy + // turns the zeros into holes, as a sparse decompress does + run(Command::new("cp") + .arg("--sparse=always") + .arg(&raw) + .arg(image)); + } + + fn bmap_ranges(bmap: &str) -> Vec<(u64, u64)> { + bmap.lines() + .filter_map(|l| { + let range = l.split_once("')?.1; + let range = range.split_once(" usize { + let ranges = bmap_ranges(bmap); + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let file = fs::File::open(image).expect("open image"); + let journal = read_ext4_journal(&file, offset) + .expect("read superblock") + .expect("ext4 with journal"); + ext4_journal_blocks(image, offset, journal.inode) + .expect("journal blocks") + .iter() + .map(|b| (offset + b * journal.block_size) / BMAP_BLOCK_SIZE) + .filter(|b| !ranges.iter().any(|(s, e)| s <= b && b <= e)) + .count() + } + + #[test] + fn generate_bmap_file_maps_ext4_journal() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_path = dir.path().join("image.wic"); + create_image_with_ext4_journal(&image_path); + let image = image_path.to_str().expect("image path"); + let bmap_path = format!("{image}.bmap"); + + run(Command::new("bmaptool") + .arg("create") + .arg("-o") + .arg(&bmap_path) + .arg(image)); + let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); + assert!(unmapped_journal_blocks(image, &bmap) > 0); + + generate_bmap_file(image).expect("generate bmap"); + let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); + assert_eq!(unmapped_journal_blocks(image, &bmap), 0); + + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + run(Command::new("e2fsck") + .arg("-fn") + .arg(format!("{image}?offset={offset}"))); + } +} diff --git a/src/file/partition.rs b/src/file/partition.rs index 04d2fc6d..cc7b28a3 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -55,6 +55,42 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result anyhow::bail!("partition {partition_num} not found in image") } +pub fn get_partitions>(path: P) -> Result> { + let path = path.as_ref(); + let mut file = + File::open(path).with_context(|| format!("failed to open image: {}", path.display()))?; + + let gpt_err = match gptman::GPT::find_from(&mut file) { + Ok(gpt) => { + return Ok(gpt + .iter() + .filter(|(_, e)| e.is_used() && e.ending_lba >= e.starting_lba) + .map(|(num, e)| PartitionData { + num, + start: e.starting_lba, + count: e.ending_lba - e.starting_lba + 1, + }) + .collect()); + } + Err(e) => e, + }; + + file.seek(SeekFrom::Start(0)) + .context("failed to seek to start of image")?; + let mbr = mbrman::MBR::read_from(&mut file, 512) + .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; + mbr.iter() + .filter(|(_, p)| p.is_used()) + .map(|(num, p)| { + Ok(PartitionData { + num: u32::try_from(num).context("MBR partition number out of range")?, + start: p.starting_lba as u64, + count: p.sectors as u64, + }) + }) + .collect() +} + pub fn is_gpt>(path: P) -> Result { let mut file = File::open(path.as_ref()).context("is_gpt: failed to open image")?; Ok(gptman::GPT::find_from(&mut file).is_ok()) From 52fe21d5cbfb87bed1ceaae052b09f0fd38a57da Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:05:32 +0200 Subject: [PATCH 06/46] fix: copy image data segments larger than 2 GiB completely libfs::copy_file copies each data segment with one copy_file_range call and ignores a short copy, so a segment over 2 GiB was cut off without error. copy_image keeps the holes and loops until each segment is copied. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/lib.rs | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 343f912e..a8ce6bf4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,7 +22,7 @@ use cli::{ }; use file::{compression::Compression, functions::FileCopyToParams}; use log::error; -use std::{fs, path::Path, path::PathBuf}; +use std::{fs, io::Read, path::Path, path::PathBuf}; use tokio::fs::remove_dir_all; use uuid::Uuid; @@ -93,9 +93,8 @@ where tmp_image_file = compression::decompress(&tmp_image_file, &source_compression)?; dest_image_file.set_extension(""); } else { - // copy sparse file (std::fs::copy isn't able) - libfs::copy_file(&image_file, &tmp_image_file).context(format!( - "error: libfs::copy_file({:?}, {:?})", + copy_image(&image_file, &tmp_image_file).context(format!( + "error: copy_image({:?}, {:?})", image_file, tmp_image_file ))?; } @@ -140,9 +139,8 @@ where tmp_image_file, dest_image_file ))?; } else { - // copy sparse file (std::fs::copy isn't able) - libfs::copy_file(&tmp_image_file, &dest_image_file).context(format!( - "error: libfs::copy_file({:?}, {:?})", + copy_image(&tmp_image_file, &dest_image_file).context(format!( + "error: copy_image({:?}, {:?})", tmp_image_file, dest_image_file ))?; } @@ -150,6 +148,35 @@ where Ok(()) } +/// Copies `from` to `to` and keeps the holes of a sparse `from`. +fn copy_image(from: &Path, to: &Path) -> Result<()> { + let src = fs::File::open(from).context("copy_image: cannot open source")?; + let len = src + .metadata() + .context("copy_image: cannot get source size")? + .len(); + let dst = fs::File::create(to).context("copy_image: cannot create destination")?; + dst.set_len(len) + .context("copy_image: cannot set destination size")?; + + // one copy_file_range call copies at most 2 GiB; io::copy repeats it until a segment is done + let sparse = libfs::probably_sparse(&src)?; + let mut pos = 0; + while pos < len { + let (data, hole) = if sparse { + libfs::next_sparse_segments(&src, &dst, pos)? + } else { + (pos, len) + }; + let copied = std::io::copy(&mut (&src).take(hole - data), &mut &dst) + .context("copy_image: cannot copy data")?; + anyhow::ensure!(copied == hole - data, "copy_image: source ended early"); + pos = hole; + } + + Ok(()) +} + struct CertInfo { cert_path: PathBuf, key_path: PathBuf, From 19bcd306f3966d384bb029b9b99de70515a7078c Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:05:33 +0200 Subject: [PATCH 07/46] fix(file): harden ext4 journal lookup for bmap generation - fall back to /usr/sbin/debugfs when debugfs is not in PATH - reject an invalid s_log_block_size before it is used as a shift - skip partitions whose superblock is past the end of the image - skip the MBR extended container entry - share SECTOR_SIZE between partition and functions Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/functions.rs | 49 +++++++++++++++++++++++++++++++------------ src/file/partition.rs | 8 ++++--- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/src/file/functions.rs b/src/file/functions.rs index 800ff5a4..241b0dd2 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,3 +1,4 @@ +use crate::file::partition::SECTOR_SIZE; use anyhow::{Context, Result}; use log::{debug, warn}; use std::collections::HashMap; @@ -10,7 +11,6 @@ use std::str::FromStr; use stdext::function_name; use uuid::Uuid; -const SECTOR_SIZE: u64 = 512; const COMPARE_BLOCK_SIZE: usize = 4096; const COPY_CHUNK_SIZE: usize = 1024 * 1024; @@ -23,6 +23,11 @@ const EXT4_S_JOURNAL_INUM: usize = 0xe0; const EXT4_MAGIC: u16 = 0xef53; const EXT4_FEATURE_COMPAT_HAS_JOURNAL: u32 = 0x4; const EXT4_MIN_BLOCK_SIZE: u64 = 1024; +const EXT4_MAX_LOG_BLOCK_SIZE: u32 = 6; + +const DEBUGFS: &str = "debugfs"; +// debugfs is installed to sbin, which is not always in the PATH of a normal user +const DEBUGFS_SBIN: &str = "/usr/sbin/debugfs"; #[derive(clap::ValueEnum, Debug, Clone, Eq, Hash, PartialEq)] #[clap(rename_all = "verbatim")] @@ -520,22 +525,33 @@ fn read_ext4_journal(image: &fs::File, offset: u64) -> Result Result> { - let mut debugfs = Command::new("debugfs"); - debugfs - .arg("-R") - .arg(format!("blocks <{inode}>")) - .arg(format!("{image_file}?offset={offset}")); - let output = debugfs - .output() - .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; + let debugfs_cmd = |program| { + let mut cmd = Command::new(program); + cmd.arg("-R") + .arg(format!("blocks <{inode}>")) + .arg(format!("{image_file}?offset={offset}")); + cmd + }; + let mut debugfs = debugfs_cmd(DEBUGFS); + let output = match debugfs.output() { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + debugfs = debugfs_cmd(DEBUGFS_SBIN); + debugfs.output() + } + output => output, + } + .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; anyhow::ensure!( output.status.success(), "ext4_journal_blocks: {debugfs:?} failed: {}", @@ -569,9 +585,16 @@ fn allocate_ext4_journals(image_file: &str) -> Result<()> { .write(true) .open(image_file) .context(format!("allocate_ext4_journals: cannot open {image_file}"))?; + let image_len = image + .metadata() + .context("allocate_ext4_journals: cannot get image size")? + .len(); for partition in get_partitions(image_file)? { let offset = partition.start * SECTOR_SIZE; + if offset + EXT4_SUPERBLOCK_OFFSET + EXT4_SUPERBLOCK_SIZE as u64 > image_len { + continue; + } let Some(journal) = read_ext4_journal(&image, offset)? else { continue; }; @@ -615,7 +638,7 @@ pub fn generate_bmap_file(image_file: &str) -> Result<()> { #[cfg(test)] mod tests { - use super::*; + use crate::file::functions::*; const PART_START_LBA: u32 = 2048; const PART_SECTORS: u32 = 16384; diff --git a/src/file/partition.rs b/src/file/partition.rs index cc7b28a3..9d9d0b8c 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -4,6 +4,8 @@ use std::io::Seek; use std::io::SeekFrom; use std::path::Path; +pub const SECTOR_SIZE: u64 = 512; + pub struct PartitionData { pub num: u32, pub start: u64, @@ -41,7 +43,7 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result // Try MBR — iter() includes logical partitions (5, 6, ...) file.seek(SeekFrom::Start(0)) .context("failed to seek to start of image")?; - let mbr = mbrman::MBR::read_from(&mut file, 512) + let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; for (i, p) in mbr.iter() { if u32::try_from(i).ok() == Some(partition_num) && p.is_used() { @@ -77,10 +79,10 @@ pub fn get_partitions>(path: P) -> Result> { file.seek(SeekFrom::Start(0)) .context("failed to seek to start of image")?; - let mbr = mbrman::MBR::read_from(&mut file, 512) + let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; mbr.iter() - .filter(|(_, p)| p.is_used()) + .filter(|(_, p)| p.is_used() && !p.is_extended()) .map(|(num, p)| { Ok(PartitionData { num: u32::try_from(num).context("MBR partition number out of range")?, From d7e724244c2ef4aa10a162ba7ab4ceff2b205186 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:05:34 +0200 Subject: [PATCH 08/46] build(docker): install e2fsprogs in the builder The deb package depends on e2fsprogs for debugfs. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 8eb12929..4fc20c1a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ bmap-tools \ ca-certificates \ + e2fsprogs \ e2tools \ fdisk \ keychain \ From 2912452cebd3eb6d2eeeae94d11149d998f708a2 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:13:38 +0200 Subject: [PATCH 09/46] fix(file): keep zero blocks as holes when decompressing Decompress wrote every zero block as data, so a bmap of a compressed input mapped the whole image. The decompressed file now seeks over zero blocks and keeps them as holes. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/compression.rs | 97 ++++++++++++++++++++++++++++++++++++-- tests/integration_tests.rs | 3 ++ 2 files changed, 96 insertions(+), 4 deletions(-) diff --git a/src/file/compression.rs b/src/file/compression.rs index 16f18626..fca98d49 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -3,11 +3,14 @@ use filemagic::Magic; use log::debug; use std::env; use std::fs::File; +use std::io::{Seek, SeekFrom, Write}; use std::path::PathBuf; use std::str::FromStr; use strum::IntoEnumIterator; use strum_macros::EnumIter; +const SPARSE_BLOCK_SIZE: u64 = 4096; + #[derive(Clone, Debug, EnumIter)] #[allow(non_camel_case_types)] pub enum Compression { @@ -76,15 +79,21 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let mut dec: Box = match &self { - Compression::bzip2 => Box::new(bzip2::write::BzDecoder::new(destination)), - Compression::gzip => Box::new(flate2::write::GzDecoder::new(destination)), - Compression::xz { .. } => Box::new(xz2::write::XzDecoder::new(destination)), + let sparse = SparseWriter(destination); + let mut dec: Box = match &self { + Compression::bzip2 => Box::new(bzip2::write::BzDecoder::new(sparse)), + Compression::gzip => Box::new(flate2::write::GzDecoder::new(sparse)), + Compression::xz { .. } => Box::new(xz2::write::XzDecoder::new(sparse)), }; let bytes_written = std::io::copy(source, &mut dec)?; dec.write_all(&[])?; dec.flush()?; + drop(dec); + + // zero blocks at the end were only seeked over + let len = destination.stream_position()?; + destination.set_len(len)?; Ok(bytes_written) } @@ -126,6 +135,27 @@ impl Compression { } } +/// Seeks over zero blocks instead of writing them, so they stay holes in a new file. +struct SparseWriter<'a>(&'a mut File); + +impl Write for SparseWriter<'_> { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let pos = self.0.stream_position()?; + let to_block_end = SPARSE_BLOCK_SIZE - pos % SPARSE_BLOCK_SIZE; + let chunk = &buf[..buf.len().min(to_block_end as usize)]; + if chunk.iter().all(|b| *b == 0) { + self.0.seek(SeekFrom::Current(chunk.len() as i64))?; + Ok(chunk.len()) + } else { + self.0.write(chunk) + } + } + + fn flush(&mut self) -> std::io::Result<()> { + self.0.flush() + } +} + pub fn decompress(image_file_name: &PathBuf, compression: &Compression) -> Result { let mut new_image_file = PathBuf::from(image_file_name); @@ -157,3 +187,62 @@ pub fn compress(image_file_name: &PathBuf, compression: &Compression) -> Result< debug!("image::compress: copied {} bytes.", bytes_written); Ok(new_image_file) } + +#[cfg(test)] +mod tests { + use crate::file::compression::*; + use std::io::Read; + use std::os::unix::fs::MetadataExt; + + // unit of `st_blocks` + const STAT_BLOCK_SIZE: u64 = 512; + const DATA_OFFSET: usize = 1024 * 1024 + 100; + const IMAGE_LEN: usize = 3 * 1024 * 1024 + 123; + + fn roundtrip(compression: Compression, image: &[u8]) -> File { + let mut source = tempfile::tempfile().expect("create source"); + source.write_all(image).expect("write source"); + source.rewind().expect("rewind source"); + + let mut compressed = tempfile::tempfile().expect("create compressed"); + compression + .compress(&mut source, &mut compressed) + .expect("compress"); + compressed.rewind().expect("rewind compressed"); + + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + compression + .decompress(&mut compressed, &mut decompressed) + .expect("decompress"); + decompressed + } + + #[test] + fn decompress_keeps_zero_blocks_as_holes() { + let mut image = vec![0u8; IMAGE_LEN]; + image[DATA_OFFSET..DATA_OFFSET + 3].copy_from_slice(b"abc"); + + for compression in [ + Compression::bzip2, + Compression::gzip, + Compression::xz { + compression_level: 1, + }, + ] { + let mut decompressed = roundtrip(compression.clone(), &image); + + let mut content = Vec::new(); + decompressed.rewind().expect("rewind decompressed"); + decompressed + .read_to_end(&mut content) + .expect("read decompressed"); + assert!(content == image, "{compression:?}: content differs"); + + let meta = decompressed.metadata().expect("metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE < meta.len(), + "{compression:?}: no holes" + ); + } + } +} diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 4052768b..c807429d 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1051,6 +1051,9 @@ fn check_bmap_generation_wic_xz() { .assert(); assert.success(); + let bmap = std::fs::read_to_string(&image_path_bmap).expect("read bmap"); + assert!(bmap_value(&bmap, "MappedBlocksCount") < bmap_value(&bmap, "BlocksCount")); + // use bmaptool to verify that the checksum of the bmap file and the image // still match after the copy operations let assert = Command::new("bmaptool") From 84ba27e6d68445033af07aa6222b7dec49d873e9 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:18:24 +0200 Subject: [PATCH 10/46] refactor(file): move bmap generation into its own module The ext4 journal lookup is only used for bmap generation, so it moves with generate_bmap_file from functions.rs to bmap.rs. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/bmap.rs | 269 ++++++++++++++++++++++++++++++++++++++++++ src/file/functions.rs | 261 ---------------------------------------- src/file/mod.rs | 1 + src/lib.rs | 2 +- 4 files changed, 271 insertions(+), 262 deletions(-) create mode 100644 src/file/bmap.rs diff --git a/src/file/bmap.rs b/src/file/bmap.rs new file mode 100644 index 00000000..e6b6352b --- /dev/null +++ b/src/file/bmap.rs @@ -0,0 +1,269 @@ +use crate::file::partition::{SECTOR_SIZE, get_partitions}; +use anyhow::{Context, Result}; +use log::debug; +use std::fs; +use std::os::unix::fs::FileExt; +use std::process::Command; + +const EXT4_SUPERBLOCK_OFFSET: u64 = 1024; +const EXT4_SUPERBLOCK_SIZE: usize = 1024; +const EXT4_S_LOG_BLOCK_SIZE: usize = 0x18; +const EXT4_S_MAGIC: usize = 0x38; +const EXT4_S_FEATURE_COMPAT: usize = 0x5c; +const EXT4_S_JOURNAL_INUM: usize = 0xe0; +const EXT4_MAGIC: u16 = 0xef53; +const EXT4_FEATURE_COMPAT_HAS_JOURNAL: u32 = 0x4; +const EXT4_MIN_BLOCK_SIZE: u64 = 1024; +const EXT4_MAX_LOG_BLOCK_SIZE: u32 = 6; + +const DEBUGFS: &str = "debugfs"; +// debugfs is installed to sbin, which is not always in the PATH of a normal user +const DEBUGFS_SBIN: &str = "/usr/sbin/debugfs"; + +struct Ext4Journal { + inode: u32, + block_size: u64, +} + +fn read_ext4_journal(image: &fs::File, offset: u64) -> Result> { + let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; + image + .read_exact_at(&mut sb, offset + EXT4_SUPERBLOCK_OFFSET) + .context("read_ext4_journal: cannot read superblock")?; + let le16 = |o: usize| u16::from_le_bytes([sb[o], sb[o + 1]]); + let le32 = |o: usize| u32::from_le_bytes([sb[o], sb[o + 1], sb[o + 2], sb[o + 3]]); + + if le16(EXT4_S_MAGIC) != EXT4_MAGIC + || le32(EXT4_S_FEATURE_COMPAT) & EXT4_FEATURE_COMPAT_HAS_JOURNAL == 0 + { + return Ok(None); + } + + // an external journal has no inode in this filesystem + let inode = le32(EXT4_S_JOURNAL_INUM); + if inode == 0 { + return Ok(None); + } + + let log_block_size = le32(EXT4_S_LOG_BLOCK_SIZE); + anyhow::ensure!( + log_block_size <= EXT4_MAX_LOG_BLOCK_SIZE, + "read_ext4_journal: invalid block size" + ); + let block_size = EXT4_MIN_BLOCK_SIZE << log_block_size; + + Ok(Some(Ext4Journal { inode, block_size })) +} + +fn ext4_journal_blocks(image_file: &str, offset: u64, inode: u32) -> Result> { + let debugfs_cmd = |program| { + let mut cmd = Command::new(program); + cmd.arg("-R") + .arg(format!("blocks <{inode}>")) + .arg(format!("{image_file}?offset={offset}")); + cmd + }; + let mut debugfs = debugfs_cmd(DEBUGFS); + let output = match debugfs.output() { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + debugfs = debugfs_cmd(DEBUGFS_SBIN); + debugfs.output() + } + output => output, + } + .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; + anyhow::ensure!( + output.status.success(), + "ext4_journal_blocks: {debugfs:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let blocks = String::from_utf8(output.stdout) + .context("ext4_journal_blocks: invalid debugfs output")? + .split_whitespace() + .map(|b| { + b.parse::() + .context(format!("ext4_journal_blocks: invalid block number {b}")) + }) + .collect::>>()?; + anyhow::ensure!( + !blocks.is_empty(), + "ext4_journal_blocks: no journal blocks at offset {offset}" + ); + + Ok(blocks) +} + +/// Rewrites the journal blocks of every ext4 partition with their own content. Blocks that +/// are holes become allocated, so a bmap maps the whole journal and a bmap-based flash +/// overwrites the journal an earlier installation left on the device. +fn allocate_ext4_journals(image_file: &str) -> Result<()> { + let image = fs::OpenOptions::new() + .read(true) + .write(true) + .open(image_file) + .context(format!("allocate_ext4_journals: cannot open {image_file}"))?; + let image_len = image + .metadata() + .context("allocate_ext4_journals: cannot get image size")? + .len(); + + for partition in get_partitions(image_file)? { + let offset = partition.start * SECTOR_SIZE; + if offset + EXT4_SUPERBLOCK_OFFSET + EXT4_SUPERBLOCK_SIZE as u64 > image_len { + continue; + } + let Some(journal) = read_ext4_journal(&image, offset)? else { + continue; + }; + let blocks = ext4_journal_blocks(image_file, offset, journal.inode)?; + debug!( + "allocate_ext4_journals: partition {}: {} journal blocks", + partition.num, + blocks.len() + ); + + let mut buf = vec![0u8; usize::try_from(journal.block_size)?]; + for block in blocks { + let pos = offset + block * journal.block_size; + image + .read_exact_at(&mut buf, pos) + .context("allocate_ext4_journals: cannot read journal block")?; + image + .write_all_at(&buf, pos) + .context("allocate_ext4_journals: cannot write journal block")?; + } + } + + image + .sync_all() + .context("allocate_ext4_journals: cannot sync image") +} + +pub fn generate_bmap_file(image_file: &str) -> Result<()> { + allocate_ext4_journals(image_file)?; + + let mut bmaptool = Command::new("bmaptool"); + bmaptool + .arg("create") + .arg("-o") + .arg(format!("{image_file}.bmap")) + .arg(image_file); + let status = bmaptool + .status() + .context(format!("generate_bmap_file: cannot run {bmaptool:?}"))?; + anyhow::ensure!(status.success(), "generate_bmap_file: {bmaptool:?} failed"); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::file::bmap::*; + use std::path::Path; + + const PART_START_LBA: u32 = 2048; + const PART_SECTORS: u32 = 16384; + const FS_BLOCK_SIZE: u64 = 1024; + const MBR_DISK_SIGNATURE: [u8; 4] = [1, 2, 3, 4]; + const MBR_TYPE_LINUX: u8 = 0x83; + const BMAP_BLOCK_SIZE: u64 = 4096; + + fn run(cmd: &mut Command) { + let status = cmd.status().expect("run command"); + assert!(status.success(), "{cmd:?} failed"); + } + + fn create_image_with_ext4_journal(image: &Path) { + let raw = image.with_extension("raw"); + let mut f = fs::File::create(&raw).expect("create image"); + f.set_len(u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let mut mbr = mbrman::MBR::new_from(&mut f, SECTOR_SIZE as u32, MBR_DISK_SIGNATURE) + .expect("create mbr"); + mbr[1] = mbrman::MBRPartitionEntry { + boot: mbrman::BOOT_INACTIVE, + first_chs: mbrman::CHS::empty(), + sys: MBR_TYPE_LINUX, + last_chs: mbrman::CHS::empty(), + starting_lba: PART_START_LBA, + sectors: PART_SECTORS, + }; + mbr.write_into(&mut f).expect("write mbr"); + + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; + run(Command::new("mkfs.ext4") + .arg("-q") + .arg("-F") + .arg("-b") + .arg(FS_BLOCK_SIZE.to_string()) + .arg("-J") + .arg("size=1") + .arg("-E") + .arg(format!("offset={offset}")) + .arg(&raw) + .arg(fs_blocks.to_string())); + // mke2fs zeroes the journal as unwritten extents, which a bmap maps; a sparse copy + // turns the zeros into holes, as a sparse decompress does + run(Command::new("cp") + .arg("--sparse=always") + .arg(&raw) + .arg(image)); + } + + fn bmap_ranges(bmap: &str) -> Vec<(u64, u64)> { + bmap.lines() + .filter_map(|l| { + let range = l.split_once("')?.1; + let range = range.split_once(" usize { + let ranges = bmap_ranges(bmap); + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let file = fs::File::open(image).expect("open image"); + let journal = read_ext4_journal(&file, offset) + .expect("read superblock") + .expect("ext4 with journal"); + ext4_journal_blocks(image, offset, journal.inode) + .expect("journal blocks") + .iter() + .map(|b| (offset + b * journal.block_size) / BMAP_BLOCK_SIZE) + .filter(|b| !ranges.iter().any(|(s, e)| s <= b && b <= e)) + .count() + } + + #[test] + fn generate_bmap_file_maps_ext4_journal() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_path = dir.path().join("image.wic"); + create_image_with_ext4_journal(&image_path); + let image = image_path.to_str().expect("image path"); + let bmap_path = format!("{image}.bmap"); + + run(Command::new("bmaptool") + .arg("create") + .arg("-o") + .arg(&bmap_path) + .arg(image)); + let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); + assert!(unmapped_journal_blocks(image, &bmap) > 0); + + generate_bmap_file(image).expect("generate bmap"); + let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); + assert_eq!(unmapped_journal_blocks(image, &bmap), 0); + + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + run(Command::new("e2fsck") + .arg("-fn") + .arg(format!("{image}?offset={offset}"))); + } +} diff --git a/src/file/functions.rs b/src/file/functions.rs index 241b0dd2..d6591e9f 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -14,21 +14,6 @@ use uuid::Uuid; const COMPARE_BLOCK_SIZE: usize = 4096; const COPY_CHUNK_SIZE: usize = 1024 * 1024; -const EXT4_SUPERBLOCK_OFFSET: u64 = 1024; -const EXT4_SUPERBLOCK_SIZE: usize = 1024; -const EXT4_S_LOG_BLOCK_SIZE: usize = 0x18; -const EXT4_S_MAGIC: usize = 0x38; -const EXT4_S_FEATURE_COMPAT: usize = 0x5c; -const EXT4_S_JOURNAL_INUM: usize = 0xe0; -const EXT4_MAGIC: u16 = 0xef53; -const EXT4_FEATURE_COMPAT_HAS_JOURNAL: u32 = 0x4; -const EXT4_MIN_BLOCK_SIZE: u64 = 1024; -const EXT4_MAX_LOG_BLOCK_SIZE: u32 = 6; - -const DEBUGFS: &str = "debugfs"; -// debugfs is installed to sbin, which is not always in the PATH of a normal user -const DEBUGFS_SBIN: &str = "/usr/sbin/debugfs"; - #[derive(clap::ValueEnum, Debug, Clone, Eq, Hash, PartialEq)] #[clap(rename_all = "verbatim")] #[allow(non_camel_case_types)] @@ -499,249 +484,3 @@ fn write_partition( .sync_all() .context("write_partition: cannot sync image") } - -struct Ext4Journal { - inode: u32, - block_size: u64, -} - -fn read_ext4_journal(image: &fs::File, offset: u64) -> Result> { - let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; - image - .read_exact_at(&mut sb, offset + EXT4_SUPERBLOCK_OFFSET) - .context("read_ext4_journal: cannot read superblock")?; - let le16 = |o: usize| u16::from_le_bytes([sb[o], sb[o + 1]]); - let le32 = |o: usize| u32::from_le_bytes([sb[o], sb[o + 1], sb[o + 2], sb[o + 3]]); - - if le16(EXT4_S_MAGIC) != EXT4_MAGIC - || le32(EXT4_S_FEATURE_COMPAT) & EXT4_FEATURE_COMPAT_HAS_JOURNAL == 0 - { - return Ok(None); - } - - // an external journal has no inode in this filesystem - let inode = le32(EXT4_S_JOURNAL_INUM); - if inode == 0 { - return Ok(None); - } - - let log_block_size = le32(EXT4_S_LOG_BLOCK_SIZE); - anyhow::ensure!( - log_block_size <= EXT4_MAX_LOG_BLOCK_SIZE, - "read_ext4_journal: invalid block size" - ); - let block_size = EXT4_MIN_BLOCK_SIZE << log_block_size; - - Ok(Some(Ext4Journal { inode, block_size })) -} - -fn ext4_journal_blocks(image_file: &str, offset: u64, inode: u32) -> Result> { - let debugfs_cmd = |program| { - let mut cmd = Command::new(program); - cmd.arg("-R") - .arg(format!("blocks <{inode}>")) - .arg(format!("{image_file}?offset={offset}")); - cmd - }; - let mut debugfs = debugfs_cmd(DEBUGFS); - let output = match debugfs.output() { - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - debugfs = debugfs_cmd(DEBUGFS_SBIN); - debugfs.output() - } - output => output, - } - .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; - anyhow::ensure!( - output.status.success(), - "ext4_journal_blocks: {debugfs:?} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - - let blocks = String::from_utf8(output.stdout) - .context("ext4_journal_blocks: invalid debugfs output")? - .split_whitespace() - .map(|b| { - b.parse::() - .context(format!("ext4_journal_blocks: invalid block number {b}")) - }) - .collect::>>()?; - anyhow::ensure!( - !blocks.is_empty(), - "ext4_journal_blocks: no journal blocks at offset {offset}" - ); - - Ok(blocks) -} - -/// Rewrites the journal blocks of every ext4 partition with their own content. Blocks that -/// are holes become allocated, so a bmap maps the whole journal and a bmap-based flash -/// overwrites the journal an earlier installation left on the device. -fn allocate_ext4_journals(image_file: &str) -> Result<()> { - use crate::file::partition::get_partitions; - - let image = fs::OpenOptions::new() - .read(true) - .write(true) - .open(image_file) - .context(format!("allocate_ext4_journals: cannot open {image_file}"))?; - let image_len = image - .metadata() - .context("allocate_ext4_journals: cannot get image size")? - .len(); - - for partition in get_partitions(image_file)? { - let offset = partition.start * SECTOR_SIZE; - if offset + EXT4_SUPERBLOCK_OFFSET + EXT4_SUPERBLOCK_SIZE as u64 > image_len { - continue; - } - let Some(journal) = read_ext4_journal(&image, offset)? else { - continue; - }; - let blocks = ext4_journal_blocks(image_file, offset, journal.inode)?; - debug!( - "allocate_ext4_journals: partition {}: {} journal blocks", - partition.num, - blocks.len() - ); - - let mut buf = vec![0u8; usize::try_from(journal.block_size)?]; - for block in blocks { - let pos = offset + block * journal.block_size; - image - .read_exact_at(&mut buf, pos) - .context("allocate_ext4_journals: cannot read journal block")?; - image - .write_all_at(&buf, pos) - .context("allocate_ext4_journals: cannot write journal block")?; - } - } - - image - .sync_all() - .context("allocate_ext4_journals: cannot sync image") -} - -pub fn generate_bmap_file(image_file: &str) -> Result<()> { - allocate_ext4_journals(image_file)?; - - let mut bmaptool = Command::new("bmaptool"); - bmaptool - .arg("create") - .arg("-o") - .arg(format!("{image_file}.bmap")) - .arg(image_file); - exec_cmd!(bmaptool); - - Ok(()) -} - -#[cfg(test)] -mod tests { - use crate::file::functions::*; - - const PART_START_LBA: u32 = 2048; - const PART_SECTORS: u32 = 16384; - const FS_BLOCK_SIZE: u64 = 1024; - const MBR_DISK_SIGNATURE: [u8; 4] = [1, 2, 3, 4]; - const MBR_TYPE_LINUX: u8 = 0x83; - const BMAP_BLOCK_SIZE: u64 = 4096; - - fn run(cmd: &mut Command) { - let status = cmd.status().expect("run command"); - assert!(status.success(), "{cmd:?} failed"); - } - - fn create_image_with_ext4_journal(image: &Path) { - let raw = image.with_extension("raw"); - let mut f = fs::File::create(&raw).expect("create image"); - f.set_len(u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE) - .expect("set image size"); - let mut mbr = mbrman::MBR::new_from(&mut f, SECTOR_SIZE as u32, MBR_DISK_SIGNATURE) - .expect("create mbr"); - mbr[1] = mbrman::MBRPartitionEntry { - boot: mbrman::BOOT_INACTIVE, - first_chs: mbrman::CHS::empty(), - sys: MBR_TYPE_LINUX, - last_chs: mbrman::CHS::empty(), - starting_lba: PART_START_LBA, - sectors: PART_SECTORS, - }; - mbr.write_into(&mut f).expect("write mbr"); - - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; - run(Command::new("mkfs.ext4") - .arg("-q") - .arg("-F") - .arg("-b") - .arg(FS_BLOCK_SIZE.to_string()) - .arg("-J") - .arg("size=1") - .arg("-E") - .arg(format!("offset={offset}")) - .arg(&raw) - .arg(fs_blocks.to_string())); - // mke2fs zeroes the journal as unwritten extents, which a bmap maps; a sparse copy - // turns the zeros into holes, as a sparse decompress does - run(Command::new("cp") - .arg("--sparse=always") - .arg(&raw) - .arg(image)); - } - - fn bmap_ranges(bmap: &str) -> Vec<(u64, u64)> { - bmap.lines() - .filter_map(|l| { - let range = l.split_once("')?.1; - let range = range.split_once(" usize { - let ranges = bmap_ranges(bmap); - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - let file = fs::File::open(image).expect("open image"); - let journal = read_ext4_journal(&file, offset) - .expect("read superblock") - .expect("ext4 with journal"); - ext4_journal_blocks(image, offset, journal.inode) - .expect("journal blocks") - .iter() - .map(|b| (offset + b * journal.block_size) / BMAP_BLOCK_SIZE) - .filter(|b| !ranges.iter().any(|(s, e)| s <= b && b <= e)) - .count() - } - - #[test] - fn generate_bmap_file_maps_ext4_journal() { - let dir = tempfile::tempdir().expect("create temp dir"); - let image_path = dir.path().join("image.wic"); - create_image_with_ext4_journal(&image_path); - let image = image_path.to_str().expect("image path"); - let bmap_path = format!("{image}.bmap"); - - run(Command::new("bmaptool") - .arg("create") - .arg("-o") - .arg(&bmap_path) - .arg(image)); - let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); - assert!(unmapped_journal_blocks(image, &bmap) > 0); - - generate_bmap_file(image).expect("generate bmap"); - let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); - assert_eq!(unmapped_journal_blocks(image, &bmap), 0); - - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - run(Command::new("e2fsck") - .arg("-fn") - .arg(format!("{image}?offset={offset}"))); - } -} diff --git a/src/file/mod.rs b/src/file/mod.rs index 6f9db481..e9ccd43e 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -1,3 +1,4 @@ +pub mod bmap; pub mod compression; pub mod functions; mod partition; diff --git a/src/lib.rs b/src/lib.rs index a8ce6bf4..c487ceda 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -114,7 +114,7 @@ where .to_str() .context("cannot get image file path")? )); - file::functions::generate_bmap_file( + file::bmap::generate_bmap_file( tmp_image_file .to_str() .context("cannot get image file path")?, From 9842fef98392f3df43fb94fb84e25f6e66e24103 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Mon, 28 Sep 2026 17:35:36 +0200 Subject: [PATCH 11/46] test(file): cover ext4 journal edge cases and a flash onto old content - a bmap flash onto a device with old content now overwrites the journal - an invalid s_log_block_size is rejected - a partition that starts past the end of the image is skipped - the MBR extended container entry is not returned as a partition Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/bmap.rs | 99 ++++++++++++++++++++++++++++++++++++++++--- src/file/partition.rs | 19 +++++++++ 2 files changed, 113 insertions(+), 5 deletions(-) diff --git a/src/file/bmap.rs b/src/file/bmap.rs index e6b6352b..f31259b7 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -168,17 +168,19 @@ mod tests { const MBR_DISK_SIGNATURE: [u8; 4] = [1, 2, 3, 4]; const MBR_TYPE_LINUX: u8 = 0x83; const BMAP_BLOCK_SIZE: u64 = 4096; + // content an earlier installation left on the device + const OLD_CONTENT_BYTE: u8 = 0xa5; + const JOURNAL_INODE: u32 = 8; + const INVALID_LOG_BLOCK_SIZE: u32 = 40; fn run(cmd: &mut Command) { let status = cmd.status().expect("run command"); assert!(status.success(), "{cmd:?} failed"); } - fn create_image_with_ext4_journal(image: &Path) { - let raw = image.with_extension("raw"); - let mut f = fs::File::create(&raw).expect("create image"); - f.set_len(u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE) - .expect("set image size"); + fn create_image_with_mbr(image: &Path, len: u64) { + let mut f = fs::File::create(image).expect("create image"); + f.set_len(len).expect("set image size"); let mut mbr = mbrman::MBR::new_from(&mut f, SECTOR_SIZE as u32, MBR_DISK_SIGNATURE) .expect("create mbr"); mbr[1] = mbrman::MBRPartitionEntry { @@ -190,6 +192,11 @@ mod tests { sectors: PART_SECTORS, }; mbr.write_into(&mut f).expect("write mbr"); + } + + fn create_image_with_ext4_journal(image: &Path) { + let raw = image.with_extension("raw"); + create_image_with_mbr(&raw, u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE); let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; @@ -241,6 +248,55 @@ mod tests { .count() } + /// Writes the blocks `bmap` maps onto a device that still holds old content, as a bmap + /// flash does. + fn bmap_flash(image: &str, bmap: &str, device: &Path) { + let image = fs::File::open(image).expect("open image"); + let len = image.metadata().expect("image metadata").len(); + fs::write( + device, + vec![OLD_CONTENT_BYTE; usize::try_from(len).expect("image size")], + ) + .expect("write old content"); + let device = fs::OpenOptions::new() + .write(true) + .open(device) + .expect("open device"); + + let mut buf = vec![0u8; usize::try_from(BMAP_BLOCK_SIZE).expect("block size")]; + for (first, last) in bmap_ranges(bmap) { + for block in first..=last { + image + .read_exact_at(&mut buf, block * BMAP_BLOCK_SIZE) + .expect("read image block"); + device + .write_all_at(&buf, block * BMAP_BLOCK_SIZE) + .expect("write device block"); + } + } + } + + fn journal_on_device_matches_image(image: &str, device: &Path) -> bool { + let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; + let image_file = fs::File::open(image).expect("open image"); + let device = fs::File::open(device).expect("open device"); + let journal = read_ext4_journal(&image_file, offset) + .expect("read superblock") + .expect("ext4 with journal"); + + let block_size = usize::try_from(journal.block_size).expect("block size"); + let (mut a, mut b) = (vec![0u8; block_size], vec![0u8; block_size]); + ext4_journal_blocks(image, offset, journal.inode) + .expect("journal blocks") + .iter() + .all(|block| { + let pos = offset + block * journal.block_size; + image_file.read_exact_at(&mut a, pos).expect("read image"); + device.read_exact_at(&mut b, pos).expect("read device"); + a == b + }) + } + #[test] fn generate_bmap_file_maps_ext4_journal() { let dir = tempfile::tempdir().expect("create temp dir"); @@ -256,14 +312,47 @@ mod tests { .arg(image)); let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); assert!(unmapped_journal_blocks(image, &bmap) > 0); + let device = dir.path().join("device"); + bmap_flash(image, &bmap, &device); + assert!(!journal_on_device_matches_image(image, &device)); generate_bmap_file(image).expect("generate bmap"); let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); assert_eq!(unmapped_journal_blocks(image, &bmap), 0); + bmap_flash(image, &bmap, &device); + assert!(journal_on_device_matches_image(image, &device)); let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; run(Command::new("e2fsck") .arg("-fn") .arg(format!("{image}?offset={offset}"))); } + + #[test] + fn read_ext4_journal_rejects_invalid_block_size() { + let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; + let mut set = |o: usize, v: &[u8]| sb[o..o + v.len()].copy_from_slice(v); + set(EXT4_S_MAGIC, &EXT4_MAGIC.to_le_bytes()); + set( + EXT4_S_FEATURE_COMPAT, + &EXT4_FEATURE_COMPAT_HAS_JOURNAL.to_le_bytes(), + ); + set(EXT4_S_JOURNAL_INUM, &JOURNAL_INODE.to_le_bytes()); + set(EXT4_S_LOG_BLOCK_SIZE, &INVALID_LOG_BLOCK_SIZE.to_le_bytes()); + + let image = tempfile::tempfile().expect("create image"); + image + .write_all_at(&sb, EXT4_SUPERBLOCK_OFFSET) + .expect("write superblock"); + assert!(read_ext4_journal(&image, 0).is_err()); + } + + #[test] + fn allocate_ext4_journals_skips_partition_past_image_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + create_image_with_mbr(&image, u64::from(PART_START_LBA) * SECTOR_SIZE); + + allocate_ext4_journals(image.to_str().expect("image path")).expect("allocate journals"); + } } diff --git a/src/file/partition.rs b/src/file/partition.rs index 9d9d0b8c..12cbc5e0 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -97,3 +97,22 @@ pub fn is_gpt>(path: P) -> Result { let mut file = File::open(path.as_ref()).context("is_gpt: failed to open image")?; Ok(gptman::GPT::find_from(&mut file).is_ok()) } + +#[cfg(test)] +mod tests { + use crate::file::partition::*; + + const TEST_IMAGE: &str = "testfiles/image.wic"; + const MBR_EXTENDED_PARTITION: u32 = 4; + + #[test] + fn get_partitions_skips_mbr_extended_entry() { + let nums: Vec = get_partitions(TEST_IMAGE) + .expect("read partitions") + .iter() + .map(|p| p.num) + .collect(); + assert!(!nums.contains(&MBR_EXTENDED_PARTITION)); + assert!(nums.contains(&(MBR_EXTENDED_PARTITION + 1))); + } +} From 68f21f4e1292a217c077e5a128739526cdf9103c Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:51:40 +0200 Subject: [PATCH 12/46] fix(deps): resolve RUSTSEC-2026-0385 (ctutils) Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f70b2cb6..fdcfe234 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -610,9 +610,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", ] From 7e7505261c80299c3a11c64b63b8ddbc61445e16 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:28:36 +0100 Subject: [PATCH 13/46] fix(file): detect truncated input and harden sparse decompress - return decoder errors at the end of the stream - truncate the destination before a sparse decompress - write runs of blocks in one call in SparseWriter - find e2fsprogs tools in sbin, reuse get_partitions, share the image block size Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- project-context.md | 3 +- src/file/bmap.rs | 44 ++++----- src/file/compression.rs | 197 +++++++++++++++++++++++++++++----------- src/file/functions.rs | 48 +++++++++- src/file/mod.rs | 3 + src/file/partition.rs | 49 ++-------- 6 files changed, 221 insertions(+), 123 deletions(-) diff --git a/project-context.md b/project-context.md index 8f3b61d6..3e1c93ad 100644 --- a/project-context.md +++ b/project-context.md @@ -50,7 +50,8 @@ - `src/docker.rs` — `docker pull --platform` + `docker save` for multi-arch images - `src/image.rs` — firmware image architecture detection (ARM32/ARM64/x86_64) - `src/file/mod.rs` — high-level image operations: identity config, certs, hostname patching -- `src/file/functions.rs` — partition read/modify/write via dd + e2cp/mcopy +- `src/file/functions.rs` — partition read via dd, modify via e2cp/mcopy, write back of changed blocks only +- `src/file/bmap.rs` — bmap generation; allocates ext4 journals first (needs `debugfs`) - `src/file/partition.rs` — GPT/MBR partition table parsing - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys diff --git a/src/file/bmap.rs b/src/file/bmap.rs index f31259b7..8b13b2c2 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,9 +1,7 @@ use crate::file::partition::{SECTOR_SIZE, get_partitions}; use anyhow::{Context, Result}; use log::debug; -use std::fs; -use std::os::unix::fs::FileExt; -use std::process::Command; +use std::{env, fs, os::unix::fs::FileExt, process::Command}; const EXT4_SUPERBLOCK_OFFSET: u64 = 1024; const EXT4_SUPERBLOCK_SIZE: usize = 1024; @@ -17,8 +15,8 @@ const EXT4_MIN_BLOCK_SIZE: u64 = 1024; const EXT4_MAX_LOG_BLOCK_SIZE: u32 = 6; const DEBUGFS: &str = "debugfs"; -// debugfs is installed to sbin, which is not always in the PATH of a normal user -const DEBUGFS_SBIN: &str = "/usr/sbin/debugfs"; +// e2fsprogs installs to sbin, which is not always in the PATH of a normal user +const SBIN_DIRS: &str = "/usr/sbin:/sbin"; struct Ext4Journal { inode: u32, @@ -55,26 +53,24 @@ fn read_ext4_journal(image: &fs::File, offset: u64) -> Result Command { + let path = + env::var("PATH").map_or_else(|_| SBIN_DIRS.to_string(), |p| format!("{p}:{SBIN_DIRS}")); + let mut cmd = Command::new(program); + cmd.env("PATH", path); + cmd +} + fn ext4_journal_blocks(image_file: &str, offset: u64, inode: u32) -> Result> { - let debugfs_cmd = |program| { - let mut cmd = Command::new(program); - cmd.arg("-R") - .arg(format!("blocks <{inode}>")) - .arg(format!("{image_file}?offset={offset}")); - cmd - }; - let mut debugfs = debugfs_cmd(DEBUGFS); - let output = match debugfs.output() { - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - debugfs = debugfs_cmd(DEBUGFS_SBIN); - debugfs.output() - } - output => output, - } - .context(format!("ext4_journal_blocks: cannot run {debugfs:?}"))?; + let output = sbin_command(DEBUGFS) + .arg("-R") + .arg(format!("blocks <{inode}>")) + .arg(format!("{image_file}?offset={offset}")) + .output() + .context(format!("ext4_journal_blocks: cannot run {DEBUGFS}"))?; anyhow::ensure!( output.status.success(), - "ext4_journal_blocks: {debugfs:?} failed: {}", + "ext4_journal_blocks: {DEBUGFS} failed: {}", String::from_utf8_lossy(&output.stderr) ); @@ -200,7 +196,7 @@ mod tests { let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; - run(Command::new("mkfs.ext4") + run(sbin_command("mkfs.ext4") .arg("-q") .arg("-F") .arg("-b") @@ -323,7 +319,7 @@ mod tests { assert!(journal_on_device_matches_image(image, &device)); let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - run(Command::new("e2fsck") + run(sbin_command("e2fsck") .arg("-fn") .arg(format!("{image}?offset={offset}"))); } diff --git a/src/file/compression.rs b/src/file/compression.rs index fca98d49..ba5a43d0 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -1,16 +1,16 @@ +use crate::file::IMAGE_BLOCK_SIZE; use anyhow::{Context, Result}; use filemagic::Magic; use log::debug; use std::env; use std::fs::File; use std::io::{Seek, SeekFrom, Write}; +use std::os::unix::fs::FileExt; use std::path::PathBuf; use std::str::FromStr; use strum::IntoEnumIterator; use strum_macros::EnumIter; -const SPARSE_BLOCK_SIZE: u64 = 4096; - #[derive(Clone, Debug, EnumIter)] #[allow(non_camel_case_types)] pub enum Compression { @@ -79,22 +79,24 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let sparse = SparseWriter(destination); - let mut dec: Box = match &self { - Compression::bzip2 => Box::new(bzip2::write::BzDecoder::new(sparse)), - Compression::gzip => Box::new(flate2::write::GzDecoder::new(sparse)), - Compression::xz { .. } => Box::new(xz2::write::XzDecoder::new(sparse)), + let sparse = SparseWriter::new(destination)?; + // only `finish` detects a truncated or corrupt stream + let (bytes_read, sparse) = match &self { + Compression::bzip2 => { + let mut dec = bzip2::write::BzDecoder::new(sparse); + (std::io::copy(source, &mut dec)?, dec.finish()?) + } + Compression::gzip => { + let mut dec = flate2::write::GzDecoder::new(sparse); + (std::io::copy(source, &mut dec)?, dec.finish()?) + } + Compression::xz { .. } => { + let mut dec = xz2::write::XzDecoder::new(sparse); + (std::io::copy(source, &mut dec)?, dec.finish()?) + } }; - - let bytes_written = std::io::copy(source, &mut dec)?; - dec.write_all(&[])?; - dec.flush()?; - drop(dec); - - // zero blocks at the end were only seeked over - let len = destination.stream_position()?; - destination.set_len(len)?; - Ok(bytes_written) + sparse.finish()?; + Ok(bytes_read) } fn marker(&self) -> &'static str { @@ -135,24 +137,55 @@ impl Compression { } } -/// Seeks over zero blocks instead of writing them, so they stay holes in a new file. -struct SparseWriter<'a>(&'a mut File); +/// Skips zero blocks, so they stay holes in the file. +struct SparseWriter<'a> { + file: &'a mut File, + pos: u64, +} + +impl<'a> SparseWriter<'a> { + fn new(file: &'a mut File) -> std::io::Result { + let pos = file.stream_position()?; + // skipped zero blocks must read back as zeros + file.set_len(pos)?; + Ok(Self { file, pos }) + } + + fn finish(self) -> std::io::Result<()> { + // extends the file over zero blocks at the end + self.file.set_len(self.pos)?; + self.file.seek(SeekFrom::Start(self.pos))?; + Ok(()) + } +} impl Write for SparseWriter<'_> { fn write(&mut self, buf: &[u8]) -> std::io::Result { - let pos = self.0.stream_position()?; - let to_block_end = SPARSE_BLOCK_SIZE - pos % SPARSE_BLOCK_SIZE; - let chunk = &buf[..buf.len().min(to_block_end as usize)]; - if chunk.iter().all(|b| *b == 0) { - self.0.seek(SeekFrom::Current(chunk.len() as i64))?; - Ok(chunk.len()) - } else { - self.0.write(chunk) + let is_zero = |b: &[u8]| b.iter().all(|x| *x == 0); + let block_offset = (self.pos % IMAGE_BLOCK_SIZE as u64) as usize; + + // one call handles a run of blocks that are all zero or all data + let mut end = buf.len().min(IMAGE_BLOCK_SIZE - block_offset); + let zero = is_zero(&buf[..end]); + while end < buf.len() { + let next = buf.len().min(end + IMAGE_BLOCK_SIZE); + if is_zero(&buf[end..next]) != zero { + break; + } + end = next; } + + let written = if zero { + end + } else { + self.file.write_at(&buf[..end], self.pos)? + }; + self.pos += written as u64; + Ok(written) } fn flush(&mut self) -> std::io::Result<()> { - self.0.flush() + self.file.flush() } } @@ -197,9 +230,31 @@ mod tests { // unit of `st_blocks` const STAT_BLOCK_SIZE: u64 = 512; const DATA_OFFSET: usize = 1024 * 1024 + 100; + const DATA: &[u8] = b"abc"; const IMAGE_LEN: usize = 3 * 1024 * 1024 + 123; + const XZ_TEST_LEVEL: u32 = 1; + // cuts into the trailer, which every format checks only at the end of the stream + const TRUNCATED_BYTES: u64 = 4; + const OLD_CONTENT_BYTE: u8 = 0xa5; + const OLD_CONTENT_EXTRA_LEN: usize = 4096; + + fn all_compressions() -> [Compression; 3] { + [ + Compression::bzip2, + Compression::gzip, + Compression::xz { + compression_level: XZ_TEST_LEVEL, + }, + ] + } + + fn test_image() -> Vec { + let mut image = vec![0u8; IMAGE_LEN]; + image[DATA_OFFSET..DATA_OFFSET + DATA.len()].copy_from_slice(DATA); + image + } - fn roundtrip(compression: Compression, image: &[u8]) -> File { + fn compress_image(compression: &Compression, image: &[u8]) -> File { let mut source = tempfile::tempfile().expect("create source"); source.write_all(image).expect("write source"); source.rewind().expect("rewind source"); @@ -209,35 +264,31 @@ mod tests { .compress(&mut source, &mut compressed) .expect("compress"); compressed.rewind().expect("rewind compressed"); + compressed + } - let mut decompressed = tempfile::tempfile().expect("create decompressed"); - compression - .decompress(&mut compressed, &mut decompressed) - .expect("decompress"); - decompressed + fn read_all(file: &mut File) -> Vec { + let mut content = Vec::new(); + file.rewind().expect("rewind"); + file.read_to_end(&mut content).expect("read"); + content } #[test] fn decompress_keeps_zero_blocks_as_holes() { - let mut image = vec![0u8; IMAGE_LEN]; - image[DATA_OFFSET..DATA_OFFSET + 3].copy_from_slice(b"abc"); + let image = test_image(); - for compression in [ - Compression::bzip2, - Compression::gzip, - Compression::xz { - compression_level: 1, - }, - ] { - let mut decompressed = roundtrip(compression.clone(), &image); - - let mut content = Vec::new(); - decompressed.rewind().expect("rewind decompressed"); - decompressed - .read_to_end(&mut content) - .expect("read decompressed"); - assert!(content == image, "{compression:?}: content differs"); + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + compression + .decompress(&mut compressed, &mut decompressed) + .expect("decompress"); + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); let meta = decompressed.metadata().expect("metadata"); assert!( meta.blocks() * STAT_BLOCK_SIZE < meta.len(), @@ -245,4 +296,48 @@ mod tests { ); } } + + #[test] + fn decompress_replaces_old_content() { + let image = test_image(); + + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + decompressed + .write_all(&vec![OLD_CONTENT_BYTE; IMAGE_LEN + OLD_CONTENT_EXTRA_LEN]) + .expect("write old content"); + decompressed.rewind().expect("rewind decompressed"); + + compression + .decompress(&mut compressed, &mut decompressed) + .expect("decompress"); + + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); + } + } + + #[test] + fn decompress_rejects_truncated_input() { + let image = test_image(); + + for compression in all_compressions() { + let mut compressed = compress_image(&compression, &image); + let len = compressed.metadata().expect("metadata").len(); + compressed + .set_len(len - TRUNCATED_BYTES) + .expect("truncate compressed"); + + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + assert!( + compression + .decompress(&mut compressed, &mut decompressed) + .is_err(), + "{compression:?}: truncated input accepted" + ); + } + } } diff --git a/src/file/functions.rs b/src/file/functions.rs index d6591e9f..5f8eb6d0 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,4 +1,4 @@ -use crate::file::partition::SECTOR_SIZE; +use crate::file::{IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE}; use anyhow::{Context, Result}; use log::{debug, warn}; use std::collections::HashMap; @@ -11,7 +11,6 @@ use std::str::FromStr; use stdext::function_name; use uuid::Uuid; -const COMPARE_BLOCK_SIZE: usize = 4096; const COPY_CHUNK_SIZE: usize = 1024 * 1024; #[derive(clap::ValueEnum, Debug, Clone, Eq, Hash, PartialEq)] @@ -468,12 +467,12 @@ fn write_partition( .context("write_partition: cannot read image")?; let blocks = src[..n] - .chunks(COMPARE_BLOCK_SIZE) - .zip(dst[..n].chunks(COMPARE_BLOCK_SIZE)); + .chunks(IMAGE_BLOCK_SIZE) + .zip(dst[..n].chunks(IMAGE_BLOCK_SIZE)); for (i, (s, d)) in blocks.enumerate() { if s != d { image - .write_all_at(s, image_offset + pos + (i * COMPARE_BLOCK_SIZE) as u64) + .write_all_at(s, image_offset + pos + (i * IMAGE_BLOCK_SIZE) as u64) .context("write_partition: cannot write image")?; } } @@ -484,3 +483,42 @@ fn write_partition( .sync_all() .context("write_partition: cannot sync image") } + +#[cfg(test)] +mod tests { + use crate::file::functions::*; + + const PART_NUM: u32 = 1; + const PART_START: u64 = 8; + const PART_SECTORS: u64 = 64; + const OLD_CONTENT_BYTE: u8 = 0xa5; + const ZEROED_BLOCK: usize = 2; + + #[test] + fn write_partition_writes_block_changed_to_zeros() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let image_len = usize::try_from((PART_START + PART_SECTORS) * SECTOR_SIZE).expect("size"); + let part_offset = usize::try_from(PART_START * SECTOR_SIZE).expect("offset"); + + fs::write(&image, vec![OLD_CONTENT_BYTE; image_len]).expect("write image"); + let mut content = vec![OLD_CONTENT_BYTE; image_len - part_offset]; + content[ZEROED_BLOCK * IMAGE_BLOCK_SIZE..(ZEROED_BLOCK + 1) * IMAGE_BLOCK_SIZE].fill(0); + fs::write(&partition, &content).expect("write partition"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &PartitionInfo { + num: PART_NUM, + start: PART_START, + count: PART_SECTORS, + }, + ) + .expect("write partition"); + + let written = fs::read(&image).expect("read image"); + assert!(written[part_offset..] == content[..], "partition differs"); + } +} diff --git a/src/file/mod.rs b/src/file/mod.rs index e9ccd43e..073eb7dd 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -2,6 +2,9 @@ pub mod bmap; pub mod compression; pub mod functions; mod partition; + +/// Granularity at which zero data in an image file is kept as a hole. +const IMAGE_BLOCK_SIZE: usize = 4096; use super::validators::{ device_update, identity::{IdentityConfig, IdentityType, validate_identity}, diff --git a/src/file/partition.rs b/src/file/partition.rs index 12cbc5e0..36fe983d 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -13,48 +13,10 @@ pub struct PartitionData { } pub fn get_partition_data>(path: P, partition_num: u32) -> Result { - let path = path.as_ref(); - let mut file = - File::open(path).with_context(|| format!("failed to open image: {}", path.display()))?; - - // Try GPT first (validates CRC32 — more robust than signature check). - // Capture any error so we can attach it as context if MBR also fails. - let gpt_err = match gptman::GPT::find_from(&mut file) { - Ok(gpt) => { - let entry = gpt - .iter() - .find(|(i, _)| *i == partition_num) - .map(|(_, e)| e) - .with_context(|| format!("GPT partition {partition_num} out of range"))?; - anyhow::ensure!(entry.is_used(), "GPT partition {partition_num} is not used"); - anyhow::ensure!( - entry.ending_lba >= entry.starting_lba, - "GPT partition {partition_num} has invalid LBA range (ending_lba < starting_lba)" - ); - return Ok(PartitionData { - num: partition_num, - start: entry.starting_lba, - count: entry.ending_lba - entry.starting_lba + 1, - }); - } - Err(e) => e, - }; - - // Try MBR — iter() includes logical partitions (5, 6, ...) - file.seek(SeekFrom::Start(0)) - .context("failed to seek to start of image")?; - let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) - .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; - for (i, p) in mbr.iter() { - if u32::try_from(i).ok() == Some(partition_num) && p.is_used() { - return Ok(PartitionData { - num: partition_num, - start: p.starting_lba as u64, - count: p.sectors as u64, - }); - } - } - anyhow::bail!("partition {partition_num} not found in image") + get_partitions(path)? + .into_iter() + .find(|p| p.num == partition_num) + .with_context(|| format!("partition {partition_num} not found in image")) } pub fn get_partitions>(path: P) -> Result> { @@ -62,6 +24,8 @@ pub fn get_partitions>(path: P) -> Result> { let mut file = File::open(path).with_context(|| format!("failed to open image: {}", path.display()))?; + // Try GPT first (validates CRC32 — more robust than signature check). + // Capture any error so we can attach it as context if MBR also fails. let gpt_err = match gptman::GPT::find_from(&mut file) { Ok(gpt) => { return Ok(gpt @@ -77,6 +41,7 @@ pub fn get_partitions>(path: P) -> Result> { Err(e) => e, }; + // Try MBR — iter() includes logical partitions (5, 6, ...) file.seek(SeekFrom::Start(0)) .context("failed to seek to start of image")?; let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) From a0da23c701028c0e513d2dd2a39e5b51b8e68438 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 08:48:18 +0100 Subject: [PATCH 14/46] fix(file): keep the ranges of the input bmap mapped Sparse decompress turns every zero block into a hole, and a bmap created from it skips them: the GPT entries, FAT sectors of boot and ext4 journals and metadata. A bmap-based flash then keeps what an earlier installation left there. Take the bmap of the input image (next to it, or --bmap) and write the zero blocks of its ranges as data. Without an input bmap, every block is written. This replaces the ext4 journal handling, so debugfs and e2fsprogs are no longer needed. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 10 + Cargo.toml | 3 +- Dockerfile | 1 - README.md | 6 + project-context.md | 2 +- src/cli.rs | 30 ++ src/file/bmap.rs | 580 ++++++++++++++++++++----------------- src/file/functions.rs | 4 +- src/file/mod.rs | 1 + src/lib.rs | 110 +++++-- tests/integration_tests.rs | 86 ++++-- 11 files changed, 501 insertions(+), 332 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fdcfe234..b824097f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1793,6 +1793,7 @@ dependencies = [ "regex", "reqwest 0.13.5", "ring", + "roxmltree", "serde", "serde_json", "serde_path_to_error", @@ -2222,6 +2223,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "roxmltree" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9da7f59328b09405b181fee1018925fad50f157afb43c0c0f5c522c256dd5907" +dependencies = [ + "memchr", +] + [[package]] name = "rustc_version" version = "0.4.1" diff --git a/Cargo.toml b/Cargo.toml index 8e90f87f..35d5c6eb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,7 @@ reqwest = { version = "0.13", default-features = false, features = [ "json", "native-tls", ] } +roxmltree = { version = "0.21", default-features = false, features = ["std"] } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = { version = "1.0", default-features = false } serde_path_to_error = { version = "0.1", default-features = false } @@ -81,5 +82,5 @@ tar = "0.4" # metadata for building with cargo-deb (https://crates.io/crates/cargo-deb) [package.metadata.deb] -depends = "bmap-tools, e2fsprogs, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" +depends = "bmap-tools, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" revision = "" diff --git a/Dockerfile b/Dockerfile index 03389a9a..9300a868 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,6 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ bmap-tools \ ca-certificates \ - e2fsprogs \ e2tools \ fdisk \ keychain \ diff --git a/README.md b/README.md index a3cc9c44..18133e7c 100644 --- a/README.md +++ b/README.md @@ -184,6 +184,12 @@ omnect-cli file copy-to-image --help - File permissions: inject `systemd-tmpfiles.d` - Wifi: inject `wpa_supplicant-wlan0.conf` +## bmap files + +Commands that change an image create a bmap file with `-b`. The bmap of the input image is the base for it: `omnect-cli` takes `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it, or the file given with `--bmap`. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges plus the blocks `omnect-cli` changed. + +Without an input bmap, the created bmap maps the whole image, and a flash writes every block. + ## ssh tunnel ### Inject ssh tunnel credentials diff --git a/project-context.md b/project-context.md index 3e1c93ad..554c6b73 100644 --- a/project-context.md +++ b/project-context.md @@ -51,7 +51,7 @@ - `src/image.rs` — firmware image architecture detection (ARM32/ARM64/x86_64) - `src/file/mod.rs` — high-level image operations: identity config, certs, hostname patching - `src/file/functions.rs` — partition read via dd, modify via e2cp/mcopy, write back of changed blocks only -- `src/file/bmap.rs` — bmap generation; allocates ext4 journals first (needs `debugfs`) +- `src/file/bmap.rs` — input bmap lookup; maps its ranges in the working image, or the whole image without one; bmap generation via `bmaptool create` - `src/file/partition.rs` — GPT/MBR partition table parsing - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys diff --git a/src/cli.rs b/src/cli.rs index 746e2537..6b104ac4 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -37,6 +37,9 @@ pub enum Docker { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -58,6 +61,9 @@ pub enum File { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -91,6 +97,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -115,6 +124,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -133,6 +145,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -157,6 +172,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -175,6 +193,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -199,6 +220,9 @@ pub enum IdentityConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -220,6 +244,9 @@ pub enum IotHubDeviceUpdate { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -339,6 +366,9 @@ pub enum SshConfig { /// optional: generate bmap file (currently not working in docker image) #[arg(short = 'b', long = "generate-bmap-file")] generate_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "bmap")] + bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 8b13b2c2..605c394b 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,149 +1,194 @@ -use crate::file::partition::{SECTOR_SIZE, get_partitions}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE}; use anyhow::{Context, Result}; -use log::debug; -use std::{env, fs, os::unix::fs::FileExt, process::Command}; - -const EXT4_SUPERBLOCK_OFFSET: u64 = 1024; -const EXT4_SUPERBLOCK_SIZE: usize = 1024; -const EXT4_S_LOG_BLOCK_SIZE: usize = 0x18; -const EXT4_S_MAGIC: usize = 0x38; -const EXT4_S_FEATURE_COMPAT: usize = 0x5c; -const EXT4_S_JOURNAL_INUM: usize = 0xe0; -const EXT4_MAGIC: u16 = 0xef53; -const EXT4_FEATURE_COMPAT_HAS_JOURNAL: u32 = 0x4; -const EXT4_MIN_BLOCK_SIZE: u64 = 1024; -const EXT4_MAX_LOG_BLOCK_SIZE: u32 = 6; - -const DEBUGFS: &str = "debugfs"; -// e2fsprogs installs to sbin, which is not always in the PATH of a normal user -const SBIN_DIRS: &str = "/usr/sbin:/sbin"; - -struct Ext4Journal { - inode: u32, - block_size: u64, +use std::{ + fs, + ops::RangeInclusive, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + process::Command, +}; + +const BMAP_EXTENSION: &str = "bmap"; +const COMPRESSION_EXTENSIONS: [&str; 5] = ["xz", "gz", "gzip", "bz2", "bzip2"]; + +/// The block ranges of an image that a bmap file maps. +#[derive(Debug)] +pub struct Bmap { + pub block_size: u64, + pub image_size: u64, + pub ranges: Vec>, } -fn read_ext4_journal(image: &fs::File, offset: u64) -> Result> { - let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; - image - .read_exact_at(&mut sb, offset + EXT4_SUPERBLOCK_OFFSET) - .context("read_ext4_journal: cannot read superblock")?; - let le16 = |o: usize| u16::from_le_bytes([sb[o], sb[o + 1]]); - let le32 = |o: usize| u32::from_le_bytes([sb[o], sb[o + 1], sb[o + 2], sb[o + 3]]); - - if le16(EXT4_S_MAGIC) != EXT4_MAGIC - || le32(EXT4_S_FEATURE_COMPAT) & EXT4_FEATURE_COMPAT_HAS_JOURNAL == 0 - { - return Ok(None); - } +impl Bmap { + pub fn read(path: &Path) -> Result { + let text = fs::read_to_string(path) + .context(format!("Bmap::read: cannot read {}", path.display()))?; + let doc = roxmltree::Document::parse(&text) + .context(format!("Bmap::read: invalid xml in {}", path.display()))?; + let root = doc.root_element(); + let child = |name: &str| { + root.children() + .find(|n| n.has_tag_name(name)) + .context(format!("Bmap::read: no {name} in {}", path.display())) + }; + let number = |name: &str| -> Result { + child(name)? + .text() + .unwrap_or_default() + .trim() + .parse() + .context(format!("Bmap::read: invalid {name} in {}", path.display())) + }; - // an external journal has no inode in this filesystem - let inode = le32(EXT4_S_JOURNAL_INUM); - if inode == 0 { - return Ok(None); + let block_size = number("BlockSize")?; + anyhow::ensure!(block_size > 0, "Bmap::read: BlockSize is 0"); + let image_size = number("ImageSize")?; + let ranges = child("BlockMap")? + .children() + .filter(|n| n.has_tag_name("Range")) + .map(|n| parse_range(n.text().unwrap_or_default())) + .collect::>()?; + + Ok(Self { + block_size, + image_size, + ranges, + }) } - let log_block_size = le32(EXT4_S_LOG_BLOCK_SIZE); - anyhow::ensure!( - log_block_size <= EXT4_MAX_LOG_BLOCK_SIZE, - "read_ext4_journal: invalid block size" - ); - let block_size = EXT4_MIN_BLOCK_SIZE << log_block_size; + pub fn whole_image(image_size: u64) -> Self { + let block_size = IMAGE_BLOCK_SIZE as u64; + let ranges = match image_size { + 0 => vec![], + size => vec![RangeInclusive::new(0, (size - 1) / block_size)], + }; - Ok(Some(Ext4Journal { inode, block_size })) + Self { + block_size, + image_size, + ranges, + } + } } -fn sbin_command(program: &str) -> Command { - let path = - env::var("PATH").map_or_else(|_| SBIN_DIRS.to_string(), |p| format!("{p}:{SBIN_DIRS}")); - let mut cmd = Command::new(program); - cmd.env("PATH", path); - cmd +fn parse_range(text: &str) -> Result> { + let text = text.trim(); + let (first, last) = text.split_once('-').unwrap_or((text, text)); + let parse = |s: &str| { + s.trim() + .parse::() + .context(format!("parse_range: invalid range {text}")) + }; + let (first, last) = (parse(first)?, parse(last)?); + anyhow::ensure!(first <= last, "parse_range: invalid range {text}"); + + Ok(first..=last) } -fn ext4_journal_blocks(image_file: &str, offset: u64, inode: u32) -> Result> { - let output = sbin_command(DEBUGFS) - .arg("-R") - .arg(format!("blocks <{inode}>")) - .arg(format!("{image_file}?offset={offset}")) - .output() - .context(format!("ext4_journal_blocks: cannot run {DEBUGFS}"))?; - anyhow::ensure!( - output.status.success(), - "ext4_journal_blocks: {DEBUGFS} failed: {}", - String::from_utf8_lossy(&output.stderr) - ); +/// Returns `explicit`, else `.bmap`, else `.bmap`, +/// whichever exists first. +pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result> { + if let Some(path) = explicit { + anyhow::ensure!( + path.is_file(), + "find_input_bmap: {} not found", + path.display() + ); + return Ok(Some(path.to_path_buf())); + } - let blocks = String::from_utf8(output.stdout) - .context("ext4_journal_blocks: invalid debugfs output")? - .split_whitespace() - .map(|b| { - b.parse::() - .context(format!("ext4_journal_blocks: invalid block number {b}")) - }) - .collect::>>()?; - anyhow::ensure!( - !blocks.is_empty(), - "ext4_journal_blocks: no journal blocks at offset {offset}" - ); + let with_bmap_extension = |p: &Path| { + let mut path = p.as_os_str().to_owned(); + path.push(format!(".{BMAP_EXTENSION}")); + PathBuf::from(path) + }; + let mut candidates = vec![with_bmap_extension(image)]; + if image + .extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| COMPRESSION_EXTENSIONS.contains(&e)) + { + candidates.push(with_bmap_extension(&image.with_extension(""))); + } - Ok(blocks) + Ok(candidates.into_iter().find(|p| p.is_file())) } -/// Rewrites the journal blocks of every ext4 partition with their own content. Blocks that -/// are holes become allocated, so a bmap maps the whole journal and a bmap-based flash -/// overwrites the journal an earlier installation left on the device. -fn allocate_ext4_journals(image_file: &str) -> Result<()> { +/// Writes the zero blocks in the ranges of `bmap` as data. A zero block can be a hole, and a +/// bmap created from the image would skip it, so a bmap-based flash would keep what an earlier +/// installation left there. +pub fn allocate_ranges(image_file: &Path, bmap: &Bmap) -> Result<()> { let image = fs::OpenOptions::new() .read(true) .write(true) .open(image_file) - .context(format!("allocate_ext4_journals: cannot open {image_file}"))?; + .context(format!( + "allocate_ranges: cannot open {}", + image_file.display() + ))?; let image_len = image .metadata() - .context("allocate_ext4_journals: cannot get image size")? + .context("allocate_ranges: cannot get image size")? .len(); + anyhow::ensure!( + bmap.image_size == image_len, + "allocate_ranges: bmap is for an image of {} bytes, the image has {image_len}", + bmap.image_size + ); - for partition in get_partitions(image_file)? { - let offset = partition.start * SECTOR_SIZE; - if offset + EXT4_SUPERBLOCK_OFFSET + EXT4_SUPERBLOCK_SIZE as u64 > image_len { - continue; - } - let Some(journal) = read_ext4_journal(&image, offset)? else { - continue; - }; - let blocks = ext4_journal_blocks(image_file, offset, journal.inode)?; - debug!( - "allocate_ext4_journals: partition {}: {} journal blocks", - partition.num, - blocks.len() + let image_blocks = image_len.div_ceil(bmap.block_size); + let mut buf = vec![0u8; COPY_CHUNK_SIZE]; + for range in &bmap.ranges { + anyhow::ensure!( + *range.end() < image_blocks, + "allocate_ranges: range {range:?} is outside the image" ); + let start = range.start() * bmap.block_size; + let end = ((range.end() + 1) * bmap.block_size).min(image_len); - let mut buf = vec![0u8; usize::try_from(journal.block_size)?]; - for block in blocks { - let pos = offset + block * journal.block_size; + let mut pos = start; + while pos < end { + let n = usize::try_from((end - pos).min(COPY_CHUNK_SIZE as u64))?; + let chunk = &mut buf[..n]; image - .read_exact_at(&mut buf, pos) - .context("allocate_ext4_journals: cannot read journal block")?; - image - .write_all_at(&buf, pos) - .context("allocate_ext4_journals: cannot write journal block")?; + .read_exact_at(chunk, pos) + .context("allocate_ranges: cannot read image")?; + write_zero_blocks(&image, chunk, pos)?; + pos += n as u64; } } image .sync_all() - .context("allocate_ext4_journals: cannot sync image") + .context("allocate_ranges: cannot sync image") } -pub fn generate_bmap_file(image_file: &str) -> Result<()> { - allocate_ext4_journals(image_file)?; +fn write_zero_blocks(image: &fs::File, chunk: &[u8], pos: u64) -> Result<()> { + let is_zero = |b: &[u8]| b.iter().all(|x| *x == 0); + let blocks: Vec<(u64, &[u8])> = if is_zero(chunk) { + vec![(pos, chunk)] + } else { + chunk + .chunks(IMAGE_BLOCK_SIZE) + .enumerate() + .filter(|(_, block)| is_zero(block)) + .map(|(i, block)| (pos + (i * IMAGE_BLOCK_SIZE) as u64, block)) + .collect() + }; + + blocks.into_iter().try_for_each(|(pos, block)| { + image + .write_all_at(block, pos) + .context("allocate_ranges: cannot write image") + }) +} +pub fn generate_bmap_file(image_file: &str) -> Result<()> { let mut bmaptool = Command::new("bmaptool"); bmaptool .arg("create") .arg("-o") - .arg(format!("{image_file}.bmap")) + .arg(format!("{image_file}.{BMAP_EXTENSION}")) .arg(image_file); let status = bmaptool .status() @@ -156,199 +201,194 @@ pub fn generate_bmap_file(image_file: &str) -> Result<()> { #[cfg(test)] mod tests { use crate::file::bmap::*; - use std::path::Path; - - const PART_START_LBA: u32 = 2048; - const PART_SECTORS: u32 = 16384; - const FS_BLOCK_SIZE: u64 = 1024; - const MBR_DISK_SIGNATURE: [u8; 4] = [1, 2, 3, 4]; - const MBR_TYPE_LINUX: u8 = 0x83; - const BMAP_BLOCK_SIZE: u64 = 4096; + + const BLOCK_SIZE: u64 = IMAGE_BLOCK_SIZE as u64; + const IMAGE_BLOCKS: u64 = 16; + const DATA_BLOCK: u64 = 0; + const ZERO_BLOCKS: RangeInclusive = 2..=3; + const UNMAPPED_BLOCK: u64 = 5; + const DATA_BYTE: u8 = 0x5a; // content an earlier installation left on the device const OLD_CONTENT_BYTE: u8 = 0xa5; - const JOURNAL_INODE: u32 = 8; - const INVALID_LOG_BLOCK_SIZE: u32 = 40; - - fn run(cmd: &mut Command) { - let status = cmd.status().expect("run command"); - assert!(status.success(), "{cmd:?} failed"); + const YOCTO_BMAP: &str = "testfiles/image.wic.bmap"; + const YOCTO_IMAGE_SIZE: u64 = 30408704; + const YOCTO_MAPPED_BLOCKS: u64 = 77; + + fn bmap_xml(image_size: u64, block_map: &str) -> String { + format!( + "{image_size}\ + {BLOCK_SIZE}{block_map}" + ) } - fn create_image_with_mbr(image: &Path, len: u64) { - let mut f = fs::File::create(image).expect("create image"); - f.set_len(len).expect("set image size"); - let mut mbr = mbrman::MBR::new_from(&mut f, SECTOR_SIZE as u32, MBR_DISK_SIGNATURE) - .expect("create mbr"); - mbr[1] = mbrman::MBRPartitionEntry { - boot: mbrman::BOOT_INACTIVE, - first_chs: mbrman::CHS::empty(), - sys: MBR_TYPE_LINUX, - last_chs: mbrman::CHS::empty(), - starting_lba: PART_START_LBA, - sectors: PART_SECTORS, - }; - mbr.write_into(&mut f).expect("write mbr"); + fn write_bmap(path: &Path, image_size: u64, ranges: &[RangeInclusive]) { + let block_map: String = ranges + .iter() + .map(|r| format!("{}-{}", r.start(), r.end())) + .collect(); + fs::write(path, bmap_xml(image_size, &block_map)).expect("write bmap"); } - fn create_image_with_ext4_journal(image: &Path) { - let raw = image.with_extension("raw"); - create_image_with_mbr(&raw, u64::from(PART_START_LBA + PART_SECTORS) * SECTOR_SIZE); - - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - let fs_blocks = u64::from(PART_SECTORS) * SECTOR_SIZE / FS_BLOCK_SIZE; - run(sbin_command("mkfs.ext4") - .arg("-q") - .arg("-F") - .arg("-b") - .arg(FS_BLOCK_SIZE.to_string()) - .arg("-J") - .arg("size=1") - .arg("-E") - .arg(format!("offset={offset}")) - .arg(&raw) - .arg(fs_blocks.to_string())); - // mke2fs zeroes the journal as unwritten extents, which a bmap maps; a sparse copy - // turns the zeros into holes, as a sparse decompress does - run(Command::new("cp") - .arg("--sparse=always") - .arg(&raw) - .arg(image)); + /// A sparse image with data in `DATA_BLOCK` and holes everywhere else. + fn sparse_image(dir: &Path) -> PathBuf { + let path = dir.join("image.wic"); + let image = fs::File::create(&path).expect("create image"); + image + .set_len(IMAGE_BLOCKS * BLOCK_SIZE) + .expect("set image size"); + image + .write_all_at(&[DATA_BYTE; IMAGE_BLOCK_SIZE], DATA_BLOCK * BLOCK_SIZE) + .expect("write data block"); + path } - fn bmap_ranges(bmap: &str) -> Vec<(u64, u64)> { - bmap.lines() - .filter_map(|l| { - let range = l.split_once("')?.1; - let range = range.split_once(" Bmap { + let image = image.to_str().expect("image path"); + generate_bmap_file(image).expect("generate bmap"); + Bmap::read(Path::new(&format!("{image}.{BMAP_EXTENSION}"))).expect("read bmap") } - fn unmapped_journal_blocks(image: &str, bmap: &str) -> usize { - let ranges = bmap_ranges(bmap); - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - let file = fs::File::open(image).expect("open image"); - let journal = read_ext4_journal(&file, offset) - .expect("read superblock") - .expect("ext4 with journal"); - ext4_journal_blocks(image, offset, journal.inode) - .expect("journal blocks") - .iter() - .map(|b| (offset + b * journal.block_size) / BMAP_BLOCK_SIZE) - .filter(|b| !ranges.iter().any(|(s, e)| s <= b && b <= e)) - .count() + fn is_mapped(bmap: &Bmap, block: u64) -> bool { + bmap.ranges.iter().any(|r| r.contains(&block)) } /// Writes the blocks `bmap` maps onto a device that still holds old content, as a bmap /// flash does. - fn bmap_flash(image: &str, bmap: &str, device: &Path) { - let image = fs::File::open(image).expect("open image"); - let len = image.metadata().expect("image metadata").len(); - fs::write( - device, - vec![OLD_CONTENT_BYTE; usize::try_from(len).expect("image size")], - ) - .expect("write old content"); - let device = fs::OpenOptions::new() - .write(true) - .open(device) - .expect("open device"); - - let mut buf = vec![0u8; usize::try_from(BMAP_BLOCK_SIZE).expect("block size")]; - for (first, last) in bmap_ranges(bmap) { - for block in first..=last { - image - .read_exact_at(&mut buf, block * BMAP_BLOCK_SIZE) - .expect("read image block"); - device - .write_all_at(&buf, block * BMAP_BLOCK_SIZE) - .expect("write device block"); - } + fn bmap_flash(image: &Path, bmap: &Bmap, device: &Path) { + let image = fs::read(image).expect("read image"); + let mut content = vec![OLD_CONTENT_BYTE; image.len()]; + for range in &bmap.ranges { + let start = usize::try_from(range.start() * bmap.block_size).expect("start"); + let end = usize::try_from((range.end() + 1) * bmap.block_size).expect("end"); + let end = end.min(image.len()); + content[start..end].copy_from_slice(&image[start..end]); } + fs::write(device, content).expect("write device"); } - fn journal_on_device_matches_image(image: &str, device: &Path) -> bool { - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - let image_file = fs::File::open(image).expect("open image"); - let device = fs::File::open(device).expect("open device"); - let journal = read_ext4_journal(&image_file, offset) - .expect("read superblock") - .expect("ext4 with journal"); - - let block_size = usize::try_from(journal.block_size).expect("block size"); - let (mut a, mut b) = (vec![0u8; block_size], vec![0u8; block_size]); - ext4_journal_blocks(image, offset, journal.inode) - .expect("journal blocks") - .iter() - .all(|block| { - let pos = offset + block * journal.block_size; - image_file.read_exact_at(&mut a, pos).expect("read image"); - device.read_exact_at(&mut b, pos).expect("read device"); - a == b - }) + #[test] + fn read_parses_yocto_bmap() { + let bmap = Bmap::read(Path::new(YOCTO_BMAP)).expect("read bmap"); + + assert_eq!(bmap.block_size, BLOCK_SIZE); + assert_eq!(bmap.image_size, YOCTO_IMAGE_SIZE); + let mapped: u64 = bmap.ranges.iter().map(|r| r.end() - r.start() + 1).sum(); + assert_eq!(mapped, YOCTO_MAPPED_BLOCKS); } #[test] - fn generate_bmap_file_maps_ext4_journal() { + fn read_parses_single_blocks_and_spans() { let dir = tempfile::tempdir().expect("create temp dir"); - let image_path = dir.path().join("image.wic"); - create_image_with_ext4_journal(&image_path); - let image = image_path.to_str().expect("image path"); - let bmap_path = format!("{image}.bmap"); - - run(Command::new("bmaptool") - .arg("create") - .arg("-o") - .arg(&bmap_path) - .arg(image)); - let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); - assert!(unmapped_journal_blocks(image, &bmap) > 0); - let device = dir.path().join("device"); - bmap_flash(image, &bmap, &device); - assert!(!journal_on_device_matches_image(image, &device)); + let path = dir.path().join("image.wic.bmap"); + let block_map = " 3 5-7 "; + fs::write(&path, bmap_xml(BLOCK_SIZE, block_map)).expect("write bmap"); - generate_bmap_file(image).expect("generate bmap"); - let bmap = fs::read_to_string(&bmap_path).expect("read bmap"); - assert_eq!(unmapped_journal_blocks(image, &bmap), 0); - bmap_flash(image, &bmap, &device); - assert!(journal_on_device_matches_image(image, &device)); - - let offset = u64::from(PART_START_LBA) * SECTOR_SIZE; - run(sbin_command("e2fsck") - .arg("-fn") - .arg(format!("{image}?offset={offset}"))); + let bmap = Bmap::read(&path).expect("read bmap"); + assert_eq!(bmap.ranges, vec![3..=3, 5..=7]); + } + + #[test] + fn read_rejects_invalid_bmaps() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic.bmap"); + let invalid = [ + "1".to_string(), + bmap_xml(BLOCK_SIZE, "7-5"), + bmap_xml(BLOCK_SIZE, "x"), + "".to_string(), + ]; + + for xml in invalid { + fs::write(&path, &xml).expect("write bmap"); + assert!(Bmap::read(&path).is_err(), "accepted {xml}"); + } } #[test] - fn read_ext4_journal_rejects_invalid_block_size() { - let mut sb = [0u8; EXT4_SUPERBLOCK_SIZE]; - let mut set = |o: usize, v: &[u8]| sb[o..o + v.len()].copy_from_slice(v); - set(EXT4_S_MAGIC, &EXT4_MAGIC.to_le_bytes()); - set( - EXT4_S_FEATURE_COMPAT, - &EXT4_FEATURE_COMPAT_HAS_JOURNAL.to_le_bytes(), + fn find_input_bmap_takes_explicit_path() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let explicit = dir.path().join("other.bmap"); + + assert!(find_input_bmap(&image, Some(&explicit)).is_err()); + fs::write(&explicit, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, Some(&explicit)).expect("find bmap"), + Some(explicit) ); - set(EXT4_S_JOURNAL_INUM, &JOURNAL_INODE.to_le_bytes()); - set(EXT4_S_LOG_BLOCK_SIZE, &INVALID_LOG_BLOCK_SIZE.to_le_bytes()); + } - let image = tempfile::tempfile().expect("create image"); - image - .write_all_at(&sb, EXT4_SUPERBLOCK_OFFSET) - .expect("write superblock"); - assert!(read_ext4_journal(&image, 0).is_err()); + #[test] + fn find_input_bmap_looks_next_to_the_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic.xz"); + let uncompressed_bmap = dir.path().join("image.wic.bmap"); + let compressed_bmap = dir.path().join("image.wic.xz.bmap"); + + assert_eq!(find_input_bmap(&image, None).expect("find bmap"), None); + fs::write(&uncompressed_bmap, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, None).expect("find bmap"), + Some(uncompressed_bmap) + ); + fs::write(&compressed_bmap, "").expect("write bmap"); + assert_eq!( + find_input_bmap(&image, None).expect("find bmap"), + Some(compressed_bmap) + ); } #[test] - fn allocate_ext4_journals_skips_partition_past_image_end() { + fn allocate_ranges_maps_zero_ranges_of_the_input_bmap() { let dir = tempfile::tempdir().expect("create temp dir"); - let image = dir.path().join("image.wic"); - create_image_with_mbr(&image, u64::from(PART_START_LBA) * SECTOR_SIZE); + let image = sparse_image(dir.path()); + let input_bmap = dir.path().join("input.bmap"); + write_bmap( + &input_bmap, + IMAGE_BLOCKS * BLOCK_SIZE, + &[DATA_BLOCK..=DATA_BLOCK, ZERO_BLOCKS], + ); + let input = Bmap::read(&input_bmap).expect("read bmap"); + + let before = created_bmap(&image); + assert!(ZERO_BLOCKS.clone().all(|b| !is_mapped(&before, b))); - allocate_ext4_journals(image.to_str().expect("image path")).expect("allocate journals"); + allocate_ranges(&image, &input).expect("allocate ranges"); + let after = created_bmap(&image); + assert!(ZERO_BLOCKS.clone().all(|b| is_mapped(&after, b))); + assert!(!is_mapped(&after, UNMAPPED_BLOCK)); + + let device = dir.path().join("device"); + bmap_flash(&image, &after, &device); + let device = fs::read(&device).expect("read device"); + let zero_start = usize::try_from(ZERO_BLOCKS.start() * BLOCK_SIZE).expect("start"); + let zero_end = usize::try_from((ZERO_BLOCKS.end() + 1) * BLOCK_SIZE).expect("end"); + assert!(device[zero_start..zero_end].iter().all(|b| *b == 0)); + } + + #[test] + fn allocate_ranges_maps_whole_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + + allocate_ranges(&image, &Bmap::whole_image(IMAGE_BLOCKS * BLOCK_SIZE)) + .expect("allocate ranges"); + assert_eq!(created_bmap(&image).ranges, vec![0..=IMAGE_BLOCKS - 1]); + } + + #[test] + fn allocate_ranges_rejects_bmap_of_another_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + + let other_size = Bmap::whole_image((IMAGE_BLOCKS + 1) * BLOCK_SIZE); + assert!(allocate_ranges(&image, &other_size).is_err()); + + let outside = Bmap { + block_size: BLOCK_SIZE, + image_size: IMAGE_BLOCKS * BLOCK_SIZE, + ranges: vec![RangeInclusive::new(IMAGE_BLOCKS, IMAGE_BLOCKS)], + }; + assert!(allocate_ranges(&image, &outside).is_err()); } } diff --git a/src/file/functions.rs b/src/file/functions.rs index 5f8eb6d0..95855d29 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,4 +1,4 @@ -use crate::file::{IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE}; use anyhow::{Context, Result}; use log::{debug, warn}; use std::collections::HashMap; @@ -11,8 +11,6 @@ use std::str::FromStr; use stdext::function_name; use uuid::Uuid; -const COPY_CHUNK_SIZE: usize = 1024 * 1024; - #[derive(clap::ValueEnum, Debug, Clone, Eq, Hash, PartialEq)] #[clap(rename_all = "verbatim")] #[allow(non_camel_case_types)] diff --git a/src/file/mod.rs b/src/file/mod.rs index 073eb7dd..d5920523 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -5,6 +5,7 @@ mod partition; /// Granularity at which zero data in an image file is kept as a hole. const IMAGE_BLOCK_SIZE: usize = 4096; +const COPY_CHUNK_SIZE: usize = 1024 * 1024; use super::validators::{ device_update, identity::{IdentityConfig, IdentityType, validate_identity}, diff --git a/src/lib.rs b/src/lib.rs index c487ceda..fcb5e4e0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,8 +20,8 @@ use cli::{ IotHubDeviceUpdate::{self, SetDeviceConfig as IotHubDeviceUpdateSet}, SshConfig::{SetCertificate, SetConnection}, }; -use file::{compression::Compression, functions::FileCopyToParams}; -use log::error; +use file::{bmap::Bmap, compression::Compression, functions::FileCopyToParams}; +use log::{error, info}; use std::{fs, io::Read, path::Path, path::PathBuf}; use tokio::fs::remove_dir_all; use uuid::Uuid; @@ -50,6 +50,7 @@ impl Drop for TempDirGuard { fn run_image_command( image_file: PathBuf, + input_bmap: Option, generate_bmap: bool, target_compression: Option, command: F, @@ -70,6 +71,7 @@ where image_file.to_str().context("cannot get image file path")? ); + let input_bmap = file::bmap::find_input_bmap(&image_file, input_bmap.as_deref())?; let mut dest_image_file = image_file.clone(); // create /tmp/{uuid}/ and copy image into @@ -99,6 +101,22 @@ where ))?; } + let bmap = match input_bmap { + Some(path) => { + info!("keep the ranges of {} mapped", path.display()); + Bmap::read(&path)? + } + None => { + info!("no input bmap found: every block of the image gets mapped"); + Bmap::whole_image( + fs::metadata(&tmp_image_file) + .context("run_image_command: cannot get image size")? + .len(), + ) + } + }; + file::bmap::allocate_ranges(&tmp_image_file, &bmap)?; + // run command command(&tmp_image_file)?; @@ -225,8 +243,9 @@ pub fn run() -> Result<()> { partition, dest, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { anyhow::ensure!( dest.to_string_lossy().ends_with(".tar.gz"), format!( @@ -265,8 +284,9 @@ pub fn run() -> Result<()> { image, payload, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { file::set_identity_config(&config, img, payload.as_deref()) })?, Command::Identity(SetDeviceCertificate { @@ -276,6 +296,7 @@ pub fn run() -> Result<()> { device_id, days, generate_bmap, + bmap, compress_image, }) => { let cert_info = create_image_cert( @@ -291,7 +312,7 @@ pub fn run() -> Result<()> { ) .context("set_device_certificate: could not create certificate")?; - run_image_command(image, generate_bmap, compress_image, |img| { + run_image_command(image, bmap, generate_bmap, compress_image, |img| { file::set_device_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -307,6 +328,7 @@ pub fn run() -> Result<()> { device_id, days, generate_bmap, + bmap, compress_image, }) => { let cert_info = create_image_cert( @@ -322,7 +344,7 @@ pub fn run() -> Result<()> { ) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command(image, generate_bmap, compress_image, |img| { + run_image_command(image, bmap, generate_bmap, compress_image, |img| { file::set_edge_ca_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -336,8 +358,9 @@ pub fn run() -> Result<()> { device_key: device_key_pem, image, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img| { + }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) })?, Command::Identity(SetIotedgeGatewayConfig { @@ -347,41 +370,65 @@ pub fn run() -> Result<()> { device_identity, device_identity_key, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { - file::set_iotedge_gateway_config( - &config, - img, - &root_ca, - &device_identity, - &device_identity_key, - ) - })?, + }) => run_image_command( + image, + bmap, + generate_bmap, + compress_image, + |img: &PathBuf| { + file::set_iotedge_gateway_config( + &config, + img, + &root_ca, + &device_identity, + &device_identity_key, + ) + }, + )?, Command::Identity(SetIotLeafSasConfig { config, image, root_ca, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { - file::set_iot_leaf_sas_config(&config, img, &root_ca) - })?, + }) => run_image_command( + image, + bmap, + generate_bmap, + compress_image, + |img: &PathBuf| file::set_iot_leaf_sas_config(&config, img, &root_ca), + )?, Command::Ssh(SetCertificate { image, root_ca, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { - file::set_ssh_tunnel_certificate(img, &root_ca) - })?, + }) => run_image_command( + image, + bmap, + generate_bmap, + compress_image, + |img: &PathBuf| file::set_ssh_tunnel_certificate(img, &root_ca), + )?, Command::IotHubDeviceUpdate(IotHubDeviceUpdateSet { iot_hub_device_update_config, image, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { - file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) - })?, + }) => run_image_command( + image, + bmap, + generate_bmap, + compress_image, + |img: &PathBuf| { + file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) + }, + )?, Command::IotHubDeviceUpdate(IotHubDeviceUpdate::ImportUpdate { import_manifest: import_manifest_path, storage_container_name, @@ -495,14 +542,19 @@ pub fn run() -> Result<()> { file_copy_params, image, generate_bmap, + bmap, compress_image, - }) => run_image_command(image, generate_bmap, compress_image, |img: &PathBuf| { - file::copy_to_image(&file_copy_params, img) - })?, + }) => run_image_command( + image, + bmap, + generate_bmap, + compress_image, + |img: &PathBuf| file::copy_to_image(&file_copy_params, img), + )?, Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, false, None, |img: &PathBuf| { + }) => run_image_command(image, None, false, None, |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index c807429d..cdf20263 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -3,9 +3,13 @@ use assert_cmd::Command; use assert_json_diff::assert_json_eq; use common::Testrunner; use httpmock::prelude::*; -use omnect_cli::ssh; +use omnect_cli::{file::bmap::Bmap, ssh}; use predicates::prelude::*; -use std::{fs::create_dir_all, os::unix::fs::MetadataExt, path::PathBuf}; +use std::{ + fs::create_dir_all, + os::unix::fs::MetadataExt, + path::{Path, PathBuf}, +}; use stdext::function_name; #[test] @@ -898,20 +902,24 @@ fn check_file_copy(tr: Testrunner, partition: &str) { // unit of `st_blocks` const STAT_BLOCK_SIZE: u64 = 512; -fn bmap_value(bmap: &str, tag: &str) -> u64 { - // the header comment of a bmap file mentions the tags too; only the closing tag is unique - let open_tag = format!("<{tag}>"); - let end = bmap - .find(&format!("")) - .expect("closing tag not found in bmap"); - let start = bmap[..end] - .rfind(&open_tag) - .expect("opening tag not found in bmap") - + open_tag.len(); - bmap[start..end] - .trim() - .parse() - .expect("tag value is no number") +fn read_bmap(path: &Path) -> Bmap { + Bmap::read(path).expect("read bmap") +} + +fn mapped_blocks(bmap: &Bmap) -> u64 { + bmap.ranges.iter().map(|r| r.end() - r.start() + 1).sum() +} + +fn maps_whole_image(bmap: &Bmap) -> bool { + mapped_blocks(bmap) == bmap.image_size.div_ceil(bmap.block_size) +} + +fn maps_all_ranges_of(bmap: &Bmap, input: &Bmap) -> bool { + input + .ranges + .iter() + .flat_map(|r| r.clone()) + .all(|b| bmap.ranges.iter().any(|r| r.contains(&b))) } #[test] @@ -939,18 +947,14 @@ fn check_file_copy_keeps_image_layout() { .assert() .success(); - // zero blocks of the input must stay mapped, e.g. zeroed ext4 journals - let bmap = std::fs::read_to_string(&bmap_path).expect("read bmap"); - assert_eq!( - bmap_value(&bmap, "MappedBlocksCount"), - bmap_value(&bmap, "BlocksCount") - ); + assert!(maps_whole_image(&read_bmap(&bmap_path))); } #[test] fn check_file_copy_keeps_holes() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); let sparse_path = image_path.with_file_name("sparse.wic"); let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); let in_file = tr.to_pathbuf("testfiles/boot.scr"); @@ -971,14 +975,17 @@ fn check_file_copy_keeps_holes() { .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) .arg("-i") .arg(&sparse_path) + .arg("--bmap") + .arg(&input_bmap_path) .arg("-b") .assert() .success(); let meta = sparse_path.metadata().expect("image metadata"); assert!(meta.blocks() * STAT_BLOCK_SIZE < meta.len()); - let bmap = std::fs::read_to_string(&bmap_path).expect("read bmap"); - assert!(bmap_value(&bmap, "MappedBlocksCount") < bmap_value(&bmap, "BlocksCount")); + let bmap = read_bmap(&bmap_path); + assert!(!maps_whole_image(&bmap)); + assert!(maps_all_ranges_of(&bmap, &read_bmap(&input_bmap_path))); } #[test] @@ -1035,7 +1042,9 @@ fn check_bmap_generation_wic_xz() { let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); let image_path_wic = image_path_wic_xz.with_extension(""); let image_path_wic_copy = image_path_wic_xz.with_extension("copy"); - let image_path_bmap = image_path_wic_xz.with_extension("bmap"); + // the Yocto bmap; omnect-cli replaces it with the one it creates + let image_path_bmap = tr.to_pathbuf("testfiles/image.wic.bmap"); + let input_bmap = read_bmap(&image_path_bmap); let in_file = tr.to_pathbuf("testfiles/boot.scr"); let in_file = in_file.to_str().unwrap(); @@ -1051,8 +1060,9 @@ fn check_bmap_generation_wic_xz() { .assert(); assert.success(); - let bmap = std::fs::read_to_string(&image_path_bmap).expect("read bmap"); - assert!(bmap_value(&bmap, "MappedBlocksCount") < bmap_value(&bmap, "BlocksCount")); + let bmap = read_bmap(&image_path_bmap); + assert!(!maps_whole_image(&bmap)); + assert!(maps_all_ranges_of(&bmap, &input_bmap)); // use bmaptool to verify that the checksum of the bmap file and the image // still match after the copy operations @@ -1065,6 +1075,28 @@ fn check_bmap_generation_wic_xz() { assert.success(); } +#[test] +fn check_bmap_generation_wic_xz_without_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_path_bmap = image_path_wic_xz.with_extension("bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path_wic_xz) + .arg("-b") + .assert() + .success(); + + assert!(maps_whole_image(&read_bmap(&image_path_bmap))); +} + #[test] fn check_image_compression() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); From fb4441d4afab6f703455044fcc23c14d9c95ec49 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:25:55 +0100 Subject: [PATCH 15/46] fix(file): address review findings on input bmap handling - widen bmap ranges to whole 4 KiB blocks, so a hole is not missed when the bmap block size is not a multiple of 4096 - copy-from-image does not read or enforce an input bmap - read the input bmap before the image is copied, and name it in errors - finish the compress stream, so a failed trailer write is an error - revert the unused partition table split - test a flash onto old content and a bmap of another image - README: describe where the created bmap goes and when it maps the whole image Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 4 +- src/file/bmap.rs | 32 +++++++++++---- src/file/compression.rs | 31 +++++++------- src/file/functions.rs | 41 ++++++++++++++++--- src/file/mod.rs | 11 +++-- src/file/partition.rs | 64 ++++++++++------------------- src/lib.rs | 75 +++++++++++++++++++++------------- tests/integration_tests.rs | 83 +++++++++++++++++++++++++++++++++----- 8 files changed, 230 insertions(+), 111 deletions(-) diff --git a/README.md b/README.md index 18133e7c..53b90332 100644 --- a/README.md +++ b/README.md @@ -186,7 +186,9 @@ omnect-cli file copy-to-image --help ## bmap files -Commands that change an image create a bmap file with `-b`. The bmap of the input image is the base for it: `omnect-cli` takes `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it, or the file given with `--bmap`. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges plus the blocks `omnect-cli` changed. +Commands that change an image create a bmap file with `-b`. It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. + +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image, otherwise the command fails. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 605c394b..918660e9 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,4 +1,4 @@ -use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; use std::{ fs, @@ -137,14 +137,18 @@ pub fn allocate_ranges(image_file: &Path, bmap: &Bmap) -> Result<()> { ); let image_blocks = image_len.div_ceil(bmap.block_size); + let image_block_size = IMAGE_BLOCK_SIZE as u64; let mut buf = vec![0u8; COPY_CHUNK_SIZE]; for range in &bmap.ranges { anyhow::ensure!( *range.end() < image_blocks, "allocate_ranges: range {range:?} is outside the image" ); - let start = range.start() * bmap.block_size; - let end = ((range.end() + 1) * bmap.block_size).min(image_len); + // check whole image blocks, also when the bmap uses a smaller block size + let start = range.start() * bmap.block_size / image_block_size * image_block_size; + let end = ((range.end() + 1) * bmap.block_size) + .next_multiple_of(image_block_size) + .min(image_len); let mut pos = start; while pos < end { @@ -158,13 +162,10 @@ pub fn allocate_ranges(image_file: &Path, bmap: &Bmap) -> Result<()> { } } - image - .sync_all() - .context("allocate_ranges: cannot sync image") + Ok(()) } fn write_zero_blocks(image: &fs::File, chunk: &[u8], pos: u64) -> Result<()> { - let is_zero = |b: &[u8]| b.iter().all(|x| *x == 0); let blocks: Vec<(u64, &[u8])> = if is_zero(chunk) { vec![(pos, chunk)] } else { @@ -207,6 +208,7 @@ mod tests { const DATA_BLOCK: u64 = 0; const ZERO_BLOCKS: RangeInclusive = 2..=3; const UNMAPPED_BLOCK: u64 = 5; + const SMALL_BLOCK_SIZE: u64 = 512; const DATA_BYTE: u8 = 0x5a; // content an earlier installation left on the device const OLD_CONTENT_BYTE: u8 = 0xa5; @@ -376,6 +378,22 @@ mod tests { assert_eq!(created_bmap(&image).ranges, vec![0..=IMAGE_BLOCKS - 1]); } + #[test] + fn allocate_ranges_maps_whole_blocks_for_a_smaller_bmap_block_size() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + // starts in DATA_BLOCK and ends in the hole after it + let first = (DATA_BLOCK + 1) * BLOCK_SIZE / SMALL_BLOCK_SIZE - 1; + let input = Bmap { + block_size: SMALL_BLOCK_SIZE, + image_size: IMAGE_BLOCKS * BLOCK_SIZE, + ranges: vec![RangeInclusive::new(first, first + 2)], + }; + + allocate_ranges(&image, &input).expect("allocate ranges"); + assert!(is_mapped(&created_bmap(&image), DATA_BLOCK + 1)); + } + #[test] fn allocate_ranges_rejects_bmap_of_another_image() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/file/compression.rs b/src/file/compression.rs index ba5a43d0..2af9cefc 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -1,4 +1,4 @@ -use crate::file::IMAGE_BLOCK_SIZE; +use crate::file::{IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; use filemagic::Magic; use log::debug; @@ -49,15 +49,17 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let mut enc: Box = match &self { - Compression::bzip2 => Box::new(bzip2::write::BzEncoder::new( - destination, - bzip2::Compression::best(), - )), - Compression::gzip => Box::new(flate2::write::GzEncoder::new( - destination, - flate2::Compression::best(), - )), + // only `finish` reports an error while writing the trailer + let (bytes_read, _) = match &self { + Compression::bzip2 => { + let mut enc = bzip2::write::BzEncoder::new(destination, bzip2::Compression::best()); + (std::io::copy(source, &mut enc)?, enc.finish()?) + } + Compression::gzip => { + let mut enc = + flate2::write::GzEncoder::new(destination, flate2::Compression::best()); + (std::io::copy(source, &mut enc)?, enc.finish()?) + } Compression::xz { compression_level: level, } => { @@ -65,13 +67,11 @@ impl Compression { .threads(num_cpus::get() as u32) .preset(*level) .encoder()?; - Box::new(xz2::write::XzEncoder::new_stream(destination, stream)) + let mut enc = xz2::write::XzEncoder::new_stream(destination, stream); + (std::io::copy(source, &mut enc)?, enc.finish()?) } }; - - let bytes_written = std::io::copy(source, &mut enc)?; - enc.flush()?; - Ok(bytes_written) + Ok(bytes_read) } pub fn decompress( @@ -161,7 +161,6 @@ impl<'a> SparseWriter<'a> { impl Write for SparseWriter<'_> { fn write(&mut self, buf: &[u8]) -> std::io::Result { - let is_zero = |b: &[u8]| b.iter().all(|x| *x == 0); let block_offset = (self.pos % IMAGE_BLOCK_SIZE as u64) as usize; // one call handles a run of blocks that are all zero or all data diff --git a/src/file/functions.rs b/src/file/functions.rs index 95855d29..b85b969e 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -485,6 +485,7 @@ fn write_partition( #[cfg(test)] mod tests { use crate::file::functions::*; + use std::os::unix::fs::MetadataExt; const PART_NUM: u32 = 1; const PART_START: u64 = 8; @@ -492,6 +493,14 @@ mod tests { const OLD_CONTENT_BYTE: u8 = 0xa5; const ZEROED_BLOCK: usize = 2; + fn partition_info() -> PartitionInfo { + PartitionInfo { + num: PART_NUM, + start: PART_START, + count: PART_SECTORS, + } + } + #[test] fn write_partition_writes_block_changed_to_zeros() { let dir = tempfile::tempdir().expect("create temp dir"); @@ -508,15 +517,37 @@ mod tests { write_partition( image.to_str().expect("image path"), partition.to_str().expect("partition path"), - &PartitionInfo { - num: PART_NUM, - start: PART_START, - count: PART_SECTORS, - }, + &partition_info(), ) .expect("write partition"); let written = fs::read(&image).expect("read image"); assert!(written[part_offset..] == content[..], "partition differs"); } + + #[test] + fn write_partition_keeps_holes_of_unchanged_blocks() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + fs::File::create(&image) + .expect("create image") + .set_len((PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + fs::File::create(&partition) + .expect("create partition") + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let allocated = || fs::metadata(&image).expect("image metadata").blocks(); + let before = allocated(); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .expect("write partition"); + + assert_eq!(allocated(), before, "unchanged zero blocks were written"); + } } diff --git a/src/file/mod.rs b/src/file/mod.rs index d5920523..6d334287 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -3,9 +3,6 @@ pub mod compression; pub mod functions; mod partition; -/// Granularity at which zero data in an image file is kept as a hole. -const IMAGE_BLOCK_SIZE: usize = 4096; -const COPY_CHUNK_SIZE: usize = 1024 * 1024; use super::validators::{ device_update, identity::{IdentityConfig, IdentityType, validate_identity}, @@ -20,6 +17,14 @@ use std::{ path::{Path, PathBuf}, }; +/// Granularity at which zero data in an image file is kept as a hole. +const IMAGE_BLOCK_SIZE: usize = 4096; +const COPY_CHUNK_SIZE: usize = 1024 * 1024; + +fn is_zero(data: &[u8]) -> bool { + data.iter().all(|b| *b == 0) +} + pub fn set_iotedge_gateway_config( config_file: &Path, image_file: &Path, diff --git a/src/file/partition.rs b/src/file/partition.rs index 36fe983d..7a617fcd 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -13,13 +13,6 @@ pub struct PartitionData { } pub fn get_partition_data>(path: P, partition_num: u32) -> Result { - get_partitions(path)? - .into_iter() - .find(|p| p.num == partition_num) - .with_context(|| format!("partition {partition_num} not found in image")) -} - -pub fn get_partitions>(path: P) -> Result> { let path = path.as_ref(); let mut file = File::open(path).with_context(|| format!("failed to open image: {}", path.display()))?; @@ -28,15 +21,21 @@ pub fn get_partitions>(path: P) -> Result> { // Capture any error so we can attach it as context if MBR also fails. let gpt_err = match gptman::GPT::find_from(&mut file) { Ok(gpt) => { - return Ok(gpt + let entry = gpt .iter() - .filter(|(_, e)| e.is_used() && e.ending_lba >= e.starting_lba) - .map(|(num, e)| PartitionData { - num, - start: e.starting_lba, - count: e.ending_lba - e.starting_lba + 1, - }) - .collect()); + .find(|(i, _)| *i == partition_num) + .map(|(_, e)| e) + .with_context(|| format!("GPT partition {partition_num} out of range"))?; + anyhow::ensure!(entry.is_used(), "GPT partition {partition_num} is not used"); + anyhow::ensure!( + entry.ending_lba >= entry.starting_lba, + "GPT partition {partition_num} has invalid LBA range (ending_lba < starting_lba)" + ); + return Ok(PartitionData { + num: partition_num, + start: entry.starting_lba, + count: entry.ending_lba - entry.starting_lba + 1, + }); } Err(e) => e, }; @@ -46,38 +45,19 @@ pub fn get_partitions>(path: P) -> Result> { .context("failed to seek to start of image")?; let mbr = mbrman::MBR::read_from(&mut file, SECTOR_SIZE as u32) .with_context(|| format!("image is neither valid GPT nor MBR (GPT error: {gpt_err})"))?; - mbr.iter() - .filter(|(_, p)| p.is_used() && !p.is_extended()) - .map(|(num, p)| { - Ok(PartitionData { - num: u32::try_from(num).context("MBR partition number out of range")?, + for (i, p) in mbr.iter() { + if u32::try_from(i).ok() == Some(partition_num) && p.is_used() { + return Ok(PartitionData { + num: partition_num, start: p.starting_lba as u64, count: p.sectors as u64, - }) - }) - .collect() + }); + } + } + anyhow::bail!("partition {partition_num} not found in image") } pub fn is_gpt>(path: P) -> Result { let mut file = File::open(path.as_ref()).context("is_gpt: failed to open image")?; Ok(gptman::GPT::find_from(&mut file).is_ok()) } - -#[cfg(test)] -mod tests { - use crate::file::partition::*; - - const TEST_IMAGE: &str = "testfiles/image.wic"; - const MBR_EXTENDED_PARTITION: u32 = 4; - - #[test] - fn get_partitions_skips_mbr_extended_entry() { - let nums: Vec = get_partitions(TEST_IMAGE) - .expect("read partitions") - .iter() - .map(|p| p.num) - .collect(); - assert!(!nums.contains(&MBR_EXTENDED_PARTITION)); - assert!(nums.contains(&(MBR_EXTENDED_PARTITION + 1))); - } -} diff --git a/src/lib.rs b/src/lib.rs index fcb5e4e0..b1861e15 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,7 +22,11 @@ use cli::{ }; use file::{bmap::Bmap, compression::Compression, functions::FileCopyToParams}; use log::{error, info}; -use std::{fs, io::Read, path::Path, path::PathBuf}; +use std::{ + fs, + io::Read, + path::{Path, PathBuf}, +}; use tokio::fs::remove_dir_all; use uuid::Uuid; @@ -48,9 +52,17 @@ impl Drop for TempDirGuard { } } +/// What an image command does with the holes and mapped blocks of the image. +enum Layout { + /// The command does not change the image. + Unchanged, + /// Keep the ranges of this bmap, or of the one found next to the image, mapped. + Keep(Option), +} + fn run_image_command( image_file: PathBuf, - input_bmap: Option, + layout: Layout, generate_bmap: bool, target_compression: Option, command: F, @@ -71,7 +83,14 @@ where image_file.to_str().context("cannot get image file path")? ); - let input_bmap = file::bmap::find_input_bmap(&image_file, input_bmap.as_deref())?; + let input_bmap = match layout { + Layout::Unchanged => None, + Layout::Keep(explicit) => Some( + file::bmap::find_input_bmap(&image_file, explicit.as_deref())? + .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) + .transpose()?, + ), + }; let mut dest_image_file = image_file.clone(); // create /tmp/{uuid}/ and copy image into @@ -101,21 +120,21 @@ where ))?; } - let bmap = match input_bmap { - Some(path) => { - info!("keep the ranges of {} mapped", path.display()); - Bmap::read(&path)? + match input_bmap { + None => {} + Some(Some((path, bmap))) => { + info!("keeping the ranges of {} mapped", path.display()); + file::bmap::allocate_ranges(&tmp_image_file, &bmap) + .context(format!("cannot use input bmap {}", path.display()))?; } - None => { - info!("no input bmap found: every block of the image gets mapped"); - Bmap::whole_image( - fs::metadata(&tmp_image_file) - .context("run_image_command: cannot get image size")? - .len(), - ) + Some(None) => { + info!("no input bmap found, so every block of the image is allocated"); + let image_size = fs::metadata(&tmp_image_file) + .context("run_image_command: cannot get image size")? + .len(); + file::bmap::allocate_ranges(&tmp_image_file, &Bmap::whole_image(image_size))?; } - }; - file::bmap::allocate_ranges(&tmp_image_file, &bmap)?; + } // run command command(&tmp_image_file)?; @@ -236,6 +255,8 @@ fn create_image_cert(image: &Path, cert_opts: CertificateOptions) -> Result Result<()> { + use Layout::{Keep, Unchanged}; + match cli::from_args() { Command::Docker(Inject { docker_image, @@ -245,7 +266,7 @@ pub fn run() -> Result<()> { generate_bmap, bmap, compress_image, - }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { + }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { anyhow::ensure!( dest.to_string_lossy().ends_with(".tar.gz"), format!( @@ -286,7 +307,7 @@ pub fn run() -> Result<()> { generate_bmap, bmap, compress_image, - }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { + }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { file::set_identity_config(&config, img, payload.as_deref()) })?, Command::Identity(SetDeviceCertificate { @@ -312,7 +333,7 @@ pub fn run() -> Result<()> { ) .context("set_device_certificate: could not create certificate")?; - run_image_command(image, bmap, generate_bmap, compress_image, |img| { + run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { file::set_device_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -344,7 +365,7 @@ pub fn run() -> Result<()> { ) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command(image, bmap, generate_bmap, compress_image, |img| { + run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { file::set_edge_ca_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -360,7 +381,7 @@ pub fn run() -> Result<()> { generate_bmap, bmap, compress_image, - }) => run_image_command(image, bmap, generate_bmap, compress_image, |img| { + }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) })?, Command::Identity(SetIotedgeGatewayConfig { @@ -374,7 +395,7 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - bmap, + Keep(bmap), generate_bmap, compress_image, |img: &PathBuf| { @@ -396,7 +417,7 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - bmap, + Keep(bmap), generate_bmap, compress_image, |img: &PathBuf| file::set_iot_leaf_sas_config(&config, img, &root_ca), @@ -409,7 +430,7 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - bmap, + Keep(bmap), generate_bmap, compress_image, |img: &PathBuf| file::set_ssh_tunnel_certificate(img, &root_ca), @@ -422,7 +443,7 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - bmap, + Keep(bmap), generate_bmap, compress_image, |img: &PathBuf| { @@ -546,7 +567,7 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - bmap, + Keep(bmap), generate_bmap, compress_image, |img: &PathBuf| file::copy_to_image(&file_copy_params, img), @@ -554,7 +575,7 @@ pub fn run() -> Result<()> { Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, None, false, None, |img: &PathBuf| { + }) => run_image_command(image, Unchanged, false, None, |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index cdf20263..d0ffca0c 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -6,7 +6,7 @@ use httpmock::prelude::*; use omnect_cli::{file::bmap::Bmap, ssh}; use predicates::prelude::*; use std::{ - fs::create_dir_all, + fs::{self, create_dir_all}, os::unix::fs::MetadataExt, path::{Path, PathBuf}, }; @@ -901,6 +901,10 @@ fn check_file_copy(tr: Testrunner, partition: &str) { // unit of `st_blocks` const STAT_BLOCK_SIZE: u64 = 512; +// content an earlier installation left on the device +const OLD_CONTENT_BYTE: u8 = 0xa5; +const OTHER_IMAGE_BMAP: &str = "40964096\ + 0"; fn read_bmap(path: &Path) -> Bmap { Bmap::read(path).expect("read bmap") @@ -922,19 +926,26 @@ fn maps_all_ranges_of(bmap: &Bmap, input: &Bmap) -> bool { .all(|b| bmap.ranges.iter().any(|r| r.contains(&b))) } +/// Writes the blocks `bmap` maps onto a device that still holds old content, as a bmap flash +/// does. +fn bmap_flash(image: &[u8], bmap: &Bmap) -> Vec { + let mut device = vec![OLD_CONTENT_BYTE; image.len()]; + for range in &bmap.ranges { + let start = usize::try_from(range.start() * bmap.block_size).expect("start"); + let end = usize::try_from((range.end() + 1) * bmap.block_size).expect("end"); + let end = end.min(image.len()); + device[start..end].copy_from_slice(&image[start..end]); + } + device +} + #[test] -fn check_file_copy_keeps_image_layout() { +fn check_file_copy_without_input_bmap_maps_whole_image() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_path = tr.to_pathbuf("testfiles/image.wic"); let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); let in_file = tr.to_pathbuf("testfiles/boot.scr"); - let meta = image_path.metadata().expect("image metadata"); - assert!( - meta.blocks() * STAT_BLOCK_SIZE >= meta.len(), - "test image must be fully allocated" - ); - Command::cargo_bin("omnect-cli") .expect("omnect-cli binary") .arg("file") @@ -984,8 +995,59 @@ fn check_file_copy_keeps_holes() { let meta = sparse_path.metadata().expect("image metadata"); assert!(meta.blocks() * STAT_BLOCK_SIZE < meta.len()); let bmap = read_bmap(&bmap_path); + let input_bmap = read_bmap(&input_bmap_path); assert!(!maps_whole_image(&bmap)); - assert!(maps_all_ranges_of(&bmap, &read_bmap(&input_bmap_path))); + assert!(maps_all_ranges_of(&bmap, &input_bmap)); + + // every data block and every block of the input bmap must replace the old content + let image = fs::read(&sparse_path).expect("read image"); + let device = bmap_flash(&image, &bmap); + let block_size = usize::try_from(input_bmap.block_size).expect("block size"); + let blocks = image.chunks(block_size).zip(device.chunks(block_size)); + for (block, (image_block, device_block)) in (0u64..).zip(blocks) { + let needed = image_block.iter().any(|b| *b != 0) + || input_bmap.ranges.iter().any(|r| r.contains(&block)); + assert!( + !needed || image_block == device_block, + "block {block} keeps old content" + ); + } +} + +#[test] +fn check_input_bmap_of_another_image() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let out_file = tr.pathbuf().join("hosts"); + fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + let image_hash = Testrunner::file_hash(&image_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .assert() + .failure() + .stderr(predicate::str::contains("cannot use input bmap")); + assert_eq!(Testrunner::file_hash(&image_path), image_hash); + + // reading from the image does not use the bmap + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!("rootA:/etc/hosts,{}", out_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .assert() + .success(); } #[test] @@ -1042,7 +1104,8 @@ fn check_bmap_generation_wic_xz() { let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); let image_path_wic = image_path_wic_xz.with_extension(""); let image_path_wic_copy = image_path_wic_xz.with_extension("copy"); - // the Yocto bmap; omnect-cli replaces it with the one it creates + // a Yocto bmap of an image with the same layout; omnect-cli replaces it with the one it + // creates let image_path_bmap = tr.to_pathbuf("testfiles/image.wic.bmap"); let input_bmap = read_bmap(&image_path_bmap); let in_file = tr.to_pathbuf("testfiles/boot.scr"); From cb3c4765a2a4437166ae039dbb956a1ee68c37a8 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 09:34:49 +0100 Subject: [PATCH 16/46] feat(cli)!: rename bmap options to --input-bmap and --output-bmap The short option -b stays. BREAKING CHANGE: --generate-bmap-file is now --output-bmap. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 6 +- src/cli.rs | 100 +++++++++---------- src/lib.rs | 200 +++++++++++++++++++++---------------- tests/integration_tests.rs | 2 +- 6 files changed, 169 insertions(+), 143 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b824097f..6bf25d20 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1763,7 +1763,7 @@ dependencies = [ [[package]] name = "omnect-cli" -version = "0.28.6" +version = "0.29.0" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 35d5c6eb..3443bc09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0" name = "omnect-cli" readme = "README.md" repository = "https://github.com/omnect/omnect-cli" -version = "0.28.6" +version = "0.29.0" [dependencies] actix-web = { version = "4.17", default-features = false, features = ["macros"] } diff --git a/README.md b/README.md index 53b90332..8062664f 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ docker run --rm -it \ omnect/omnect-cli:latest file copy-to-image --files /source/my-source-file,boot:/my-dest-file -i /source/my-image.wic ``` - **Note1**: `-b` option to create bmap file is not supported by omnect-cli docker image.
+ **Note1**: `-b` (`--output-bmap`) option to create a bmap file is not supported by omnect-cli docker image.
**Note2**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.
**Note3**: The docker inject command is not supported by omnect-cli docker image.
. @@ -186,9 +186,9 @@ omnect-cli file copy-to-image --help ## bmap files -Commands that change an image create a bmap file with `-b`. It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. +Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image, otherwise the command fails. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image, otherwise the command fails. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/cli.rs b/src/cli.rs index 6b104ac4..8d7278a7 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -34,12 +34,12 @@ pub enum Docker { /// destination path of the docker image in the firmware image (must end in ".tar.gz") #[clap(short = 'e', long = "dest")] dest: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -58,12 +58,12 @@ pub enum File { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -94,12 +94,12 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -121,12 +121,12 @@ pub enum IdentityConfig { /// path to device identity certificate key file #[arg(short = 'k', long = "device_identity_key")] device_identity_key: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -142,12 +142,12 @@ pub enum IdentityConfig { /// path to root ca certificate file #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -169,12 +169,12 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -190,12 +190,12 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -217,12 +217,12 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -241,12 +241,12 @@ pub enum IotHubDeviceUpdate { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, @@ -363,12 +363,12 @@ pub enum SshConfig { /// path to public key of the ssh root ca #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: generate bmap file (currently not working in docker image) - #[arg(short = 'b', long = "generate-bmap-file")] - generate_bmap: bool, + /// optional: create a bmap of the output image (currently not working in docker image) + #[arg(short = 'b', long = "output-bmap")] + output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "bmap")] - bmap: Option, + #[arg(long = "input-bmap")] + input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] compress_image: Option, diff --git a/src/lib.rs b/src/lib.rs index b1861e15..db5c98a0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -63,7 +63,7 @@ enum Layout { fn run_image_command( image_file: PathBuf, layout: Layout, - generate_bmap: bool, + output_bmap: bool, target_compression: Option, command: F, ) -> Result<()> @@ -72,7 +72,7 @@ where { if let Ok("true") | Ok("1") = std::env::var("CONTAINERIZED").as_deref() { anyhow::ensure!( - !generate_bmap, + !output_bmap, "run_image_command: generating bmap file is not supported in containerized environments." ); } @@ -140,7 +140,7 @@ where command(&tmp_image_file)?; // create and copy back bmap file if one was created - if generate_bmap { + if output_bmap { let mut target_bmap = image_file .parent() .context("cannot get parent dir of image path")? @@ -263,61 +263,71 @@ pub fn run() -> Result<()> { image, partition, dest, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, - }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { - anyhow::ensure!( - dest.to_string_lossy().ends_with(".tar.gz"), - format!( - "invalid destination file path \"{}\". Must end in \".tar.gz\".", - dest.to_string_lossy(), - ), - ); - - let arch = image::image_arch(img)?; - - let docker_path = docker::pull_image(&docker_image, arch)?; - - let result = file::copy_to_image( - &[FileCopyToParams::new( - &docker_path, - partition.clone(), - &dest, - )], - img, - ); - std::fs::remove_file(docker_path)?; - - if result.is_ok() { - println!( - "Stored {} to {}:{}", - docker_image, - partition, - dest.to_string_lossy(), + }) => run_image_command( + image, + Keep(input_bmap), + output_bmap, + compress_image, + |img| { + anyhow::ensure!( + dest.to_string_lossy().ends_with(".tar.gz"), + format!( + "invalid destination file path \"{}\". Must end in \".tar.gz\".", + dest.to_string_lossy(), + ), ); - } - result - })?, + let arch = image::image_arch(img)?; + + let docker_path = docker::pull_image(&docker_image, arch)?; + + let result = file::copy_to_image( + &[FileCopyToParams::new( + &docker_path, + partition.clone(), + &dest, + )], + img, + ); + std::fs::remove_file(docker_path)?; + + if result.is_ok() { + println!( + "Stored {} to {}:{}", + docker_image, + partition, + dest.to_string_lossy(), + ); + } + + result + }, + )?, Command::Identity(SetConfig { config, image, payload, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, - }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { - file::set_identity_config(&config, img, payload.as_deref()) - })?, + }) => run_image_command( + image, + Keep(input_bmap), + output_bmap, + compress_image, + |img| file::set_identity_config(&config, img, payload.as_deref()), + )?, Command::Identity(SetDeviceCertificate { intermediate_full_chain_cert, intermediate_key, image, device_id, days, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => { let cert_info = create_image_cert( @@ -333,14 +343,20 @@ pub fn run() -> Result<()> { ) .context("set_device_certificate: could not create certificate")?; - run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { - file::set_device_cert( - Some(&intermediate_full_chain_cert), - &cert_info.cert_path, - &cert_info.key_path, - img, - ) - })? + run_image_command( + image, + Keep(input_bmap), + output_bmap, + compress_image, + |img| { + file::set_device_cert( + Some(&intermediate_full_chain_cert), + &cert_info.cert_path, + &cert_info.key_path, + img, + ) + }, + )? } Command::Identity(SetEdgeCaCertificate { intermediate_full_chain_cert, @@ -348,8 +364,8 @@ pub fn run() -> Result<()> { image, device_id, days, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => { let cert_info = create_image_cert( @@ -365,38 +381,48 @@ pub fn run() -> Result<()> { ) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { - file::set_edge_ca_cert( - Some(&intermediate_full_chain_cert), - &cert_info.cert_path, - &cert_info.key_path, - img, - ) - })? + run_image_command( + image, + Keep(input_bmap), + output_bmap, + compress_image, + |img| { + file::set_edge_ca_cert( + Some(&intermediate_full_chain_cert), + &cert_info.cert_path, + &cert_info.key_path, + img, + ) + }, + )? } Command::Identity(SetDeviceCertificateNoEst { device_cert: device_cert_pem, device_key: device_key_pem, image, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, - }) => run_image_command(image, Keep(bmap), generate_bmap, compress_image, |img| { - file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) - })?, + }) => run_image_command( + image, + Keep(input_bmap), + output_bmap, + compress_image, + |img| file::set_device_cert(None, &device_cert_pem, &device_key_pem, img), + )?, Command::Identity(SetIotedgeGatewayConfig { config, image, root_ca, device_identity, device_identity_key, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => run_image_command( image, - Keep(bmap), - generate_bmap, + Keep(input_bmap), + output_bmap, compress_image, |img: &PathBuf| { file::set_iotedge_gateway_config( @@ -412,39 +438,39 @@ pub fn run() -> Result<()> { config, image, root_ca, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => run_image_command( image, - Keep(bmap), - generate_bmap, + Keep(input_bmap), + output_bmap, compress_image, |img: &PathBuf| file::set_iot_leaf_sas_config(&config, img, &root_ca), )?, Command::Ssh(SetCertificate { image, root_ca, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => run_image_command( image, - Keep(bmap), - generate_bmap, + Keep(input_bmap), + output_bmap, compress_image, |img: &PathBuf| file::set_ssh_tunnel_certificate(img, &root_ca), )?, Command::IotHubDeviceUpdate(IotHubDeviceUpdateSet { iot_hub_device_update_config, image, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => run_image_command( image, - Keep(bmap), - generate_bmap, + Keep(input_bmap), + output_bmap, compress_image, |img: &PathBuf| { file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) @@ -562,13 +588,13 @@ pub fn run() -> Result<()> { Command::File(CopyToImage { file_copy_params, image, - generate_bmap, - bmap, + output_bmap, + input_bmap, compress_image, }) => run_image_command( image, - Keep(bmap), - generate_bmap, + Keep(input_bmap), + output_bmap, compress_image, |img: &PathBuf| file::copy_to_image(&file_copy_params, img), )?, diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index d0ffca0c..fb7f165e 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -986,7 +986,7 @@ fn check_file_copy_keeps_holes() { .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) .arg("-i") .arg(&sparse_path) - .arg("--bmap") + .arg("--input-bmap") .arg(&input_bmap_path) .arg("-b") .assert() From 46d03622f658b7e8672718fe052272fd85b377c5 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:32:22 +0100 Subject: [PATCH 17/46] fix(file): map zero blocks that a copy writes into a hole read_partition turned every zero block into a hole, and write_partition skipped blocks that are zero in both files. A zero block of a copied file that landed in free space stayed a hole in the image, so a bmap created from it did not map the block, and a bmap flash kept the old content there. read_partition now keeps the holes of the image, and write_partition also writes a block that is data in the partition file but a hole in the image. The sparse copy of copy_image moves to a shared helper that takes offsets. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.lock | 17 +---- Cargo.toml | 2 +- src/file/functions.rs | 96 +++++++++++++++++++----- src/file/mod.rs | 1 + src/file/sparse.rs | 150 +++++++++++++++++++++++++++++++++++++ src/lib.rs | 18 +---- tests/integration_tests.rs | 80 ++++++++++++++++++-- 7 files changed, 305 insertions(+), 59 deletions(-) create mode 100644 src/file/sparse.rs diff --git a/Cargo.lock b/Cargo.lock index 6bf25d20..38a63979 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1528,21 +1528,6 @@ version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" -[[package]] -name = "libfs" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b79c3145dcaad30b6d7b1e30c9d2d23799b882e9a2e9d1fc7bcc0699c5279ff9" -dependencies = [ - "cfg-if", - "libc", - "linux-raw-sys", - "log", - "rustix", - "thiserror 2.0.21", - "xattr", -] - [[package]] name = "libredox" version = "0.1.25" @@ -1782,7 +1767,6 @@ dependencies = [ "hmac", "httpmock", "keyring-core", - "libfs", "log", "mbrman", "num_cpus", @@ -1794,6 +1778,7 @@ dependencies = [ "reqwest 0.13.5", "ring", "roxmltree", + "rustix", "serde", "serde_json", "serde_path_to_error", diff --git a/Cargo.toml b/Cargo.toml index 3443bc09..e9081bae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,6 @@ hmac = { version = "0.13", default-features = false } mbrman = { version = "0.6", default-features = false } omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" } keyring-core = { version = "1.0", default-features = false } -libfs = { version = "0.9", default-features = false, features = ["use_linux"] } log = { version = "0.4", default-features = false } num_cpus = { version = "1.17", default-features = false } oauth2 = { version = "5.0", default-features = false, features = [ @@ -45,6 +44,7 @@ reqwest = { version = "0.13", default-features = false, features = [ "native-tls", ] } roxmltree = { version = "0.21", default-features = false, features = ["std"] } +rustix = { version = "1.1", default-features = false, features = ["fs", "std"] } serde = { version = "1.0", default-features = false, features = ["derive"] } serde_json = { version = "1.0", default-features = false } serde_path_to_error = { version = "0.1", default-features = false } diff --git a/src/file/functions.rs b/src/file/functions.rs index b85b969e..22c3a15e 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,9 +1,10 @@ -use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE, sparse}; use anyhow::{Context, Result}; use log::{debug, warn}; use std::collections::HashMap; use std::fmt::{self, Display}; use std::fs; +use std::ops::Range; use std::os::unix::fs::FileExt; use std::path::{Path, PathBuf}; use std::process::Command; @@ -414,20 +415,22 @@ fn read_partition( return Ok(()); } - let mut dd = Command::new("dd"); - dd.arg(format!("if={image_file}")) - .arg(format!("of={partition_file}")) - .arg(format!("bs={SECTOR_SIZE}")) - .arg(format!("skip={}", partition_info.start)) - .arg(format!("count={}", partition_info.count)) - .arg("conv=sparse") - .arg("status=none"); - exec_cmd!(dd); - - let mut sync = Command::new("sync"); - exec_cmd!(sync); - - Ok(()) + let image = + fs::File::open(image_file).context(format!("read_partition: cannot open {image_file}"))?; + let partition = fs::File::create(partition_file) + .context(format!("read_partition: cannot create {partition_file}"))?; + let len = partition_info.count * SECTOR_SIZE; + partition + .set_len(len) + .context("read_partition: cannot set partition file size")?; + + sparse::copy_range( + &image, + partition_info.start * SECTOR_SIZE, + &partition, + 0, + len, + ) } fn write_partition( @@ -451,7 +454,11 @@ fn write_partition( .min(partition_info.count * SECTOR_SIZE); // Only write blocks that differ: unchanged blocks keep their allocation, so a bmap - // created from the result still maps every block the source image mapped. + // created from the result still maps every block the source image mapped. A zero block + // that a tool wrote into a hole is data in the partition file and must become data in the + // image too, else a bmap flash keeps the old content there. + let partition_data = sparse::data_ranges(&partition, 0, len)?; + let image_data = sparse::data_ranges(&image, image_offset, image_offset + len)?; let mut src = vec![0u8; COPY_CHUNK_SIZE]; let mut dst = vec![0u8; COPY_CHUNK_SIZE]; let mut pos = 0; @@ -468,9 +475,13 @@ fn write_partition( .chunks(IMAGE_BLOCK_SIZE) .zip(dst[..n].chunks(IMAGE_BLOCK_SIZE)); for (i, (s, d)) in blocks.enumerate() { - if s != d { + let block_pos = pos + (i * IMAGE_BLOCK_SIZE) as u64; + let block = block_pos..block_pos + s.len() as u64; + let image_block = image_offset + block.start..image_offset + block.end; + if s != d || (has_data(&partition_data, &block) && has_hole(&image_data, &image_block)) + { image - .write_all_at(s, image_offset + pos + (i * IMAGE_BLOCK_SIZE) as u64) + .write_all_at(s, image_block.start) .context("write_partition: cannot write image")?; } } @@ -482,6 +493,18 @@ fn write_partition( .context("write_partition: cannot sync image") } +fn has_data(data_ranges: &[Range], block: &Range) -> bool { + let i = data_ranges.partition_point(|r| r.end <= block.start); + data_ranges.get(i).is_some_and(|r| r.start < block.end) +} + +fn has_hole(data_ranges: &[Range], block: &Range) -> bool { + let i = data_ranges.partition_point(|r| r.end <= block.start); + !data_ranges + .get(i) + .is_some_and(|r| r.start <= block.start && block.end <= r.end) +} + #[cfg(test)] mod tests { use crate::file::functions::*; @@ -550,4 +573,41 @@ mod tests { assert_eq!(allocated(), before, "unchanged zero blocks were written"); } + + #[test] + fn write_partition_maps_zero_block_written_into_a_hole() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let image_file = fs::File::create(&image).expect("create image"); + image_file + .set_len((PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let partition_file = fs::File::create(&partition).expect("create partition"); + partition_file + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let block = (ZEROED_BLOCK * IMAGE_BLOCK_SIZE) as u64; + partition_file + .write_all_at(&[0; IMAGE_BLOCK_SIZE], block) + .expect("write zero block"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .expect("write partition"); + + let block = PART_START * SECTOR_SIZE + block; + assert_eq!( + sparse::data_ranges( + &image_file, + 0, + image_file.metadata().expect("metadata").len() + ) + .expect("data ranges"), + vec![block..block + IMAGE_BLOCK_SIZE as u64] + ); + } } diff --git a/src/file/mod.rs b/src/file/mod.rs index 6d334287..f8999cef 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -2,6 +2,7 @@ pub mod bmap; pub mod compression; pub mod functions; mod partition; +pub(crate) mod sparse; use super::validators::{ device_update, diff --git a/src/file/sparse.rs b/src/file/sparse.rs new file mode 100644 index 00000000..1c9d62bd --- /dev/null +++ b/src/file/sparse.rs @@ -0,0 +1,150 @@ +use anyhow::{Context, Result}; +use rustix::{ + fs::{SeekFrom, seek}, + io::Errno, +}; +use std::{ + fs::File, + io::{self, Read, Seek}, + ops::Range, +}; + +/// Returns the byte ranges of `file` between `start` and `end` that hold data. +pub fn data_ranges(file: &File, start: u64, end: u64) -> Result>> { + let mut ranges = vec![]; + let mut pos = start; + while pos < end { + let data = match seek(file, SeekFrom::Data(pos)) { + Ok(data) => data, + // no data after pos + Err(Errno::NXIO) => break, + Err(e) => return Err(e).context("data_ranges: cannot find data"), + }; + if data >= end { + break; + } + let hole = seek(file, SeekFrom::Hole(data)).context("data_ranges: cannot find hole")?; + ranges.push(data..hole.min(end)); + pos = hole; + } + + Ok(ranges) +} + +/// Copies `len` bytes of `src` at `src_offset` to `dst` at `dst_offset`. Holes of `src` are +/// skipped, so they stay holes in a new `dst`. +pub fn copy_range( + src: &File, + src_offset: u64, + dst: &File, + dst_offset: u64, + len: u64, +) -> Result<()> { + let src_len = src + .metadata() + .context("copy_range: cannot get source size")? + .len(); + anyhow::ensure!( + src_offset + len <= src_len, + "copy_range: source has {src_len} bytes, cannot copy {len} bytes at {src_offset}" + ); + + for data in data_ranges(src, src_offset, src_offset + len)? { + let mut src = src; + let mut dst = dst; + src.seek(io::SeekFrom::Start(data.start)) + .context("copy_range: cannot seek source")?; + dst.seek(io::SeekFrom::Start(data.start - src_offset + dst_offset)) + .context("copy_range: cannot seek destination")?; + // one copy_file_range call copies at most 2 GiB; io::copy repeats it until a range is done + let copied = io::copy(&mut src.take(data.end - data.start), &mut dst) + .context("copy_range: cannot copy data")?; + anyhow::ensure!( + copied == data.end - data.start, + "copy_range: source ended early" + ); + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::file::sparse::*; + use std::os::unix::fs::FileExt; + + const BLOCK_SIZE: u64 = 4096; + const FILE_BLOCKS: u64 = 8; + const DATA_BLOCK: u64 = 2; + const DATA_BYTE: u8 = 0x5a; + const DST_OFFSET_BLOCKS: u64 = 1; + + /// A sparse file with data in `DATA_BLOCK` and holes everywhere else. + fn sparse_file(dir: &std::path::Path) -> File { + let file = File::options() + .read(true) + .write(true) + .create_new(true) + .open(dir.join("src")) + .expect("create file"); + file.set_len(FILE_BLOCKS * BLOCK_SIZE).expect("set size"); + file.write_all_at(&[DATA_BYTE; BLOCK_SIZE as usize], DATA_BLOCK * BLOCK_SIZE) + .expect("write data block"); + file + } + + #[test] + fn data_ranges_finds_data_between_holes() { + let dir = tempfile::tempdir().expect("create temp dir"); + let file = sparse_file(dir.path()); + let data = DATA_BLOCK * BLOCK_SIZE..(DATA_BLOCK + 1) * BLOCK_SIZE; + + assert_eq!( + data_ranges(&file, 0, FILE_BLOCKS * BLOCK_SIZE).expect("data ranges"), + vec![data.clone()] + ); + assert_eq!( + data_ranges(&file, 0, data.start).expect("data ranges"), + vec![] + ); + assert_eq!( + data_ranges(&file, data.start + 1, data.end - 1).expect("data ranges"), + vec![data.start + 1..data.end - 1] + ); + } + + #[test] + fn copy_range_keeps_holes() { + let dir = tempfile::tempdir().expect("create temp dir"); + let src = sparse_file(dir.path()); + let dst = File::options() + .read(true) + .write(true) + .create_new(true) + .open(dir.path().join("dst")) + .expect("create file"); + let len = (FILE_BLOCKS - DST_OFFSET_BLOCKS) * BLOCK_SIZE; + dst.set_len(len).expect("set size"); + + copy_range(&src, DST_OFFSET_BLOCKS * BLOCK_SIZE, &dst, 0, len).expect("copy range"); + + let data_start = (DATA_BLOCK - DST_OFFSET_BLOCKS) * BLOCK_SIZE; + assert_eq!( + data_ranges(&dst, 0, len).expect("data ranges"), + vec![data_start..data_start + BLOCK_SIZE] + ); + let mut block = [0u8; BLOCK_SIZE as usize]; + dst.read_exact_at(&mut block, data_start) + .expect("read block"); + assert!(block.iter().all(|b| *b == DATA_BYTE)); + } + + #[test] + fn copy_range_rejects_range_after_source_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let src = sparse_file(dir.path()); + let dst = File::create(dir.path().join("dst")).expect("create file"); + + assert!(copy_range(&src, BLOCK_SIZE, &dst, 0, FILE_BLOCKS * BLOCK_SIZE).is_err()); + } +} diff --git a/src/lib.rs b/src/lib.rs index db5c98a0..9cc77c8c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,7 +24,6 @@ use file::{bmap::Bmap, compression::Compression, functions::FileCopyToParams}; use log::{error, info}; use std::{ fs, - io::Read, path::{Path, PathBuf}, }; use tokio::fs::remove_dir_all; @@ -196,22 +195,7 @@ fn copy_image(from: &Path, to: &Path) -> Result<()> { dst.set_len(len) .context("copy_image: cannot set destination size")?; - // one copy_file_range call copies at most 2 GiB; io::copy repeats it until a segment is done - let sparse = libfs::probably_sparse(&src)?; - let mut pos = 0; - while pos < len { - let (data, hole) = if sparse { - libfs::next_sparse_segments(&src, &dst, pos)? - } else { - (pos, len) - }; - let copied = std::io::copy(&mut (&src).take(hole - data), &mut &dst) - .context("copy_image: cannot copy data")?; - anyhow::ensure!(copied == hole - data, "copy_image: source ended early"); - pos = hole; - } - - Ok(()) + file::sparse::copy_range(&src, 0, &dst, 0, len) } struct CertInfo { diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index fb7f165e..e570738a 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -903,6 +903,9 @@ fn check_file_copy(tr: Testrunner, partition: &str) { const STAT_BLOCK_SIZE: u64 = 512; // content an earlier installation left on the device const OLD_CONTENT_BYTE: u8 = 0xa5; +// a file with a zero range that spans whole image blocks +const ZERO_BLOCKS_FILE_EDGE: usize = 4096; +const ZERO_BLOCKS_FILE_ZEROS: usize = 64 * 1024; const OTHER_IMAGE_BMAP: &str = "40964096\ 0"; @@ -939,6 +942,16 @@ fn bmap_flash(image: &[u8], bmap: &Bmap) -> Vec { device } +fn sparse_copy(from: &Path, to: &Path) { + let status = std::process::Command::new("cp") + .arg("--sparse=always") + .arg(from) + .arg(to) + .status() + .expect("run cp"); + assert!(status.success(), "sparse copy of the test image failed"); +} + #[test] fn check_file_copy_without_input_bmap_maps_whole_image() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); @@ -970,13 +983,7 @@ fn check_file_copy_keeps_holes() { let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); let in_file = tr.to_pathbuf("testfiles/boot.scr"); - let status = std::process::Command::new("cp") - .arg("--sparse=always") - .arg(&image_path) - .arg(&sparse_path) - .status() - .expect("run cp"); - assert!(status.success(), "sparse copy of the test image failed"); + sparse_copy(&image_path, &sparse_path); Command::cargo_bin("omnect-cli") .expect("omnect-cli binary") @@ -1014,6 +1021,65 @@ fn check_file_copy_keeps_holes() { } } +#[test] +fn check_file_copy_maps_zero_blocks_of_copied_files() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); + let device_path = tr.pathbuf().join("device.wic"); + let in_file = tr.pathbuf().join("zero-blocks.bin"); + let in_content = [ + vec![0x11; ZERO_BLOCKS_FILE_EDGE], + vec![0; ZERO_BLOCKS_FILE_ZEROS], + vec![0x22; ZERO_BLOCKS_FILE_EDGE], + ] + .concat(); + fs::write(&in_file, &in_content).expect("write input file"); + sparse_copy(&image_path, &sparse_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .arg("--input-bmap") + .arg(&input_bmap_path) + .arg("-b") + .assert() + .success(); + + let image = fs::read(&sparse_path).expect("read image"); + fs::write(&device_path, bmap_flash(&image, &read_bmap(&bmap_path))).expect("write device"); + + for partition in ["factory", "boot"] { + let out_file = tr.pathbuf().join(format!("{partition}-out.bin")); + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!( + "{partition}:/my-file,{}", + out_file.to_string_lossy() + )) + .arg("-i") + .arg(&device_path) + .assert() + .success(); + assert!( + fs::read(&out_file).expect("read output file") == in_content, + "{partition}: file differs after a bmap flash" + ); + } +} + #[test] fn check_input_bmap_of_another_image() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); From 02a712bba1057d3ddb5c87d25f040d30d73cce40 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:36:06 +0100 Subject: [PATCH 18/46] fix(file): reject an input bmap whose checksums do not match the image All omnect partitions have a fixed size, so builds of one machine have the same image size, and the bmap of another build passed the size check. It was then used without a warning, and a bmap flash kept old content in blocks the image needs. Each range of the input bmap is now checked against its sha256 before the command changes the image. A mismatch, or a bmap without sha256 range checksums, stops the command with a hint to --input-bmap. The test bmap was not in git, and its checksums matched neither test image. It is now added, with checksums for image.wic, and image.wic.xz is rebuilt from image.wic, so the bmap fits both. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/bmap.rs | 151 ++++++++++++++++++++++++++++++++++--- src/lib.rs | 3 +- testfiles/image.wic.bmap | 69 +++++++++++++++++ testfiles/image.wic.xz | Bin 7868 -> 7228 bytes tests/integration_tests.rs | 29 +++++++ 5 files changed, 240 insertions(+), 12 deletions(-) create mode 100644 testfiles/image.wic.bmap diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 918660e9..c7125a65 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,5 +1,6 @@ use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; +use sha2::{Digest, Sha256}; use std::{ fs, ops::RangeInclusive, @@ -10,6 +11,7 @@ use std::{ const BMAP_EXTENSION: &str = "bmap"; const COMPRESSION_EXTENSIONS: [&str; 5] = ["xz", "gz", "gzip", "bz2", "bzip2"]; +const SHA256_CHECKSUM_TYPE: &str = "sha256"; /// The block ranges of an image that a bmap file maps. #[derive(Debug)] @@ -17,6 +19,8 @@ pub struct Bmap { pub block_size: u64, pub image_size: u64, pub ranges: Vec>, + /// The sha256 of each range, if the bmap has them. + pub sha256: Option>, } impl Bmap { @@ -43,16 +47,31 @@ impl Bmap { let block_size = number("BlockSize")?; anyhow::ensure!(block_size > 0, "Bmap::read: BlockSize is 0"); let image_size = number("ImageSize")?; - let ranges = child("BlockMap")? + let range_nodes: Vec<_> = child("BlockMap")? .children() .filter(|n| n.has_tag_name("Range")) + .collect(); + let ranges = range_nodes + .iter() .map(|n| parse_range(n.text().unwrap_or_default())) .collect::>()?; + let sha256 = child("ChecksumType") + .ok() + .and_then(|n| n.text()) + .is_some_and(|t| t.trim() == SHA256_CHECKSUM_TYPE) + .then(|| { + range_nodes + .iter() + .map(|n| n.attribute("chksum").map(str::to_owned)) + .collect::>() + }) + .flatten(); Ok(Self { block_size, image_size, ranges, + sha256, }) } @@ -67,6 +86,7 @@ impl Bmap { block_size, image_size, ranges, + sha256: None, } } } @@ -114,6 +134,66 @@ pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result Result<()> { + let sha256 = bmap + .sha256 + .as_ref() + .context("verify_checksums: the bmap has no sha256 range checksums")?; + let image = fs::File::open(image_file).context(format!( + "verify_checksums: cannot open {}", + image_file.display() + ))?; + let image_len = image + .metadata() + .context("verify_checksums: cannot get image size")? + .len(); + check_layout(bmap, image_len)?; + + let mut buf = vec![0u8; COPY_CHUNK_SIZE]; + for (range, expected) in bmap.ranges.iter().zip(sha256) { + let end = ((range.end() + 1) * bmap.block_size).min(image_len); + let mut pos = range.start() * bmap.block_size; + let mut hasher = Sha256::new(); + while pos < end { + let n = usize::try_from((end - pos).min(COPY_CHUNK_SIZE as u64))?; + image + .read_exact_at(&mut buf[..n], pos) + .context("verify_checksums: cannot read image")?; + hasher.update(&buf[..n]); + pos += n as u64; + } + let actual = hex(&hasher.finalize()); + anyhow::ensure!( + actual.eq_ignore_ascii_case(expected.trim()), + "verify_checksums: range {range:?} of the image does not match the bmap; \ + the bmap is for another image, pass the right one with --input-bmap" + ); + } + + Ok(()) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +/// Fails if `bmap` is for an image of another size or maps blocks outside the image. +fn check_layout(bmap: &Bmap, image_len: u64) -> Result<()> { + anyhow::ensure!( + bmap.image_size == image_len, + "check_layout: bmap is for an image of {} bytes, the image has {image_len}", + bmap.image_size + ); + let image_blocks = image_len.div_ceil(bmap.block_size); + if let Some(range) = bmap.ranges.iter().find(|r| *r.end() >= image_blocks) { + anyhow::bail!("check_layout: range {range:?} is outside the image"); + } + + Ok(()) +} + /// Writes the zero blocks in the ranges of `bmap` as data. A zero block can be a hole, and a /// bmap created from the image would skip it, so a bmap-based flash would keep what an earlier /// installation left there. @@ -130,20 +210,11 @@ pub fn allocate_ranges(image_file: &Path, bmap: &Bmap) -> Result<()> { .metadata() .context("allocate_ranges: cannot get image size")? .len(); - anyhow::ensure!( - bmap.image_size == image_len, - "allocate_ranges: bmap is for an image of {} bytes, the image has {image_len}", - bmap.image_size - ); + check_layout(bmap, image_len)?; - let image_blocks = image_len.div_ceil(bmap.block_size); let image_block_size = IMAGE_BLOCK_SIZE as u64; let mut buf = vec![0u8; COPY_CHUNK_SIZE]; for range in &bmap.ranges { - anyhow::ensure!( - *range.end() < image_blocks, - "allocate_ranges: range {range:?} is outside the image" - ); // check whole image blocks, also when the bmap uses a smaller block size let start = range.start() * bmap.block_size / image_block_size * image_block_size; let end = ((range.end() + 1) * bmap.block_size) @@ -209,6 +280,7 @@ mod tests { const ZERO_BLOCKS: RangeInclusive = 2..=3; const UNMAPPED_BLOCK: u64 = 5; const SMALL_BLOCK_SIZE: u64 = 512; + const PARTIAL_BLOCK_LEN: usize = 100; const DATA_BYTE: u8 = 0x5a; // content an earlier installation left on the device const OLD_CONTENT_BYTE: u8 = 0xa5; @@ -276,6 +348,11 @@ mod tests { assert_eq!(bmap.image_size, YOCTO_IMAGE_SIZE); let mapped: u64 = bmap.ranges.iter().map(|r| r.end() - r.start() + 1).sum(); assert_eq!(mapped, YOCTO_MAPPED_BLOCKS); + assert_eq!( + bmap.sha256.map(|c| c.len()), + Some(bmap.ranges.len()), + "one checksum per range" + ); } #[test] @@ -289,6 +366,15 @@ mod tests { assert_eq!(bmap.ranges, vec![3..=3, 5..=7]); } + #[test] + fn read_has_no_checksums_without_sha256_checksum_type() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic.bmap"); + fs::write(&path, bmap_xml(BLOCK_SIZE, "0")).expect("write bmap"); + + assert_eq!(Bmap::read(&path).expect("read bmap").sha256, None); + } + #[test] fn read_rejects_invalid_bmaps() { let dir = tempfile::tempdir().expect("create temp dir"); @@ -388,12 +474,54 @@ mod tests { block_size: SMALL_BLOCK_SIZE, image_size: IMAGE_BLOCKS * BLOCK_SIZE, ranges: vec![RangeInclusive::new(first, first + 2)], + sha256: None, }; allocate_ranges(&image, &input).expect("allocate ranges"); assert!(is_mapped(&created_bmap(&image), DATA_BLOCK + 1)); } + #[test] + fn verify_checksums_accepts_bmap_of_the_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + // a mapped last block that is shorter than the block size + fs::OpenOptions::new() + .write(true) + .open(&image) + .expect("open image") + .write_all_at(&[DATA_BYTE; PARTIAL_BLOCK_LEN], IMAGE_BLOCKS * BLOCK_SIZE) + .expect("write partial block"); + + verify_checksums(&image, &created_bmap(&image)).expect("verify checksums"); + } + + #[test] + fn verify_checksums_rejects_changed_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let bmap = created_bmap(&image); + fs::OpenOptions::new() + .write(true) + .open(&image) + .expect("open image") + .write_all_at(&[!DATA_BYTE], DATA_BLOCK * BLOCK_SIZE) + .expect("change data block"); + + let err = verify_checksums(&image, &bmap).expect_err("changed image accepted"); + assert!(err.to_string().contains("--input-bmap"), "{err}"); + } + + #[test] + fn verify_checksums_rejects_bmap_without_checksums() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + let mut bmap = created_bmap(&image); + bmap.sha256 = None; + + assert!(verify_checksums(&image, &bmap).is_err()); + } + #[test] fn allocate_ranges_rejects_bmap_of_another_image() { let dir = tempfile::tempdir().expect("create temp dir"); @@ -406,6 +534,7 @@ mod tests { block_size: BLOCK_SIZE, image_size: IMAGE_BLOCKS * BLOCK_SIZE, ranges: vec![RangeInclusive::new(IMAGE_BLOCKS, IMAGE_BLOCKS)], + sha256: None, }; assert!(allocate_ranges(&image, &outside).is_err()); } diff --git a/src/lib.rs b/src/lib.rs index 9cc77c8c..d7dc96f3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -123,7 +123,8 @@ where None => {} Some(Some((path, bmap))) => { info!("keeping the ranges of {} mapped", path.display()); - file::bmap::allocate_ranges(&tmp_image_file, &bmap) + file::bmap::verify_checksums(&tmp_image_file, &bmap) + .and_then(|_| file::bmap::allocate_ranges(&tmp_image_file, &bmap)) .context(format!("cannot use input bmap {}", path.display()))?; } Some(None) => { diff --git a/testfiles/image.wic.bmap b/testfiles/image.wic.bmap new file mode 100644 index 00000000..7b84de94 --- /dev/null +++ b/testfiles/image.wic.bmap @@ -0,0 +1,69 @@ + + + + + + 30408704 + + + 4096 + + + 7424 + + + 77 + + + sha256 + + + 43b997b9550b07ad422eddddba268d0f0607660f664592a7d749034a4f8b9eea + + + + 0-1 + 1024-1025 + 2048-2055 + 2058-2059 + 2062-2063 + 2190-2191 + 3072-3079 + 3082-3083 + 4094-4103 + 4106-4107 + 5104-5105 + 5120-5127 + 5130-5131 + 6128-6129 + 6144-6151 + 6154-6155 + 7152-7153 + 7168-7175 + 7178-7179 + 7423 + + diff --git a/testfiles/image.wic.xz b/testfiles/image.wic.xz index b86e9327a458826a63bbc40906595a2e692e93c2..6053d9aeffdce7a0b021c338ac0971d6b949b7a2 100644 GIT binary patch delta 2903 zcmV-d3#jzGJ-j#={Wp48S^xk5{}S0j0stWa8~^|S5YJf5u^>za0S=RG1{;4R(zOkZ z7QxXzq#AUP@D6n848He@xtep0KVxkbTN5p!jApms0-6`*l{&w;p@;6@e5}BvbNVg1 zfYj04-{F`Zv}c;saDeR>2rA)B@YfDC;x-(!=#nW)fmy&c0N<^>O-VC(EhH1FUXWM8 z=PdES8CXJ0Gxb#P6+b6~o^^leGG@UqR*ngpmtD`PRc{|RD@BIzG&b(VM?8C0bo2(P zEoM4V@g~EH(Zc#r>~n-2g-rxScj80Hg#uul0VSv-dzlQB4f9J4-i)$*c9d{lq1 zy0j+7Q)o<4fE7Yo+KIENM%H!wO_2u#Eq#5})#Swor&vPvJA^mLeO+RXUyY(J0rLKQz$cnkuIP!f>5$OkM4qsok z)JTSsar~?3hLZ|5G6P@^Mrb(7obdWPR2H}55qM`t!vzUlxrwEPA1z90hT6aW&V=GQ z2l@OWsvalz_~!)Tw_Sh=%5YzFs+TG6*s1DM@>Gapf_;A274m<}ESv?D{{Y=mR7v@l zyk=+*-=*-XDZBFF?tp@pTq84dq4H`rR}yl3@S0oZ9dE)Cbs12O#0~)FXLSf=S^&sw z@Ai5;aP!3_WS+DdqS$m@S|5AO%m>GnDI;UFiCAvvi*sqDl!WV~vLsS8KJzU3vc^8o z!ig+*G=5kMq!NFvPuo4WgLfF>F6J*1LgXwLlfx2d?^z1>bHddD&$@$DF(y6%o&St6Hp9i}$s9XmQ|WiZaVxgi1O- zUsRu`DR$qD5VG%*w?-&Kg*IAU>w_Nn?C6FFB6%?T(R&!YFg_f;JiaVL8sRL5@gS5|HSyp!N1QW zAXMcER24qZ+kBi>ol~G;bb|K1n3KLeCU6 z#}ZyG^juO$lI=G6iKqXaUYFaIQSsSf`dZd*DI|Zil`0oTh)Krt9qpVOV6+G3M@<6- zev>|wgPsx&cFTXywwv8kNqT*dO49jN)!r=j_f584;qDysai3IXAL^Igv6C(U)+Fks;$x62oYre|6O3p$K z_a@X#s@)KB!Y$mn8>3MK35!i;G}|=kl>|Q)FNTOt87p#NDa97?n2xfVo%^1Q+BVyw%AjLB8P1=etFOMYQlRp-{1VG?0 z^RsXbGy#7B_W;E;CGqb&(3Mc_QwnEgX=11`fPH`;l&qeEN%s%ke=`upV5BT@79I&< zxE-Czf6iXV1ZDHdM0_-p94OwKw96$73Q*#7)Djb@?yPm*wAg8Lj*tQ%fvj~2(*r4P zk<1~iyVEQA3VNg+`Jzj+x`+L4+<78ln5Y;=xTSyFLZ>6qB?-AJ#x#AiMOR?I&DT8N zF3;%ZT-Yjbikp+6wAOA{7|#w8(n!HGJFFQ&Ha(r1n-wykx+`~TUH4`JpN`^U7jOG+ z^)`~m0LLioD0iR-v$zh^D&LLuX~uLum^~2SoPOApaUVU8B;jzLrz2nND9eS>m6(ct zDv^^<7Wf3s143=Ha1Ap7f87D$0D)u^$rjwSxcTrhhwOEzx`ni+u&C(AZv8Eeg0SYF zGS0}1hf_3;HvIU{n5A;#?8Uk3;&Oz5K}YsQaL^!ow=3TB=8;7GqHgWI?g7EVtp=xu zjQ$Fk{nkn71*VN2OEc1M4V@KNPv|_wR^C3$3kR3Rlj$BG_UmkMe??5Oi_yW^4+b^Vh!dVH6#?}NB?5V_sY z?TszfQSN>}kA=koX|(cy9_xhoFFM{dS18*-1kb|lVSU#y>PUz|zGvPlYTVX+7MNoXN~xD@6g zH|{omS*||?i$WfpeK#^akc1Femjg#J+%Hz~LKUdK<8EM%eN?C=o8yg4#33niB9-sf%0XefT79atC4Li97|A*W=X-z_B2MIy> z0QFsLY(4VpH2a_b6#=aP*Wdiz<2&d~zbvO<>Z#0P;ro{q z+|Wz8gxV=W@|grlECkm&w`hrt<#{WO06(GIS7G%II2?72jshj`?nHT5nRNm44}e2OI)wt1aUVU8B;m6^76<_Z-Q5^{vn?DY0e@AX>ai>6-D0<%A^U3X z$x+;bKsO)7kOl*0C11jF?r>uo85aUH-+Q}QVI5c z&P%T}uMwiV_GVZ_d=zH;GpF&QV2$fwyh#sf1@bnjk2@29xW}wZ<{y&@{9(97duf2oPoe87PkQGK{6CW&hkRz%4`R~$t)=F%riY5AOc?^QPgz0|KI=s00001mNx4R+Ug25-i=vh8_0wxQBwLKnO@Tg2`o5wLi?`RXddn{1pq3e*H z!-TgNYC|qT%?o9W2oGAm@|DYmfWFPtGOxPQU>NWOgLv-HF{So;n9T3%&o@!Sj|M0nL;>c2 zgS96%kh#ju)&G>Fl_xqcYf*pOdA%z_zL65Io3&I@P%WkmLxc^GX9(Za&>JJwVO0G1 zfK1S(M1|Iv%zJ=+B+G&rfAMj!MUp7id4~f~nnV~XNworjkJGCsbvBK6ZD6UtOLwwh zB$WJj-9*h(Ivsf@?Z97VbfC0OA*`Te@0L;i-B0pyu3fq22nUSKi`D*r)xsK%ga3Sg ztvRzd@erlOgG{tyOGx$ZAjo$wCO0aoyZyjJN7#G`W7 z!YNN`mN~|Ic7N&~t)t@hlb)^AQasE5l?8UVBpemF$CH|$$P!Af1BD!{PPCK zH`fc#AS-GkO#!8RIz;|BxcJ1juN=lDe%4(TCHfn=JHHz6TJ(NFS(`MWEEQVRkcwM~0yV zL{dsQ_8D7^TBR8dYiC>caV1R6jp)OQvabNoJvSfRqIDXw=z3k6R*1}Wrc4oQI9F{U z>6U7_EStik6tYkTRiA4`!$;1o5DH4o1QlMS3UTMNe>+WkzpVxyFP&c(1Hv3&Qngv* z^-B*Z*{?^d%}r_V%2x z@r$bFMfGQKHenswvC*~#2CM#a2H4nxzE!*?Uvv6VxdMd}89h&< zz$UZV-XH8EM#cae(^_U0K@f{7x*L<>7B+wAU@_jLc^v)Bq{wgb(xgtM-<^s%xH@9n z^aR+f)Tvbz@i;8b)tz#3J=A3hbn}R@u>y;&M+B+H<`|`~(8h-G6s9692V{7nT%3uH{Lq_#$uw(b)6=OPCUfo>D#TIvmPx zv@%cq6VAwIJ9cLNJ*7v#PlD;}J=JL}#R`FHSlEJ5D`?>XgIECEg~973l2m9d7EyNJ zjjehRa){@?w2^JZ>Lp=ViCmctZNs4OLWduS44GIO+k}u9*O_En$qlsR#=+2ZNnFHj zvXbYZ$j%1@?w{VJLd%FhTmX;IPOl~?laCLl1WN$ly|XU`GXZ}!0s;WVJTHQc(NQk$ zr|Y+l&4{;A=lJLv-hIZJ=;>Q?#aYzHl`TlkAURhFzTtdX`!-=Le!PC#We<}3M8`_w z?c@Z!p!yathX++ft6?WTwypH$9?O^FMe=A$$iv8jdC>Fs%enncppu_UQr-H)@V zoU!8FJzjJm9{PVe&ez+@7T?oDRt&4=owN@eILoB-3oYV}ovo3x{b&Io;O>t%2W-9J zCE>Z*rcu#?50RiHF$i0fP>N%2?!-C7O$#i`seM^CX+hJvNH)@HSxF*?phM+yTFfe5 zT6G5va;ZP*#%}<9mj+z*suM#?@8$d(C2S3kVA6wWS)@cvA4mWGtp#DtKUgrJ=F417 z8`M{GZKmPK{&^v_s56gZgne33Vv=-8T*Pg%lINhv&IbhUpWdZH%ZNW*0FTg4uO=vy zj}O2ESIlX5vo8fR0e>q5N&tamAIgp8?eA!EU)iuh1B&!Tv(vrcf^-c0`%B`2@0Nj}y1=E=yGoWcj_yU9n|0_fo0S!;qaC9_UKy#q#0K;`ICHJE3b) zzXNG!af9P9kkvYF8WZL-{%G`4Q#T-`r!X5->OZ5ud|9$*M}L?$)%dI@Rk5k|f8a(s zY~#Z2yfDv@r@ut9p6b1nsYvKe%g3FE(n{f9Brzlis2(7=_-n@9iy|#29`Go};i37i zf`0qg!xX&)uZ=jb9Z^dfFtxj{)?kVE1=jK2Q8@dY5|p-!*sx8_v9__z=bfeGKHw|s zfR=dK3GM`Al79#Z5cZZ;4vrwk{7nd(WPHt*zl%J$->#pzkgVRv_(SX8#+oI`eR#&0 zFKLnDRt-v5DsEP4U0rF^6+{rakg%1KzB}-P#FZk6iG1a$n$UQq_>8q8$&$$I_FBBV z{vcQ|c{1*6o0{^u9H1gEN0St`i>1d6DbVXOGPPpdI)58-4DI#?AGGIb>Gu;OSbH(5 z+_E0e@FWf6IGNL1IzJx;wq1Q>co*uM{ZIMhT-=@+@;4hSFqgnQ%DN!wZ*5)I|3~7v zjB5)~Zu2t7z~T%2A%2EZ`=aB;HB5xDPVa2<&DbG~=hkAw@4BOKb7w%kw=V{6jCMs6 zLWM&xf`5JHx|O&}8KSUT-F4UcbZ|@<_&his?a<<$yYEbWN$PIWp_1GcsqKk~ggLk- zxZs?IVwnm0;xcGdDx1#dN#g%ZegOTA1$?H`KMXfIk%O$*N1vU5iS#`Tk^G(8F~>1} zK~Kge(3L==?bxnFYtD?9#Sf`lL4kg(8f5us@y{ zctc-*aGgFP^%UdaxL*-cly^hK)fkQyBzcqVQ9XF~+A(B~z!?xN7o%(mRpi0oze!Sl0RfY+1Plqd%|iR1|9JsHv%wZ10e=T=Q4VPRK;r0V z1Q7m`BKL_bclmA+1C$ydLn8?F8D=&9pZ^sBy8zaad1<`z(+sVUx@iHWMC_fUI>_OYoS>G(fE#n##YTRqX@gEQ!%zX(#qR5SV}5 zmXI%eX1wTaNemF(s=N_2BAZ`|l&8Z3794V;-Pm2_xn70M=B==Lag9a}0h)^*3O*Hp zn5vVo6fToa3lx*E1bdUP1V;%Y)PX;r|JVUNv%wZ10e@9%Gfl8wJ!;*Xb*#$^6g+NI z5XDv=d(#!_b(Pwpvh!4q^k>FccueJ3Q*t~7Pj!B)_rC#QJPF(8EX2>PEx7HO?#y~5G50xOZuoVaalivXb1VIt;ShKDfB>{hs0c-$!z_s0E z{NTGC*k+$ru#u<_!jK8>0W<*K{rs4d$Pxc@uI8a0F5EMEf&cU!9mRC3TDE5how|@q z4gBK;pJe^bdPIp$ZHpj-LMqC@Ugf3wk1(CiP$t95i{3!L`b%Px^xMUwA$mu|+(iQ| zQs#GlUt4z4I|Z*3rY%%qK``-Tw-NapPg!1C{U2_sX~WY-ubXI_tV9T*z~#jx2N?#U zsH&Qe6#Wxv!7mk#DUK*W$|6zaDyM6}Uqb6kon&Pfy__hd#iR7FmGX#~8(P(q5gK Date: Wed, 28 Oct 2026 13:58:10 +0100 Subject: [PATCH 19/46] docs(file): explain a checksum mismatch after a command without -b A command without -b changes the image but leaves the bmap next to it, so the next command on that image finds a bmap with old checksums and fails. The error and the README now name this case and the way out: use -b on each command, or pass the matching bmap with --input-bmap. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/bmap.rs | 5 +++-- tests/integration_tests.rs | 31 +++++++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8062664f..1e2f8441 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ omnect-cli file copy-to-image --help Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image, otherwise the command fails. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` changes the image but not the bmap next to it, so when you run several commands on one image, use `-b` on each of them, or pass the matching bmap with `--input-bmap`. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/file/bmap.rs b/src/file/bmap.rs index c7125a65..b3b09733 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -167,8 +167,9 @@ pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { let actual = hex(&hasher.finalize()); anyhow::ensure!( actual.eq_ignore_ascii_case(expected.trim()), - "verify_checksums: range {range:?} of the image does not match the bmap; \ - the bmap is for another image, pass the right one with --input-bmap" + "verify_checksums: range {range:?} of the image does not match the bmap. It is the \ + bmap of another image, or of this image before a command without -b changed it. \ + Pass the right bmap with --input-bmap." ); } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 422237d1..472948cb 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1145,6 +1145,37 @@ fn check_input_bmap_of_another_image_with_the_same_size() { assert_eq!(Testrunner::file_hash(&image_path), image_hash); } +#[test] +fn check_chained_commands_with_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_xz_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_path = image_xz_path.with_extension(""); + tr.to_pathbuf("testfiles/image.wic.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let copy_to_image = |image: &Path, output_bmap: bool| { + let mut cmd = Command::cargo_bin("omnect-cli").expect("omnect-cli binary"); + cmd.arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(image); + if output_bmap { + cmd.arg("-b"); + } + cmd.assert() + }; + + // without -b, the bmap next to the image still has the checksums of the original image + copy_to_image(&image_xz_path, false).success(); + copy_to_image(&image_path, true) + .failure() + .stderr(predicate::str::contains("without -b changed it")); + + copy_to_image(&image_xz_path, true).success(); + copy_to_image(&image_path, true).success(); +} + #[test] fn check_bmap_generation_wic() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); From 279f8a80505fb4187d6a8879a5fca8193ec081c8 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:16:57 +0100 Subject: [PATCH 20/46] feat(file): write the output bmap in Rust and keep the image sparse omnect-cli wrote the zero blocks of the input bmap ranges into the image, or every zero block without an input bmap, only so that bmaptool create mapped them. That fully allocated the image in /tmp and in the output, also without -b, and a sparse copy of the output lost the ranges again. The bmap is now written by omnect-cli: the ranges of the input bmap, or the whole image without one, plus every block the image file stores as data, with a sha256 per range. bmaptool copy writes a mapped hole as zeros, so the flash is the same, and the image stays sparse. The input bmap is only read with -b, --input-bmap requires -b, and -b works in the Docker image because bmaptool is not needed at runtime any more. The bmap is copied next to the image after the image was written back, so a failed write back does not leave a bmap for an image that is not there. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Cargo.toml | 2 +- Dockerfile | 1 - README.md | 7 +- project-context.md | 5 +- src/cli.rs | 40 ++--- src/file/bmap.rs | 319 ++++++++++++++++++++----------------- src/file/sparse.rs | 21 +++ src/lib.rs | 248 ++++++++++++---------------- tests/integration_tests.rs | 62 +++++++ 9 files changed, 387 insertions(+), 318 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e9081bae..c1b0c5d2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -82,5 +82,5 @@ tar = "0.4" # metadata for building with cargo-deb (https://crates.io/crates/cargo-deb) [package.metadata.deb] -depends = "bmap-tools, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" +depends = "e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools" revision = "" diff --git a/Dockerfile b/Dockerfile index 9300a868..9da78d24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,7 +13,6 @@ COPY ${debian_dir}/omnect-cli_${omnect_cli_version}_amd64.deb omnect-cli_${omnec RUN apt-get update && \ apt-get install -y --no-install-recommends \ - bmap-tools \ ca-certificates \ e2tools \ fdisk \ diff --git a/README.md b/README.md index 1e2f8441..eda55c06 100644 --- a/README.md +++ b/README.md @@ -48,9 +48,8 @@ docker run --rm -it \ omnect/omnect-cli:latest file copy-to-image --files /source/my-source-file,boot:/my-dest-file -i /source/my-image.wic ``` - **Note1**: `-b` (`--output-bmap`) option to create a bmap file is not supported by omnect-cli docker image.
- **Note2**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.
- **Note3**: The docker inject command is not supported by omnect-cli docker image.
. + **Note1**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.
+ **Note2**: The docker inject command is not supported by omnect-cli docker image.
. # Build from sources @@ -188,7 +187,7 @@ omnect-cli file copy-to-image --help Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` changes the image but not the bmap next to it, so when you run several commands on one image, use `-b` on each of them, or pass the matching bmap with `--input-bmap`. Every range the input bmap maps stays mapped, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). The created bmap maps these ranges, the blocks `omnect-cli` changed, and all other blocks that the image file stores as data. For an uncompressed image file that is not sparse, this is the whole image. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` changes the image but not the bmap next to it, so when you run several commands on one image, use `-b` on each of them, or pass the matching bmap with `--input-bmap`. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/project-context.md b/project-context.md index 554c6b73..3b8bbd1a 100644 --- a/project-context.md +++ b/project-context.md @@ -50,8 +50,9 @@ - `src/docker.rs` — `docker pull --platform` + `docker save` for multi-arch images - `src/image.rs` — firmware image architecture detection (ARM32/ARM64/x86_64) - `src/file/mod.rs` — high-level image operations: identity config, certs, hostname patching -- `src/file/functions.rs` — partition read via dd, modify via e2cp/mcopy, write back of changed blocks only -- `src/file/bmap.rs` — input bmap lookup; maps its ranges in the working image, or the whole image without one; bmap generation via `bmaptool create` +- `src/file/functions.rs` — sparse partition read, modify via e2cp/mcopy, write back of changed blocks only +- `src/file/bmap.rs` — input bmap lookup and checksum check; writes the output bmap: input ranges (or the whole image) plus data blocks +- `src/file/sparse.rs` — data/hole ranges via `SEEK_DATA`/`SEEK_HOLE`, sparse range copy - `src/file/partition.rs` — GPT/MBR partition table parsing - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys diff --git a/src/cli.rs b/src/cli.rs index 8d7278a7..cece8d9a 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -34,11 +34,11 @@ pub enum Docker { /// destination path of the docker image in the firmware image (must end in ".tar.gz") #[clap(short = 'e', long = "dest")] dest: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -58,11 +58,11 @@ pub enum File { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -94,11 +94,11 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -121,11 +121,11 @@ pub enum IdentityConfig { /// path to device identity certificate key file #[arg(short = 'k', long = "device_identity_key")] device_identity_key: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -142,11 +142,11 @@ pub enum IdentityConfig { /// path to root ca certificate file #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -169,11 +169,11 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -190,11 +190,11 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -217,11 +217,11 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -241,11 +241,11 @@ pub enum IotHubDeviceUpdate { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] @@ -363,11 +363,11 @@ pub enum SshConfig { /// path to public key of the ssh root ca #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: create a bmap of the output image (currently not working in docker image) + /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] output_bmap: bool, /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap")] + #[arg(long = "input-bmap", requires = "output_bmap")] input_bmap: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] diff --git a/src/file/bmap.rs b/src/file/bmap.rs index b3b09733..62a2cf80 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,17 +1,18 @@ -use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, is_zero}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, sparse}; use anyhow::{Context, Result}; use sha2::{Digest, Sha256}; use std::{ fs, - ops::RangeInclusive, + ops::{Range, RangeInclusive}, os::unix::fs::FileExt, path::{Path, PathBuf}, - process::Command, }; -const BMAP_EXTENSION: &str = "bmap"; +pub const BMAP_EXTENSION: &str = "bmap"; const COMPRESSION_EXTENSIONS: [&str; 5] = ["xz", "gz", "gzip", "bz2", "bzip2"]; const SHA256_CHECKSUM_TYPE: &str = "sha256"; +const SHA256_HEX_LEN: usize = 64; +const BMAP_VERSION: &str = "2.0"; /// The block ranges of an image that a bmap file maps. #[derive(Debug)] @@ -74,21 +75,6 @@ impl Bmap { sha256, }) } - - pub fn whole_image(image_size: u64) -> Self { - let block_size = IMAGE_BLOCK_SIZE as u64; - let ranges = match image_size { - 0 => vec![], - size => vec![RangeInclusive::new(0, (size - 1) / block_size)], - }; - - Self { - block_size, - image_size, - ranges, - sha256: None, - } - } } fn parse_range(text: &str) -> Result> { @@ -153,18 +139,9 @@ pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { let mut buf = vec![0u8; COPY_CHUNK_SIZE]; for (range, expected) in bmap.ranges.iter().zip(sha256) { - let end = ((range.end() + 1) * bmap.block_size).min(image_len); - let mut pos = range.start() * bmap.block_size; - let mut hasher = Sha256::new(); - while pos < end { - let n = usize::try_from((end - pos).min(COPY_CHUNK_SIZE as u64))?; - image - .read_exact_at(&mut buf[..n], pos) - .context("verify_checksums: cannot read image")?; - hasher.update(&buf[..n]); - pos += n as u64; - } - let actual = hex(&hasher.finalize()); + let bytes = + range.start() * bmap.block_size..((range.end() + 1) * bmap.block_size).min(image_len); + let actual = range_sha256(&image, bytes, &mut buf)?; anyhow::ensure!( actual.eq_ignore_ascii_case(expected.trim()), "verify_checksums: range {range:?} of the image does not match the bmap. It is the \ @@ -176,6 +153,105 @@ pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { Ok(()) } +/// Writes a bmap of `image_file` to `bmap_file`. It maps the ranges of `input`, or the whole +/// image without one, and every block that the image file stores as data. A range of `input` +/// can be all zeros and a hole in the image, and a bmap flash still has to write it, else the +/// device keeps what an earlier installation left there. +pub fn write_bmap(image_file: &Path, input: Option<&Bmap>, bmap_file: &Path) -> Result<()> { + let image = fs::File::open(image_file) + .context(format!("write_bmap: cannot open {}", image_file.display()))?; + let image_len = image + .metadata() + .context("write_bmap: cannot get image size")? + .len(); + let block_size = IMAGE_BLOCK_SIZE as u64; + + let ranges = mapped_ranges(&image, image_len, input)?; + + let mut buf = vec![0u8; COPY_CHUNK_SIZE]; + let mut block_map = String::new(); + for range in &ranges { + let bytes = range.start() * block_size..((range.end() + 1) * block_size).min(image_len); + let sha256 = range_sha256(&image, bytes, &mut buf)?; + block_map.push_str(&format!( + " {}-{} \n", + range.start(), + range.end() + )); + } + let blocks = image_len.div_ceil(block_size); + let mapped: u64 = ranges.iter().map(|r| r.end() - r.start() + 1).sum(); + + // like bmaptool, the file checksum is taken with zeros in place of itself + let zeros = "0".repeat(SHA256_HEX_LEN); + let xml = format!( + "\n\ + \n \ + {image_len} \n \ + {block_size} \n \ + {blocks} \n \ + {mapped} \n \ + {SHA256_CHECKSUM_TYPE} \n \ + {zeros} \n \ + \n{block_map} \n\ + \n" + ); + let xml = xml.replacen(&zeros, &hex(&Sha256::digest(xml.as_bytes())), 1); + + fs::write(bmap_file, xml).context(format!("write_bmap: cannot write {}", bmap_file.display())) +} + +/// The blocks of the created bmap: the ranges of `input`, or the whole image without one, and +/// every block that the image file stores as data. +fn mapped_ranges( + image: &fs::File, + image_len: u64, + input: Option<&Bmap>, +) -> Result>> { + let block_size = IMAGE_BLOCK_SIZE as u64; + let mut ranges: Vec> = match input { + Some(bmap) => { + check_layout(bmap, image_len)?; + bmap.ranges + .iter() + .map(|r| r.start() * bmap.block_size..(r.end() + 1) * bmap.block_size) + .collect() + } + None => vec![Range { + start: 0, + end: image_len, + }], + }; + ranges.extend(sparse::data_ranges(image, 0, image_len)?); + Ok(sparse::merge( + ranges + .into_iter() + .map(|r| { + r.start / block_size * block_size..r.end.min(image_len).next_multiple_of(block_size) + }) + .filter(|r| !r.is_empty()) + .collect(), + ) + .into_iter() + .map(|r| r.start / block_size..=r.end / block_size - 1) + .collect()) +} + +fn range_sha256(image: &fs::File, bytes: Range, buf: &mut [u8]) -> Result { + let mut hasher = Sha256::new(); + let mut pos = bytes.start; + while pos < bytes.end { + let n = usize::try_from((bytes.end - pos).min(buf.len() as u64))?; + image + .read_exact_at(&mut buf[..n], pos) + .context("range_sha256: cannot read image")?; + hasher.update(&buf[..n]); + pos += n as u64; + } + + Ok(hex(&hasher.finalize())) +} + fn hex(bytes: &[u8]) -> String { bytes.iter().map(|b| format!("{b:02x}")).collect() } @@ -195,82 +271,6 @@ fn check_layout(bmap: &Bmap, image_len: u64) -> Result<()> { Ok(()) } -/// Writes the zero blocks in the ranges of `bmap` as data. A zero block can be a hole, and a -/// bmap created from the image would skip it, so a bmap-based flash would keep what an earlier -/// installation left there. -pub fn allocate_ranges(image_file: &Path, bmap: &Bmap) -> Result<()> { - let image = fs::OpenOptions::new() - .read(true) - .write(true) - .open(image_file) - .context(format!( - "allocate_ranges: cannot open {}", - image_file.display() - ))?; - let image_len = image - .metadata() - .context("allocate_ranges: cannot get image size")? - .len(); - check_layout(bmap, image_len)?; - - let image_block_size = IMAGE_BLOCK_SIZE as u64; - let mut buf = vec![0u8; COPY_CHUNK_SIZE]; - for range in &bmap.ranges { - // check whole image blocks, also when the bmap uses a smaller block size - let start = range.start() * bmap.block_size / image_block_size * image_block_size; - let end = ((range.end() + 1) * bmap.block_size) - .next_multiple_of(image_block_size) - .min(image_len); - - let mut pos = start; - while pos < end { - let n = usize::try_from((end - pos).min(COPY_CHUNK_SIZE as u64))?; - let chunk = &mut buf[..n]; - image - .read_exact_at(chunk, pos) - .context("allocate_ranges: cannot read image")?; - write_zero_blocks(&image, chunk, pos)?; - pos += n as u64; - } - } - - Ok(()) -} - -fn write_zero_blocks(image: &fs::File, chunk: &[u8], pos: u64) -> Result<()> { - let blocks: Vec<(u64, &[u8])> = if is_zero(chunk) { - vec![(pos, chunk)] - } else { - chunk - .chunks(IMAGE_BLOCK_SIZE) - .enumerate() - .filter(|(_, block)| is_zero(block)) - .map(|(i, block)| (pos + (i * IMAGE_BLOCK_SIZE) as u64, block)) - .collect() - }; - - blocks.into_iter().try_for_each(|(pos, block)| { - image - .write_all_at(block, pos) - .context("allocate_ranges: cannot write image") - }) -} - -pub fn generate_bmap_file(image_file: &str) -> Result<()> { - let mut bmaptool = Command::new("bmaptool"); - bmaptool - .arg("create") - .arg("-o") - .arg(format!("{image_file}.{BMAP_EXTENSION}")) - .arg(image_file); - let status = bmaptool - .status() - .context(format!("generate_bmap_file: cannot run {bmaptool:?}"))?; - anyhow::ensure!(status.success(), "generate_bmap_file: {bmaptool:?} failed"); - - Ok(()) -} - #[cfg(test)] mod tests { use crate::file::bmap::*; @@ -296,7 +296,7 @@ mod tests { ) } - fn write_bmap(path: &Path, image_size: u64, ranges: &[RangeInclusive]) { + fn write_input_bmap(path: &Path, image_size: u64, ranges: &[RangeInclusive]) { let block_map: String = ranges .iter() .map(|r| format!("{}-{}", r.start(), r.end())) @@ -317,10 +317,10 @@ mod tests { path } - fn created_bmap(image: &Path) -> Bmap { - let image = image.to_str().expect("image path"); - generate_bmap_file(image).expect("generate bmap"); - Bmap::read(Path::new(&format!("{image}.{BMAP_EXTENSION}"))).expect("read bmap") + fn created_bmap(image: &Path, input: Option<&Bmap>) -> Bmap { + let path = image.with_extension(BMAP_EXTENSION); + write_bmap(image, input, &path).expect("write bmap"); + Bmap::read(&path).expect("read bmap") } fn is_mapped(bmap: &Bmap, block: u64) -> bool { @@ -428,27 +428,29 @@ mod tests { } #[test] - fn allocate_ranges_maps_zero_ranges_of_the_input_bmap() { + fn write_bmap_maps_zero_ranges_of_the_input_bmap() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); let input_bmap = dir.path().join("input.bmap"); - write_bmap( - &input_bmap, - IMAGE_BLOCKS * BLOCK_SIZE, - &[DATA_BLOCK..=DATA_BLOCK, ZERO_BLOCKS], - ); + write_input_bmap(&input_bmap, IMAGE_BLOCKS * BLOCK_SIZE, &[ZERO_BLOCKS]); let input = Bmap::read(&input_bmap).expect("read bmap"); - let before = created_bmap(&image); - assert!(ZERO_BLOCKS.clone().all(|b| !is_mapped(&before, b))); - - allocate_ranges(&image, &input).expect("allocate ranges"); - let after = created_bmap(&image); - assert!(ZERO_BLOCKS.clone().all(|b| is_mapped(&after, b))); - assert!(!is_mapped(&after, UNMAPPED_BLOCK)); + let bmap = created_bmap(&image, Some(&input)); + assert!(ZERO_BLOCKS.clone().all(|b| is_mapped(&bmap, b))); + assert!( + is_mapped(&bmap, DATA_BLOCK), + "data outside the input ranges" + ); + assert!(!is_mapped(&bmap, UNMAPPED_BLOCK)); + let file = fs::File::open(&image).expect("open image"); + assert_eq!( + sparse::data_ranges(&file, 0, IMAGE_BLOCKS * BLOCK_SIZE).expect("data ranges"), + vec![DATA_BLOCK * BLOCK_SIZE..(DATA_BLOCK + 1) * BLOCK_SIZE], + "image is not sparse any more" + ); let device = dir.path().join("device"); - bmap_flash(&image, &after, &device); + bmap_flash(&image, &bmap, &device); let device = fs::read(&device).expect("read device"); let zero_start = usize::try_from(ZERO_BLOCKS.start() * BLOCK_SIZE).expect("start"); let zero_end = usize::try_from((ZERO_BLOCKS.end() + 1) * BLOCK_SIZE).expect("end"); @@ -456,17 +458,18 @@ mod tests { } #[test] - fn allocate_ranges_maps_whole_image() { + fn write_bmap_without_input_maps_whole_image() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); - allocate_ranges(&image, &Bmap::whole_image(IMAGE_BLOCKS * BLOCK_SIZE)) - .expect("allocate ranges"); - assert_eq!(created_bmap(&image).ranges, vec![0..=IMAGE_BLOCKS - 1]); + assert_eq!( + created_bmap(&image, None).ranges, + vec![0..=IMAGE_BLOCKS - 1] + ); } #[test] - fn allocate_ranges_maps_whole_blocks_for_a_smaller_bmap_block_size() { + fn write_bmap_maps_whole_blocks_for_a_smaller_bmap_block_size() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); // starts in DATA_BLOCK and ends in the hole after it @@ -478,8 +481,35 @@ mod tests { sha256: None, }; - allocate_ranges(&image, &input).expect("allocate ranges"); - assert!(is_mapped(&created_bmap(&image), DATA_BLOCK + 1)); + assert!(is_mapped( + &created_bmap(&image, Some(&input)), + DATA_BLOCK + 1 + )); + } + + #[test] + fn write_bmap_is_accepted_by_bmaptool() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = sparse_image(dir.path()); + // a mapped last block that is shorter than the block size + fs::OpenOptions::new() + .write(true) + .open(&image) + .expect("open image") + .write_all_at(&[DATA_BYTE; PARTIAL_BLOCK_LEN], IMAGE_BLOCKS * BLOCK_SIZE) + .expect("write partial block"); + let bmap = image.with_extension(BMAP_EXTENSION); + write_bmap(&image, None, &bmap).expect("write bmap"); + + let status = std::process::Command::new("bmaptool") + .arg("copy") + .arg("--bmap") + .arg(&bmap) + .arg(&image) + .arg(dir.path().join("device")) + .status() + .expect("run bmaptool"); + assert!(status.success(), "bmaptool rejected the bmap"); } #[test] @@ -494,14 +524,14 @@ mod tests { .write_all_at(&[DATA_BYTE; PARTIAL_BLOCK_LEN], IMAGE_BLOCKS * BLOCK_SIZE) .expect("write partial block"); - verify_checksums(&image, &created_bmap(&image)).expect("verify checksums"); + verify_checksums(&image, &created_bmap(&image, None)).expect("verify checksums"); } #[test] fn verify_checksums_rejects_changed_image() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); - let bmap = created_bmap(&image); + let bmap = created_bmap(&image, None); fs::OpenOptions::new() .write(true) .open(&image) @@ -517,26 +547,29 @@ mod tests { fn verify_checksums_rejects_bmap_without_checksums() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); - let mut bmap = created_bmap(&image); + let mut bmap = created_bmap(&image, None); bmap.sha256 = None; assert!(verify_checksums(&image, &bmap).is_err()); } #[test] - fn allocate_ranges_rejects_bmap_of_another_image() { + fn bmap_of_another_image_is_rejected() { let dir = tempfile::tempdir().expect("create temp dir"); let image = sparse_image(dir.path()); - - let other_size = Bmap::whole_image((IMAGE_BLOCKS + 1) * BLOCK_SIZE); - assert!(allocate_ranges(&image, &other_size).is_err()); - + let bmap_file = dir.path().join("out.bmap"); + let mut other_size = created_bmap(&image, None); + other_size.image_size += BLOCK_SIZE; let outside = Bmap { block_size: BLOCK_SIZE, image_size: IMAGE_BLOCKS * BLOCK_SIZE, ranges: vec![RangeInclusive::new(IMAGE_BLOCKS, IMAGE_BLOCKS)], - sha256: None, + sha256: Some(vec![String::new()]), }; - assert!(allocate_ranges(&image, &outside).is_err()); + + for bmap in [&other_size, &outside] { + assert!(verify_checksums(&image, bmap).is_err()); + assert!(write_bmap(&image, Some(bmap), &bmap_file).is_err()); + } } } diff --git a/src/file/sparse.rs b/src/file/sparse.rs index 1c9d62bd..4193ddd6 100644 --- a/src/file/sparse.rs +++ b/src/file/sparse.rs @@ -31,6 +31,19 @@ pub fn data_ranges(file: &File, start: u64, end: u64) -> Result>> Ok(ranges) } +/// Sorts `ranges` and joins the ones that overlap or touch. +pub fn merge(mut ranges: Vec>) -> Vec> { + ranges.sort_by_key(|r| r.start); + let mut merged: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + match merged.last_mut() { + Some(last) if range.start <= last.end => last.end = last.end.max(range.end), + _ => merged.push(range), + } + } + merged +} + /// Copies `len` bytes of `src` at `src_offset` to `dst` at `dst_offset`. Holes of `src` are /// skipped, so they stay holes in a new `dst`. pub fn copy_range( @@ -113,6 +126,14 @@ mod tests { ); } + #[test] + fn merge_joins_overlapping_and_touching_ranges() { + assert_eq!( + merge(vec![8..10, 0..2, 1..4, 4..5, 12..12]), + vec![0..5, 8..10, 12..12] + ); + } + #[test] fn copy_range_keeps_holes() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/lib.rs b/src/lib.rs index d7dc96f3..257b8548 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -20,7 +20,11 @@ use cli::{ IotHubDeviceUpdate::{self, SetDeviceConfig as IotHubDeviceUpdateSet}, SshConfig::{SetCertificate, SetConnection}, }; -use file::{bmap::Bmap, compression::Compression, functions::FileCopyToParams}; +use file::{ + bmap::{BMAP_EXTENSION, Bmap}, + compression::Compression, + functions::FileCopyToParams, +}; use log::{error, info}; use std::{ fs, @@ -51,44 +55,28 @@ impl Drop for TempDirGuard { } } -/// What an image command does with the holes and mapped blocks of the image. -enum Layout { - /// The command does not change the image. - Unchanged, - /// Keep the ranges of this bmap, or of the one found next to the image, mapped. - Keep(Option), -} - fn run_image_command( image_file: PathBuf, - layout: Layout, output_bmap: bool, + input_bmap: Option, target_compression: Option, command: F, ) -> Result<()> where F: FnOnce(&PathBuf) -> Result<()>, { - if let Ok("true") | Ok("1") = std::env::var("CONTAINERIZED").as_deref() { - anyhow::ensure!( - !output_bmap, - "run_image_command: generating bmap file is not supported in containerized environments." - ); - } - anyhow::ensure!( image_file.try_exists().is_ok_and(|exists| exists), "run_image_command: image doesn't exist {}", image_file.to_str().context("cannot get image file path")? ); - let input_bmap = match layout { - Layout::Unchanged => None, - Layout::Keep(explicit) => Some( - file::bmap::find_input_bmap(&image_file, explicit.as_deref())? - .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) - .transpose()?, - ), + let input_bmap = if output_bmap { + file::bmap::find_input_bmap(&image_file, input_bmap.as_deref())? + .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) + .transpose()? + } else { + None }; let mut dest_image_file = image_file.clone(); @@ -119,48 +107,30 @@ where ))?; } - match input_bmap { - None => {} - Some(Some((path, bmap))) => { - info!("keeping the ranges of {} mapped", path.display()); - file::bmap::verify_checksums(&tmp_image_file, &bmap) - .and_then(|_| file::bmap::allocate_ranges(&tmp_image_file, &bmap)) - .context(format!("cannot use input bmap {}", path.display()))?; - } - Some(None) => { - info!("no input bmap found, so every block of the image is allocated"); - let image_size = fs::metadata(&tmp_image_file) - .context("run_image_command: cannot get image size")? - .len(); - file::bmap::allocate_ranges(&tmp_image_file, &Bmap::whole_image(image_size))?; - } + if let Some((path, bmap)) = &input_bmap { + file::bmap::verify_checksums(&tmp_image_file, bmap) + .context(format!("cannot use input bmap {}", path.display()))?; } // run command command(&tmp_image_file)?; - // create and copy back bmap file if one was created + let tmp_bmap = PathBuf::from(format!( + "{}.{BMAP_EXTENSION}", + tmp_image_file + .to_str() + .context("cannot get image file path")? + )); if output_bmap { - let mut target_bmap = image_file - .parent() - .context("cannot get parent dir of image path")? - .to_path_buf(); - let tmp_bmap = PathBuf::from(format!( - "{}.bmap", - tmp_image_file - .to_str() - .context("cannot get image file path")? - )); - file::bmap::generate_bmap_file( - tmp_image_file - .to_str() - .context("cannot get image file path")?, + match &input_bmap { + Some((path, _)) => info!("the bmap maps the ranges of {}", path.display()), + None => info!("no input bmap found, so the bmap maps the whole image"), + } + file::bmap::write_bmap( + &tmp_image_file, + input_bmap.as_ref().map(|(_, bmap)| bmap), + &tmp_bmap, )?; - target_bmap.push(tmp_bmap.file_name().context("cannot get bmap file name")?); - std::fs::copy(&tmp_bmap, &target_bmap).context(format!( - "error: std::fs::copy({:?}, {:?})", - tmp_bmap, target_bmap - ))?; } // if applicable compress image @@ -182,6 +152,18 @@ where ))?; } + // copy the bmap last, so it never describes an image that was not written back + if output_bmap { + let target_bmap = image_file + .parent() + .context("cannot get parent dir of image path")? + .join(tmp_bmap.file_name().context("cannot get bmap file name")?); + std::fs::copy(&tmp_bmap, &target_bmap).context(format!( + "error: std::fs::copy({:?}, {:?})", + tmp_bmap, target_bmap + ))?; + } + Ok(()) } @@ -240,8 +222,6 @@ fn create_image_cert(image: &Path, cert_opts: CertificateOptions) -> Result Result<()> { - use Layout::{Keep, Unchanged}; - match cli::from_args() { Command::Docker(Inject { docker_image, @@ -251,46 +231,40 @@ pub fn run() -> Result<()> { output_bmap, input_bmap, compress_image, - }) => run_image_command( - image, - Keep(input_bmap), - output_bmap, - compress_image, - |img| { - anyhow::ensure!( - dest.to_string_lossy().ends_with(".tar.gz"), - format!( - "invalid destination file path \"{}\". Must end in \".tar.gz\".", - dest.to_string_lossy(), - ), + }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + anyhow::ensure!( + dest.to_string_lossy().ends_with(".tar.gz"), + format!( + "invalid destination file path \"{}\". Must end in \".tar.gz\".", + dest.to_string_lossy(), + ), + ); + + let arch = image::image_arch(img)?; + + let docker_path = docker::pull_image(&docker_image, arch)?; + + let result = file::copy_to_image( + &[FileCopyToParams::new( + &docker_path, + partition.clone(), + &dest, + )], + img, + ); + std::fs::remove_file(docker_path)?; + + if result.is_ok() { + println!( + "Stored {} to {}:{}", + docker_image, + partition, + dest.to_string_lossy(), ); + } - let arch = image::image_arch(img)?; - - let docker_path = docker::pull_image(&docker_image, arch)?; - - let result = file::copy_to_image( - &[FileCopyToParams::new( - &docker_path, - partition.clone(), - &dest, - )], - img, - ); - std::fs::remove_file(docker_path)?; - - if result.is_ok() { - println!( - "Stored {} to {}:{}", - docker_image, - partition, - dest.to_string_lossy(), - ); - } - - result - }, - )?, + result + })?, Command::Identity(SetConfig { config, image, @@ -298,13 +272,9 @@ pub fn run() -> Result<()> { output_bmap, input_bmap, compress_image, - }) => run_image_command( - image, - Keep(input_bmap), - output_bmap, - compress_image, - |img| file::set_identity_config(&config, img, payload.as_deref()), - )?, + }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + file::set_identity_config(&config, img, payload.as_deref()) + })?, Command::Identity(SetDeviceCertificate { intermediate_full_chain_cert, intermediate_key, @@ -328,20 +298,14 @@ pub fn run() -> Result<()> { ) .context("set_device_certificate: could not create certificate")?; - run_image_command( - image, - Keep(input_bmap), - output_bmap, - compress_image, - |img| { - file::set_device_cert( - Some(&intermediate_full_chain_cert), - &cert_info.cert_path, - &cert_info.key_path, - img, - ) - }, - )? + run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + file::set_device_cert( + Some(&intermediate_full_chain_cert), + &cert_info.cert_path, + &cert_info.key_path, + img, + ) + })? } Command::Identity(SetEdgeCaCertificate { intermediate_full_chain_cert, @@ -366,20 +330,14 @@ pub fn run() -> Result<()> { ) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command( - image, - Keep(input_bmap), - output_bmap, - compress_image, - |img| { - file::set_edge_ca_cert( - Some(&intermediate_full_chain_cert), - &cert_info.cert_path, - &cert_info.key_path, - img, - ) - }, - )? + run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + file::set_edge_ca_cert( + Some(&intermediate_full_chain_cert), + &cert_info.cert_path, + &cert_info.key_path, + img, + ) + })? } Command::Identity(SetDeviceCertificateNoEst { device_cert: device_cert_pem, @@ -388,13 +346,9 @@ pub fn run() -> Result<()> { output_bmap, input_bmap, compress_image, - }) => run_image_command( - image, - Keep(input_bmap), - output_bmap, - compress_image, - |img| file::set_device_cert(None, &device_cert_pem, &device_key_pem, img), - )?, + }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) + })?, Command::Identity(SetIotedgeGatewayConfig { config, image, @@ -406,8 +360,8 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - Keep(input_bmap), output_bmap, + input_bmap, compress_image, |img: &PathBuf| { file::set_iotedge_gateway_config( @@ -428,8 +382,8 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - Keep(input_bmap), output_bmap, + input_bmap, compress_image, |img: &PathBuf| file::set_iot_leaf_sas_config(&config, img, &root_ca), )?, @@ -441,8 +395,8 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - Keep(input_bmap), output_bmap, + input_bmap, compress_image, |img: &PathBuf| file::set_ssh_tunnel_certificate(img, &root_ca), )?, @@ -454,8 +408,8 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - Keep(input_bmap), output_bmap, + input_bmap, compress_image, |img: &PathBuf| { file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) @@ -578,15 +532,15 @@ pub fn run() -> Result<()> { compress_image, }) => run_image_command( image, - Keep(input_bmap), output_bmap, + input_bmap, compress_image, |img: &PathBuf| file::copy_to_image(&file_copy_params, img), )?, Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, Unchanged, false, None, |img: &PathBuf| { + }) => run_image_command(image, false, None, None, |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 472948cb..b76a8648 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1098,6 +1098,7 @@ fn check_input_bmap_of_another_image() { .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) .arg("-i") .arg(&image_path) + .arg("-b") .assert() .failure() .stderr(predicate::str::contains("cannot use input bmap")); @@ -1116,6 +1117,66 @@ fn check_input_bmap_of_another_image() { .success(); } +#[test] +fn check_file_copy_without_output_bmap_keeps_image_sparse() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let sparse_path = image_path.with_file_name("sparse.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + sparse_copy(&image_path, &sparse_path); + // without -b, a bmap next to the image is not used + fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},factory:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&sparse_path) + .assert() + .success(); + + let meta = sparse_path.metadata().expect("image metadata"); + assert!( + meta.blocks() * STAT_BLOCK_SIZE < meta.len(), + "image is not sparse" + ); + assert_eq!( + fs::read_to_string(&bmap_path).expect("read bmap"), + OTHER_IMAGE_BMAP + ); +} + +#[test] +fn check_no_output_bmap_when_image_write_back_fails() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let mut permissions = image_path.metadata().expect("image metadata").permissions(); + permissions.set_readonly(true); + fs::set_permissions(&image_path, permissions).expect("make image read-only"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .assert() + .failure(); + assert!( + !bmap_path.exists(), + "bmap written for an image that was not written back" + ); +} + #[test] fn check_input_bmap_of_another_image_with_the_same_size() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); @@ -1138,6 +1199,7 @@ fn check_input_bmap_of_another_image_with_the_same_size() { .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) .arg("-i") .arg(&image_path) + .arg("-b") .assert() .failure() .stderr(predicate::str::contains("cannot use input bmap")) From 57a1580fff3a225e9dd24f0c5878a271ffe202f1 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:16:58 +0100 Subject: [PATCH 21/46] perf(file): compare only blocks that hold data in a partition write back write_partition read and compared the whole partition, also where both the partition file and the image are holes and no write can be needed. It now goes only over the data ranges of both files. A test pins that a zero block written into an unaligned partition is still mapped. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/functions.rs | 90 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 82 insertions(+), 8 deletions(-) diff --git a/src/file/functions.rs b/src/file/functions.rs index 22c3a15e..3c9429f9 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -459,11 +459,49 @@ fn write_partition( // image too, else a bmap flash keeps the old content there. let partition_data = sparse::data_ranges(&partition, 0, len)?; let image_data = sparse::data_ranges(&image, image_offset, image_offset + len)?; + // a block that is a hole in both files is zero in both, so it never needs a write + let block_size = IMAGE_BLOCK_SIZE as u64; + let spans = sparse::merge( + partition_data + .iter() + .cloned() + .chain( + image_data + .iter() + .map(|r| r.start - image_offset..r.end - image_offset), + ) + .map(|r| r.start / block_size * block_size..r.end.next_multiple_of(block_size).min(len)) + .collect(), + ); let mut src = vec![0u8; COPY_CHUNK_SIZE]; let mut dst = vec![0u8; COPY_CHUNK_SIZE]; - let mut pos = 0; - while pos < len { - let n = usize::try_from((len - pos).min(COPY_CHUNK_SIZE as u64))?; + for span in spans { + write_changed_blocks( + &partition, + &image, + image_offset, + span, + (&partition_data, &image_data), + (&mut src, &mut dst), + )?; + } + + image + .sync_all() + .context("write_partition: cannot sync image") +} + +fn write_changed_blocks( + partition: &fs::File, + image: &fs::File, + image_offset: u64, + span: Range, + (partition_data, image_data): (&[Range], &[Range]), + (src, dst): (&mut [u8], &mut [u8]), +) -> Result<()> { + let mut pos = span.start; + while pos < span.end { + let n = usize::try_from((span.end - pos).min(COPY_CHUNK_SIZE as u64))?; partition .read_exact_at(&mut src[..n], pos) .context("write_partition: cannot read partition file")?; @@ -478,8 +516,7 @@ fn write_partition( let block_pos = pos + (i * IMAGE_BLOCK_SIZE) as u64; let block = block_pos..block_pos + s.len() as u64; let image_block = image_offset + block.start..image_offset + block.end; - if s != d || (has_data(&partition_data, &block) && has_hole(&image_data, &image_block)) - { + if s != d || (has_data(partition_data, &block) && has_hole(image_data, &image_block)) { image .write_all_at(s, image_block.start) .context("write_partition: cannot write image")?; @@ -488,9 +525,7 @@ fn write_partition( pos += n as u64; } - image - .sync_all() - .context("write_partition: cannot sync image") + Ok(()) } fn has_data(data_ranges: &[Range], block: &Range) -> bool { @@ -515,6 +550,8 @@ mod tests { const PART_SECTORS: u64 = 64; const OLD_CONTENT_BYTE: u8 = 0xa5; const ZEROED_BLOCK: usize = 2; + // the first partition of an old MBR layout + const UNALIGNED_PART_START: u64 = 63; fn partition_info() -> PartitionInfo { PartitionInfo { @@ -574,6 +611,43 @@ mod tests { assert_eq!(allocated(), before, "unchanged zero blocks were written"); } + #[test] + fn write_partition_maps_zero_block_of_an_unaligned_partition() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + let info = PartitionInfo { + num: PART_NUM, + start: UNALIGNED_PART_START, + count: PART_SECTORS, + }; + let image_file = fs::File::create(&image).expect("create image"); + image_file + .set_len((UNALIGNED_PART_START + PART_SECTORS) * SECTOR_SIZE) + .expect("set image size"); + let partition_file = fs::File::create(&partition).expect("create partition"); + partition_file + .set_len(PART_SECTORS * SECTOR_SIZE) + .expect("set partition size"); + let block = (ZEROED_BLOCK * IMAGE_BLOCK_SIZE) as u64; + partition_file + .write_all_at(&[0; IMAGE_BLOCK_SIZE], block) + .expect("write zero block"); + + write_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &info, + ) + .expect("write partition"); + + let block = UNALIGNED_PART_START * SECTOR_SIZE + block; + let block = block..block + IMAGE_BLOCK_SIZE as u64; + let image_data = + sparse::data_ranges(&image_file, block.start, block.end).expect("data ranges"); + assert_eq!(image_data, vec![block], "zero block is still a hole"); + } + #[test] fn write_partition_maps_zero_block_written_into_a_hole() { let dir = tempfile::tempdir().expect("create temp dir"); From b59788cff1b328cb9239e13507bb3df192cf01dd Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:25:15 +0100 Subject: [PATCH 22/46] test(file): check the bmaptool format only in the integration tests The unit tests run where bmaptool is not installed. The integration tests already flash the created bmap with bmaptool copy, and the range checksums of a short last block are checked by verify_checksums. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/bmap.rs | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 62a2cf80..b6ea230c 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -487,31 +487,6 @@ mod tests { )); } - #[test] - fn write_bmap_is_accepted_by_bmaptool() { - let dir = tempfile::tempdir().expect("create temp dir"); - let image = sparse_image(dir.path()); - // a mapped last block that is shorter than the block size - fs::OpenOptions::new() - .write(true) - .open(&image) - .expect("open image") - .write_all_at(&[DATA_BYTE; PARTIAL_BLOCK_LEN], IMAGE_BLOCKS * BLOCK_SIZE) - .expect("write partial block"); - let bmap = image.with_extension(BMAP_EXTENSION); - write_bmap(&image, None, &bmap).expect("write bmap"); - - let status = std::process::Command::new("bmaptool") - .arg("copy") - .arg("--bmap") - .arg(&bmap) - .arg(&image) - .arg(dir.path().join("device")) - .status() - .expect("run bmaptool"); - assert!(status.success(), "bmaptool rejected the bmap"); - } - #[test] fn verify_checksums_accepts_bmap_of_the_image() { let dir = tempfile::tempdir().expect("create temp dir"); From 53ebcd8c12e1d3cda13c1cb2c592d73d7aae03fb Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:11:27 +0100 Subject: [PATCH 23/46] chore(docker): drop dd and sync from the image No code calls them any more. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- Dockerfile | 2 -- project-context.md | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9da78d24..91ba58a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,13 +33,11 @@ RUN < Date: Wed, 28 Oct 2026 19:11:28 +0100 Subject: [PATCH 24/46] fix(file): decompress images named .gz and .bz2 The output path was the same as the source for these names, so creating it truncated the source before it was read. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/compression.rs | 82 +++++++++++++++++++++++++++++++++++------ 1 file changed, 70 insertions(+), 12 deletions(-) diff --git a/src/file/compression.rs b/src/file/compression.rs index 2af9cefc..f34b73d4 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -2,12 +2,14 @@ use crate::file::{IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; use filemagic::Magic; use log::debug; -use std::env; -use std::fs::File; -use std::io::{Seek, SeekFrom, Write}; -use std::os::unix::fs::FileExt; -use std::path::PathBuf; -use std::str::FromStr; +use std::{ + env, + fs::File, + io::{Seek, SeekFrom, Write}, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + str::FromStr, +}; use strum::IntoEnumIterator; use strum_macros::EnumIter; @@ -108,13 +110,24 @@ impl Compression { } fn extension(&self) -> &'static str { + self.extensions()[0] + } + + /// The first one is used for a compressed image that omnect-cli creates. + fn extensions(&self) -> &'static [&'static str] { match &self { - Compression::bzip2 => "bzip2", - Compression::gzip => "gzip", - Compression::xz { .. } => "xz", + Compression::bzip2 => &["bzip2", "bz2"], + Compression::gzip => &["gzip", "gz"], + Compression::xz { .. } => &["xz"], } } + pub fn has_extension(path: &Path) -> bool { + path.extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| Compression::iter().any(|c| c.extensions().contains(&e))) + } + pub fn from_file(image_file_name: &PathBuf) -> Result> { let detector = Magic::open(Default::default()) .context("image::compression: failed to open libmagic")?; @@ -193,10 +206,18 @@ pub fn decompress(image_file_name: &PathBuf, compression: &Compression) -> Resul if new_image_file .extension() - .is_some_and(|ext| ext == compression.extension()) + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| compression.extensions().contains(&ext)) { new_image_file.set_extension(""); } + // creating the destination would truncate the source + anyhow::ensure!( + new_image_file != *image_file_name, + "decompress: {} has no {:?} file extension", + image_file_name.display(), + compression.extensions() + ); let mut destination = File::create(&new_image_file)?; let mut source = File::open(image_file_name)?; @@ -223,8 +244,7 @@ pub fn compress(image_file_name: &PathBuf, compression: &Compression) -> Result< #[cfg(test)] mod tests { use crate::file::compression::*; - use std::io::Read; - use std::os::unix::fs::MetadataExt; + use std::{fs, io::Read, os::unix::fs::MetadataExt}; // unit of `st_blocks` const STAT_BLOCK_SIZE: u64 = 512; @@ -339,4 +359,42 @@ mod tests { ); } } + + #[test] + fn decompress_file_strips_every_extension_of_the_compression() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = test_image(); + + for compression in all_compressions() { + for extension in compression.extensions() { + let path = dir.path().join(format!("image.wic.{extension}")); + let mut compressed = compress_image(&compression, &image); + fs::write(&path, read_all(&mut compressed)).expect("write compressed"); + + let decompressed = decompress(&path, &compression).expect("decompress"); + assert_eq!(decompressed, dir.path().join("image.wic")); + assert!( + fs::read(&decompressed).expect("read decompressed") == image, + "{compression:?}: content of .{extension} differs" + ); + assert!(Compression::has_extension(&path)); + } + } + } + + #[test] + fn decompress_file_rejects_unknown_extension() { + let dir = tempfile::tempdir().expect("create temp dir"); + let compression = Compression::gzip; + let path = dir.path().join("image.wic"); + let mut compressed = compress_image(&compression, &test_image()); + let content = read_all(&mut compressed); + fs::write(&path, &content).expect("write compressed"); + + assert!(decompress(&path, &compression).is_err()); + assert!( + fs::read(&path).expect("read compressed") == content, + "source was changed" + ); + } } From cdc299c44f12d16b0a01809decc9119aa3969064 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:11:29 +0100 Subject: [PATCH 25/46] refactor(cli): share the bmap and compression options of image commands Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/cli.rs | 125 ++++++++++------------------------- src/file/bmap.rs | 28 ++++---- src/file/functions.rs | 18 ++--- src/lib.rs | 148 ++++++++++++++---------------------------- 4 files changed, 105 insertions(+), 214 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index cece8d9a..954171f6 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -2,12 +2,25 @@ use crate::file::{ compression::Compression, functions::{FileCopyFromParams, FileCopyToParams, Partition}, }; -use clap::Parser; +use clap::{Args, Parser}; use std::path::PathBuf; use url::Url; const COPYRIGHT: &str = "Copyright © 2021 by conplement AG"; +#[derive(Args, Debug, Default)] +pub struct ImageOutput { + /// optional: create a bmap of the output image + #[arg(short = 'b', long = "output-bmap")] + pub output_bmap: bool, + /// optional: bmap of the input image; default: .bmap or .bmap next to the image + #[arg(long = "input-bmap", requires = "output_bmap")] + pub input_bmap: Option, + /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') + #[arg(short = 'p', long = "pack-image", value_enum)] + pub compress_image: Option, +} + // ToDo: command completion #[derive(Parser, Debug)] #[command(after_help = COPYRIGHT)] @@ -34,15 +47,8 @@ pub enum Docker { /// destination path of the docker image in the firmware image (must end in ".tar.gz") #[clap(short = 'e', long = "dest")] dest: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, } @@ -58,15 +64,8 @@ pub enum File { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// copy files from image CopyFromImage { @@ -94,15 +93,8 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// EXPERIMENTAL: set transparent gateway config.toml file and additional certificates and keys SetIotedgeGatewayConfig { @@ -121,15 +113,8 @@ pub enum IdentityConfig { /// path to device identity certificate key file #[arg(short = 'k', long = "device_identity_key")] device_identity_key: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// EXPERIMENTAL: set leaf device config.toml file and additional certificate SetIotLeafSasConfig { @@ -142,15 +127,8 @@ pub enum IdentityConfig { /// path to root ca certificate file #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set certificates in order to support X.509 based DPS provisioning and certificate renewal via EST SetDeviceCertificate { @@ -169,15 +147,8 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set certificates in order to support X.509 based DPS provisioning WITHOUT certificate renewal via EST SetDeviceCertificateNoEst { @@ -190,15 +161,8 @@ pub enum IdentityConfig { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// generate and set bootstrap certificate for edge ca issuance/renewal. SetEdgeCaCertificate { @@ -217,15 +181,8 @@ pub enum IdentityConfig { /// period of validity in days #[arg(short = 'D', long = "days")] days: u32, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, } @@ -241,15 +198,8 @@ pub enum IotHubDeviceUpdate { /// path to wic image file (optionally compressed with xz, bzip2 or gzip) #[arg(short = 'i', long = "image")] image: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// import update to azure iot-hub ImportUpdate { @@ -363,15 +313,8 @@ pub enum SshConfig { /// path to public key of the ssh root ca #[arg(short = 'r', long = "root_ca")] root_ca: PathBuf, - /// optional: create a bmap of the output image - #[arg(short = 'b', long = "output-bmap")] - output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image - #[arg(long = "input-bmap", requires = "output_bmap")] - input_bmap: Option, - /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') - #[arg(short = 'p', long = "pack-image", value_enum)] - compress_image: Option, + #[command(flatten)] + output: ImageOutput, }, /// set ssh connection parameters (see diff --git a/src/file/bmap.rs b/src/file/bmap.rs index b6ea230c..fbcdd295 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,4 +1,4 @@ -use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, sparse}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, compression::Compression, sparse}; use anyhow::{Context, Result}; use sha2::{Digest, Sha256}; use std::{ @@ -8,8 +8,7 @@ use std::{ path::{Path, PathBuf}, }; -pub const BMAP_EXTENSION: &str = "bmap"; -const COMPRESSION_EXTENSIONS: [&str; 5] = ["xz", "gz", "gzip", "bz2", "bzip2"]; +const BMAP_EXTENSION: &str = "bmap"; const SHA256_CHECKSUM_TYPE: &str = "sha256"; const SHA256_HEX_LEN: usize = 64; const BMAP_VERSION: &str = "2.0"; @@ -103,23 +102,20 @@ pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result PathBuf { + let mut path = image.as_os_str().to_owned(); + path.push(format!(".{BMAP_EXTENSION}")); + PathBuf::from(path) +} + /// Fails if a range of `image_file` does not match its checksum in `bmap`, so the bmap of /// another image with the same size is not used. pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { @@ -201,8 +197,6 @@ pub fn write_bmap(image_file: &Path, input: Option<&Bmap>, bmap_file: &Path) -> fs::write(bmap_file, xml).context(format!("write_bmap: cannot write {}", bmap_file.display())) } -/// The blocks of the created bmap: the ranges of `input`, or the whole image without one, and -/// every block that the image file stores as data. fn mapped_ranges( image: &fs::File, image_len: u64, diff --git a/src/file/functions.rs b/src/file/functions.rs index 3c9429f9..e40f4147 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -1,14 +1,16 @@ use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, partition::SECTOR_SIZE, sparse}; use anyhow::{Context, Result}; use log::{debug, warn}; -use std::collections::HashMap; -use std::fmt::{self, Display}; -use std::fs; -use std::ops::Range; -use std::os::unix::fs::FileExt; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::str::FromStr; +use std::{ + collections::HashMap, + fmt::{self, Display}, + fs, + ops::Range, + os::unix::fs::FileExt, + path::{Path, PathBuf}, + process::Command, + str::FromStr, +}; use stdext::function_name; use uuid::Uuid; diff --git a/src/lib.rs b/src/lib.rs index 257b8548..2fd57ead 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,11 +17,12 @@ use cli::{ SetConfig, SetDeviceCertificate, SetDeviceCertificateNoEst, SetEdgeCaCertificate, SetIotLeafSasConfig, SetIotedgeGatewayConfig, }, + ImageOutput, IotHubDeviceUpdate::{self, SetDeviceConfig as IotHubDeviceUpdateSet}, SshConfig::{SetCertificate, SetConnection}, }; use file::{ - bmap::{BMAP_EXTENSION, Bmap}, + bmap::{self, Bmap}, compression::Compression, functions::FileCopyToParams, }; @@ -55,16 +56,16 @@ impl Drop for TempDirGuard { } } -fn run_image_command( - image_file: PathBuf, - output_bmap: bool, - input_bmap: Option, - target_compression: Option, - command: F, -) -> Result<()> +fn run_image_command(image_file: PathBuf, output: ImageOutput, command: F) -> Result<()> where F: FnOnce(&PathBuf) -> Result<()>, { + let ImageOutput { + output_bmap, + input_bmap, + compress_image: target_compression, + } = output; + anyhow::ensure!( image_file.try_exists().is_ok_and(|exists| exists), "run_image_command: image doesn't exist {}", @@ -72,7 +73,7 @@ where ); let input_bmap = if output_bmap { - file::bmap::find_input_bmap(&image_file, input_bmap.as_deref())? + bmap::find_input_bmap(&image_file, input_bmap.as_deref())? .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) .transpose()? } else { @@ -108,25 +109,20 @@ where } if let Some((path, bmap)) = &input_bmap { - file::bmap::verify_checksums(&tmp_image_file, bmap) + bmap::verify_checksums(&tmp_image_file, bmap) .context(format!("cannot use input bmap {}", path.display()))?; } // run command command(&tmp_image_file)?; - let tmp_bmap = PathBuf::from(format!( - "{}.{BMAP_EXTENSION}", - tmp_image_file - .to_str() - .context("cannot get image file path")? - )); + let tmp_bmap = bmap::bmap_path(&tmp_image_file); if output_bmap { match &input_bmap { Some((path, _)) => info!("the bmap maps the ranges of {}", path.display()), None => info!("no input bmap found, so the bmap maps the whole image"), } - file::bmap::write_bmap( + bmap::write_bmap( &tmp_image_file, input_bmap.as_ref().map(|(_, bmap)| bmap), &tmp_bmap, @@ -228,10 +224,8 @@ pub fn run() -> Result<()> { image, partition, dest, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { anyhow::ensure!( dest.to_string_lossy().ends_with(".tar.gz"), format!( @@ -269,10 +263,8 @@ pub fn run() -> Result<()> { config, image, payload, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { file::set_identity_config(&config, img, payload.as_deref()) })?, Command::Identity(SetDeviceCertificate { @@ -281,9 +273,7 @@ pub fn run() -> Result<()> { image, device_id, days, - output_bmap, - input_bmap, - compress_image, + output, }) => { let cert_info = create_image_cert( &image, @@ -298,7 +288,7 @@ pub fn run() -> Result<()> { ) .context("set_device_certificate: could not create certificate")?; - run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + run_image_command(image, output, |img| { file::set_device_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -313,9 +303,7 @@ pub fn run() -> Result<()> { image, device_id, days, - output_bmap, - input_bmap, - compress_image, + output, }) => { let cert_info = create_image_cert( &image, @@ -330,7 +318,7 @@ pub fn run() -> Result<()> { ) .context("set_edge_ca_certificate: could not create certificate")?; - run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + run_image_command(image, output, |img| { file::set_edge_ca_cert( Some(&intermediate_full_chain_cert), &cert_info.cert_path, @@ -343,10 +331,8 @@ pub fn run() -> Result<()> { device_cert: device_cert_pem, device_key: device_key_pem, image, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command(image, output_bmap, input_bmap, compress_image, |img| { + output, + }) => run_image_command(image, output, |img| { file::set_device_cert(None, &device_cert_pem, &device_key_pem, img) })?, Command::Identity(SetIotedgeGatewayConfig { @@ -355,66 +341,38 @@ pub fn run() -> Result<()> { root_ca, device_identity, device_identity_key, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command( - image, - output_bmap, - input_bmap, - compress_image, - |img: &PathBuf| { - file::set_iotedge_gateway_config( - &config, - img, - &root_ca, - &device_identity, - &device_identity_key, - ) - }, - )?, + output, + }) => run_image_command(image, output, |img: &PathBuf| { + file::set_iotedge_gateway_config( + &config, + img, + &root_ca, + &device_identity, + &device_identity_key, + ) + })?, Command::Identity(SetIotLeafSasConfig { config, image, root_ca, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command( - image, - output_bmap, - input_bmap, - compress_image, - |img: &PathBuf| file::set_iot_leaf_sas_config(&config, img, &root_ca), - )?, + output, + }) => run_image_command(image, output, |img: &PathBuf| { + file::set_iot_leaf_sas_config(&config, img, &root_ca) + })?, Command::Ssh(SetCertificate { image, root_ca, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command( - image, - output_bmap, - input_bmap, - compress_image, - |img: &PathBuf| file::set_ssh_tunnel_certificate(img, &root_ca), - )?, + output, + }) => run_image_command(image, output, |img: &PathBuf| { + file::set_ssh_tunnel_certificate(img, &root_ca) + })?, Command::IotHubDeviceUpdate(IotHubDeviceUpdateSet { iot_hub_device_update_config, image, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command( - image, - output_bmap, - input_bmap, - compress_image, - |img: &PathBuf| { - file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) - }, - )?, + output, + }) => run_image_command(image, output, |img: &PathBuf| { + file::set_iot_hub_device_update_config(&iot_hub_device_update_config, img) + })?, Command::IotHubDeviceUpdate(IotHubDeviceUpdate::ImportUpdate { import_manifest: import_manifest_path, storage_container_name, @@ -527,20 +485,14 @@ pub fn run() -> Result<()> { Command::File(CopyToImage { file_copy_params, image, - output_bmap, - input_bmap, - compress_image, - }) => run_image_command( - image, - output_bmap, - input_bmap, - compress_image, - |img: &PathBuf| file::copy_to_image(&file_copy_params, img), - )?, + output, + }) => run_image_command(image, output, |img: &PathBuf| { + file::copy_to_image(&file_copy_params, img) + })?, Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, false, None, None, |img: &PathBuf| { + }) => run_image_command(image, ImageOutput::default(), |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } From 92ce896a034aa14f3e53486c8f41f7758f5cec27 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:11:29 +0100 Subject: [PATCH 26/46] test(file): make the write back failure test work as root Also check that --input-bmap requires -b. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- tests/integration_tests.rs | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index b76a8648..4cf7ea55 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1153,12 +1153,12 @@ fn check_file_copy_without_output_bmap_keeps_image_sparse() { #[test] fn check_no_output_bmap_when_image_write_back_fails() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); - let image_path = tr.to_pathbuf("testfiles/image.wic"); - let bmap_path = PathBuf::from(format!("{}.bmap", image_path.to_string_lossy())); + let image_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let written_image_path = image_path.with_extension(""); + let bmap_path = image_path.with_extension("bmap"); let in_file = tr.to_pathbuf("testfiles/boot.scr"); - let mut permissions = image_path.metadata().expect("image metadata").permissions(); - permissions.set_readonly(true); - fs::set_permissions(&image_path, permissions).expect("make image read-only"); + // a directory in place of the written image makes the write back fail, also for root + create_dir_all(&written_image_path).expect("create directory"); Command::cargo_bin("omnect-cli") .expect("omnect-cli binary") @@ -1177,6 +1177,29 @@ fn check_no_output_bmap_when_image_write_back_fails() { ); } +#[test] +fn check_input_bmap_requires_output_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let image_hash = Testrunner::file_hash(&image_path); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("--input-bmap") + .arg(&bmap_path) + .assert() + .failure(); + assert_eq!(Testrunner::file_hash(&image_path), image_hash); +} + #[test] fn check_input_bmap_of_another_image_with_the_same_size() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); @@ -1292,8 +1315,7 @@ fn check_bmap_generation_wic_xz() { let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); let image_path_wic = image_path_wic_xz.with_extension(""); let image_path_wic_copy = image_path_wic_xz.with_extension("copy"); - // a Yocto bmap of an image with the same layout; omnect-cli replaces it with the one it - // creates + // the Yocto bmap of the image; omnect-cli replaces it with the one it creates let image_path_bmap = tr.to_pathbuf("testfiles/image.wic.bmap"); let input_bmap = read_bmap(&image_path_bmap); let in_file = tr.to_pathbuf("testfiles/boot.scr"); From 04bb5f1bb216590d8bab6de7087d21fd46e849b7 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:20:23 +0100 Subject: [PATCH 27/46] fix(file): name the output bmap after the output image The bmap was named after the uncompressed image. When the output image is another file than the input image, e.g. with -p, it replaced the bmap of the input image, which stays unchanged. bmaptool finds .bmap first, also for a compressed image. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/bmap.rs | 6 +++--- src/lib.rs | 5 +---- tests/integration_tests.rs | 40 +++++++++++++++++++++++++++++++++++++- 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index eda55c06..9f30eb97 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ omnect-cli file copy-to-image --help ## bmap files -Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic.xz`, and replaces a file with this name. +Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` changes the image but not the bmap next to it, so when you run several commands on one image, use `-b` on each of them, or pass the matching bmap with `--input-bmap`. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. diff --git a/src/file/bmap.rs b/src/file/bmap.rs index fbcdd295..793a9fd5 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -140,9 +140,9 @@ pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { let actual = range_sha256(&image, bytes, &mut buf)?; anyhow::ensure!( actual.eq_ignore_ascii_case(expected.trim()), - "verify_checksums: range {range:?} of the image does not match the bmap. It is the \ - bmap of another image, or of this image before a command without -b changed it. \ - Pass the right bmap with --input-bmap." + "verify_checksums: range {range:?} of the image does not match the bmap, so it is the \ + bmap of another image or of an older version of this image. Pass the right bmap \ + with --input-bmap." ); } diff --git a/src/lib.rs b/src/lib.rs index 2fd57ead..263a4b4b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -150,10 +150,7 @@ where // copy the bmap last, so it never describes an image that was not written back if output_bmap { - let target_bmap = image_file - .parent() - .context("cannot get parent dir of image path")? - .join(tmp_bmap.file_name().context("cannot get bmap file name")?); + let target_bmap = bmap::bmap_path(&dest_image_file); std::fs::copy(&tmp_bmap, &target_bmap).context(format!( "error: std::fs::copy({:?}, {:?})", tmp_bmap, target_bmap diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 4cf7ea55..0f146f11 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1255,7 +1255,7 @@ fn check_chained_commands_with_input_bmap() { copy_to_image(&image_xz_path, false).success(); copy_to_image(&image_path, true) .failure() - .stderr(predicate::str::contains("without -b changed it")); + .stderr(predicate::str::contains("older version of this image")); copy_to_image(&image_xz_path, true).success(); copy_to_image(&image_path, true).success(); @@ -1370,6 +1370,44 @@ fn check_bmap_generation_wic_xz_without_input_bmap() { assert!(maps_whole_image(&read_bmap(&image_path_bmap))); } +#[test] +fn check_bmap_of_compressed_output_keeps_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let input_bmap_hash = Testrunner::file_hash(&input_bmap_path); + let image_path_xz = image_path.with_extension("wic.xz"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_path) + .arg("-b") + .arg("-p") + .arg("xz") + .assert() + .success(); + + assert_eq!( + Testrunner::file_hash(&input_bmap_path), + input_bmap_hash, + "bmap of the unchanged input image was replaced" + ); + // bmaptool finds .bmap without --bmap and checks the image against it + Command::new("bmaptool") + .arg("copy") + .arg(&image_path_xz) + .arg(image_path.with_extension("copy")) + .assert() + .success() + .stderr(predicate::str::contains("image.wic.xz.bmap")); +} + #[test] fn check_image_compression() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); From beafb45cd5329b9b2de891fca5fc1279f73c6e72 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:20:33 +0100 Subject: [PATCH 28/46] perf(file): do not sync the temp image after a partition write SEEK_DATA also sees data that is not flushed yet, and the partition files that e2cp and mcopy write were never synced either. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/functions.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/file/functions.rs b/src/file/functions.rs index e40f4147..09c4312f 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -488,9 +488,7 @@ fn write_partition( )?; } - image - .sync_all() - .context("write_partition: cannot sync image") + Ok(()) } fn write_changed_blocks( From d7ba2da5e32623f92d41dccb5b64a0a1a027214d Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:21:10 +0100 Subject: [PATCH 29/46] test(file): cover a failed compress trailer and a partition past the image end Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/compression.rs | 19 +++++++++++++++++++ src/file/functions.rs | 20 ++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/src/file/compression.rs b/src/file/compression.rs index f34b73d4..4c79da2f 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -397,4 +397,23 @@ mod tests { "source was changed" ); } + + #[test] + fn compress_fails_when_the_trailer_cannot_be_written() { + let dir = tempfile::tempdir().expect("create temp dir"); + // an empty source makes `finish` the first write to the destination + let source = dir.path().join("empty.wic"); + fs::write(&source, []).expect("write source"); + for compression in all_compressions() { + let mut source = fs::File::open(&source).expect("open source"); + let mut full = fs::OpenOptions::new() + .write(true) + .open("/dev/full") + .expect("open /dev/full"); + assert!( + compression.compress(&mut source, &mut full).is_err(), + "{compression:?}: write error was ignored" + ); + } + } } diff --git a/src/file/functions.rs b/src/file/functions.rs index 09c4312f..c1255e33 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -561,6 +561,26 @@ mod tests { } } + #[test] + fn read_partition_fails_for_partition_past_the_image_end() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join("image.wic"); + let partition = dir.path().join("partition.img"); + fs::File::create(&image) + .expect("create image") + .set_len((PART_START + PART_SECTORS - 1) * SECTOR_SIZE) + .expect("set image size"); + + assert!( + read_partition( + image.to_str().expect("image path"), + partition.to_str().expect("partition path"), + &partition_info(), + ) + .is_err() + ); + } + #[test] fn write_partition_writes_block_changed_to_zeros() { let dir = tempfile::tempdir().expect("create temp dir"); From 1ce9e1a31cce8066a4ba77b17f9c5233b09e410f Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:15:53 +0100 Subject: [PATCH 30/46] refactor(file): share the sparse copy into a new file Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/functions.rs | 15 ++++----------- src/file/sparse.rs | 27 ++++++++++++++++++--------- src/lib.rs | 5 +---- 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/src/file/functions.rs b/src/file/functions.rs index c1255e33..9c9ffecc 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -419,19 +419,12 @@ fn read_partition( let image = fs::File::open(image_file).context(format!("read_partition: cannot open {image_file}"))?; - let partition = fs::File::create(partition_file) - .context(format!("read_partition: cannot create {partition_file}"))?; - let len = partition_info.count * SECTOR_SIZE; - partition - .set_len(len) - .context("read_partition: cannot set partition file size")?; - - sparse::copy_range( + + sparse::copy_to_new_file( &image, partition_info.start * SECTOR_SIZE, - &partition, - 0, - len, + Path::new(partition_file), + partition_info.count * SECTOR_SIZE, ) } diff --git a/src/file/sparse.rs b/src/file/sparse.rs index 4193ddd6..3f5a4005 100644 --- a/src/file/sparse.rs +++ b/src/file/sparse.rs @@ -7,6 +7,7 @@ use std::{ fs::File, io::{self, Read, Seek}, ops::Range, + path::Path, }; /// Returns the byte ranges of `file` between `start` and `end` that hold data. @@ -81,6 +82,17 @@ pub fn copy_range( Ok(()) } +/// Creates `dst` with `len` bytes and copies `len` bytes of `src` at `src_offset` into it. +/// Holes of `src` stay holes in `dst`. +pub fn copy_to_new_file(src: &File, src_offset: u64, dst: &Path, len: u64) -> Result<()> { + let dst = + File::create(dst).context(format!("copy_to_new_file: cannot create {}", dst.display()))?; + dst.set_len(len) + .context("copy_to_new_file: cannot set destination size")?; + + copy_range(src, src_offset, &dst, 0, len) +} + #[cfg(test)] mod tests { use crate::file::sparse::*; @@ -135,20 +147,17 @@ mod tests { } #[test] - fn copy_range_keeps_holes() { + fn copy_to_new_file_keeps_holes() { let dir = tempfile::tempdir().expect("create temp dir"); let src = sparse_file(dir.path()); - let dst = File::options() - .read(true) - .write(true) - .create_new(true) - .open(dir.path().join("dst")) - .expect("create file"); + let dst_path = dir.path().join("dst"); let len = (FILE_BLOCKS - DST_OFFSET_BLOCKS) * BLOCK_SIZE; - dst.set_len(len).expect("set size"); - copy_range(&src, DST_OFFSET_BLOCKS * BLOCK_SIZE, &dst, 0, len).expect("copy range"); + copy_to_new_file(&src, DST_OFFSET_BLOCKS * BLOCK_SIZE, &dst_path, len) + .expect("copy to new file"); + let dst = File::open(&dst_path).expect("open file"); + assert_eq!(dst.metadata().expect("metadata").len(), len); let data_start = (DATA_BLOCK - DST_OFFSET_BLOCKS) * BLOCK_SIZE; assert_eq!( data_ranges(&dst, 0, len).expect("data ranges"), diff --git a/src/lib.rs b/src/lib.rs index 263a4b4b..a6709365 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -167,11 +167,8 @@ fn copy_image(from: &Path, to: &Path) -> Result<()> { .metadata() .context("copy_image: cannot get source size")? .len(); - let dst = fs::File::create(to).context("copy_image: cannot create destination")?; - dst.set_len(len) - .context("copy_image: cannot set destination size")?; - file::sparse::copy_range(&src, 0, &dst, 0, len) + file::sparse::copy_to_new_file(&src, 0, to, len) } struct CertInfo { From c18fc17fb6c8a7e51790fb841733de9bb4961e22 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:16:59 +0100 Subject: [PATCH 31/46] fix(file): do not write the image back after copy-from-image copy-from-image only reads the image. Writing it back truncated the source first, so an error during the copy lost the source, and for a compressed image it left a decompressed copy next to it. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/cli.rs | 2 +- src/lib.rs | 98 ++++++++++++++++++++++++++------------ tests/integration_tests.rs | 23 +++++++++ 3 files changed, 92 insertions(+), 31 deletions(-) diff --git a/src/cli.rs b/src/cli.rs index 954171f6..25fb8505 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -8,7 +8,7 @@ use url::Url; const COPYRIGHT: &str = "Copyright © 2021 by conplement AG"; -#[derive(Args, Debug, Default)] +#[derive(Args, Debug)] pub struct ImageOutput { /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] diff --git a/src/lib.rs b/src/lib.rs index a6709365..18b66e7c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,8 +36,20 @@ use uuid::Uuid; use crate::file::compression; +/// A new directory `/tmp/{uuid}`, removed on drop. struct TempDirGuard(PathBuf); +impl TempDirGuard { + fn new() -> Result { + let tmp_dir = PathBuf::from(format!("/tmp/{}", Uuid::new_v4())); + fs::create_dir_all(&tmp_dir).context(format!( + "TempDirGuard::new: cannot create {}", + tmp_dir.display() + ))?; + Ok(Self(tmp_dir)) + } +} + impl Drop for TempDirGuard { fn drop(&mut self) { let Ok(rt) = tokio::runtime::Builder::new_current_thread() @@ -66,11 +78,7 @@ where compress_image: target_compression, } = output; - anyhow::ensure!( - image_file.try_exists().is_ok_and(|exists| exists), - "run_image_command: image doesn't exist {}", - image_file.to_str().context("cannot get image file path")? - ); + ensure_image_exists(&image_file)?; let input_bmap = if output_bmap { bmap::find_input_bmap(&image_file, input_bmap.as_deref())? @@ -81,32 +89,12 @@ where }; let mut dest_image_file = image_file.clone(); - // create /tmp/{uuid}/ and copy image into - let tmp_dir = PathBuf::from(format!("/tmp/{}", Uuid::new_v4())); - fs::create_dir_all(tmp_dir.clone()).context(format!( - "run_image_command: couldn't create destination path {}", - tmp_dir.to_str().context("cannot get tmp dir name")? - ))?; - - let _guard = TempDirGuard(tmp_dir.clone()); - - let mut tmp_image_file = tmp_dir.join( - image_file - .file_name() - .context("cannot get image file name")?, - ); - - // if applicable decompress image to *.wic - if let Some(source_compression) = Compression::from_file(&image_file)? { - std::fs::copy(&image_file, &tmp_image_file)?; - tmp_image_file = compression::decompress(&tmp_image_file, &source_compression)?; + let tmp_dir = TempDirGuard::new()?; + let source_compression = Compression::from_file(&image_file)?; + if source_compression.is_some() { dest_image_file.set_extension(""); - } else { - copy_image(&image_file, &tmp_image_file).context(format!( - "error: copy_image({:?}, {:?})", - image_file, tmp_image_file - ))?; } + let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; if let Some((path, bmap)) = &input_bmap { bmap::verify_checksums(&tmp_image_file, bmap) @@ -160,6 +148,56 @@ where Ok(()) } +/// Runs `command` on a temp copy of the image and writes nothing back. +fn run_read_only_image_command(image_file: PathBuf, command: F) -> Result<()> +where + F: FnOnce(&PathBuf) -> Result<()>, +{ + ensure_image_exists(&image_file)?; + + let tmp_dir = TempDirGuard::new()?; + let source_compression = Compression::from_file(&image_file)?; + let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; + + command(&tmp_image_file) +} + +fn ensure_image_exists(image_file: &Path) -> Result<()> { + anyhow::ensure!( + image_file.try_exists().is_ok_and(|exists| exists), + "image doesn't exist {}", + image_file.display() + ); + Ok(()) +} + +/// Copies `image_file` into `tmp_dir`, decompressed if it is compressed, and returns the copy. +fn temp_image( + image_file: &Path, + source_compression: Option<&Compression>, + tmp_dir: &Path, +) -> Result { + let tmp_image_file = tmp_dir.join( + image_file + .file_name() + .context("cannot get image file name")?, + ); + + match source_compression { + Some(c) => { + std::fs::copy(image_file, &tmp_image_file)?; + compression::decompress(&tmp_image_file, c) + } + None => { + copy_image(image_file, &tmp_image_file).context(format!( + "error: copy_image({:?}, {:?})", + image_file, tmp_image_file + ))?; + Ok(tmp_image_file) + } + } +} + /// Copies `from` to `to` and keeps the holes of a sparse `from`. fn copy_image(from: &Path, to: &Path) -> Result<()> { let src = fs::File::open(from).context("copy_image: cannot open source")?; @@ -486,7 +524,7 @@ pub fn run() -> Result<()> { Command::File(CopyFromImage { file_copy_params, image, - }) => run_image_command(image, ImageOutput::default(), |img: &PathBuf| { + }) => run_read_only_image_command(image, |img: &PathBuf| { file::copy_from_image(&file_copy_params, img) })?, } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 0f146f11..01e2b221 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1489,6 +1489,29 @@ fn check_image_decompression() { assert_eq!(image_path_wic_xz_hash1, image_path_wic_xz_hash2); } +#[test] +fn check_copy_from_image_writes_no_image() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_hash = Testrunner::file_hash(&image_path_wic_xz); + let out_file = tr.pathbuf().join("hosts"); + + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-from-image") + .arg("-f") + .arg(format!("rootA:/etc/hosts,{}", out_file.to_string_lossy())) + .arg("-i") + .arg(&image_path_wic_xz) + .assert() + .success(); + + assert!(out_file.exists()); + assert!(!image_path_wic_xz.with_extension("").exists()); + assert_eq!(Testrunner::file_hash(&image_path_wic_xz), image_hash); +} + #[tokio::test] async fn check_ssh_tunnel_setup() { let tr = Testrunner::new("check_ssh_tunnel_setup"); From c51066fd64a311aee1f04954965ceb808055b9d8 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:20:45 +0100 Subject: [PATCH 32/46] feat(file)!: fail on output name conflicts, add --output-image and --no-input-bmap omnect-cli derives the output image and bmap names from the input image. A command now fails instead of overwriting an existing file that is not the input image, replacing the input bmap of an image that stays (e.g. x.wic.bmap for x.wic.xz), or leaving a bmap next to the output image that no longer matches it. --output-image chooses the output path, and --no-input-bmap skips a bmap next to the image that does not match. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 10 +- src/cli.rs | 10 ++ src/file/bmap.rs | 5 +- src/file/compression.rs | 13 ++- src/file/mod.rs | 1 + src/file/output.rs | 182 +++++++++++++++++++++++++++++++++++++ src/lib.rs | 37 +++++--- tests/integration_tests.rs | 145 +++++++++++++++++++++++------ 8 files changed, 351 insertions(+), 52 deletions(-) create mode 100644 src/file/output.rs diff --git a/README.md b/README.md index 9f30eb97..c7e714c3 100644 --- a/README.md +++ b/README.md @@ -183,11 +183,17 @@ omnect-cli file copy-to-image --help - File permissions: inject `systemd-tmpfiles.d` - Wifi: inject `wpa_supplicant-wlan0.conf` +## Output image + +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression, e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. + +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. `copy-from-image` only reads the image and writes nothing. + ## bmap files -Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. +Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The command fails if this file is the input bmap of an image that stays, e.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`. Choose another output with `-o` then. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` changes the image but not the bmap next to it, so when you run several commands on one image, use `-b` on each of them, or pass the matching bmap with `--input-bmap`. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if a bmap is next to its output image, because the bmap would no longer match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/cli.rs b/src/cli.rs index 25fb8505..7b824c6f 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -16,6 +16,16 @@ pub struct ImageOutput { /// optional: bmap of the input image; default: .bmap or .bmap next to the image #[arg(long = "input-bmap", requires = "output_bmap")] pub input_bmap: Option, + /// optional: do not use a bmap next to the image, so the bmap maps the whole image + #[arg( + long = "no-input-bmap", + requires = "output_bmap", + conflicts_with = "input_bmap" + )] + pub no_input_bmap: bool, + /// optional: write the output image to this path; default: the input image, without the compression extension, plus the --pack-image extension + #[arg(short = 'o', long = "output-image")] + pub output_image: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') #[arg(short = 'p', long = "pack-image", value_enum)] pub compress_image: Option, diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 793a9fd5..96b13b5d 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -141,8 +141,7 @@ pub fn verify_checksums(image_file: &Path, bmap: &Bmap) -> Result<()> { anyhow::ensure!( actual.eq_ignore_ascii_case(expected.trim()), "verify_checksums: range {range:?} of the image does not match the bmap, so it is the \ - bmap of another image or of an older version of this image. Pass the right bmap \ - with --input-bmap." + bmap of another image or of an older version of this image" ); } @@ -509,7 +508,7 @@ mod tests { .expect("change data block"); let err = verify_checksums(&image, &bmap).expect_err("changed image accepted"); - assert!(err.to_string().contains("--input-bmap"), "{err}"); + assert!(err.to_string().contains("does not match the bmap"), "{err}"); } #[test] diff --git a/src/file/compression.rs b/src/file/compression.rs index 4c79da2f..85e8fc99 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -122,6 +122,13 @@ impl Compression { } } + /// `path` with the extension omnect-cli gives a compressed image appended. + pub fn compressed_path(&self, path: &Path) -> PathBuf { + let mut compressed = path.as_os_str().to_owned(); + compressed.push(format!(".{}", self.extension())); + PathBuf::from(compressed) + } + pub fn has_extension(path: &Path) -> bool { path.extension() .and_then(|e| e.to_str()) @@ -228,11 +235,7 @@ pub fn decompress(image_file_name: &PathBuf, compression: &Compression) -> Resul } pub fn compress(image_file_name: &PathBuf, compression: &Compression) -> Result { - let new_image_file = PathBuf::from(format!( - "{}.{}", - image_file_name.to_str().unwrap(), - compression.extension() - )); + let new_image_file = compression.compressed_path(image_file_name); let mut destination = File::create(&new_image_file)?; let mut source = File::open(image_file_name)?; debug!("compress {image_file_name:?} to {new_image_file:?}"); diff --git a/src/file/mod.rs b/src/file/mod.rs index f8999cef..92d9c677 100644 --- a/src/file/mod.rs +++ b/src/file/mod.rs @@ -1,6 +1,7 @@ pub mod bmap; pub mod compression; pub mod functions; +pub(crate) mod output; mod partition; pub(crate) mod sparse; diff --git a/src/file/output.rs b/src/file/output.rs new file mode 100644 index 00000000..4db32785 --- /dev/null +++ b/src/file/output.rs @@ -0,0 +1,182 @@ +use crate::file::{bmap, compression::Compression}; +use anyhow::{Context, Result}; +use std::{ + fs, + path::{Path, PathBuf}, +}; + +/// Returns `explicit`, else `image` without its last extension if it is compressed, with the +/// extension of `target` appended. +pub fn output_image( + image: &Path, + source_compressed: bool, + target: Option<&Compression>, + explicit: Option<&Path>, +) -> PathBuf { + if let Some(path) = explicit { + return path.to_path_buf(); + } + + let mut output = image.to_path_buf(); + if source_compressed { + output.set_extension(""); + } + match target { + Some(c) => c.compressed_path(&output), + None => output, + } +} + +/// Fails if writing `output_image` and its bmap would overwrite a file the user did not name, +/// replace the bmap of an image that stays, or leave a bmap that no longer matches the image. +pub fn check_conflicts( + input_image: &Path, + output_image: &Path, + explicit_output: bool, + output_bmap: bool, + input_bmap: Option<&Path>, +) -> Result<()> { + let in_place = same_file(input_image, output_image)?; + let target_bmap = bmap::bmap_path(output_image); + + anyhow::ensure!( + explicit_output || in_place || !output_image.exists(), + "{} exists; choose the output image with --output-image", + output_image.display() + ); + if output_bmap { + if let Some(input_bmap) = input_bmap { + anyhow::ensure!( + in_place || !same_file(input_bmap, &target_bmap)?, + "{} is the bmap of {} and would be replaced; choose the output image with \ + --output-image", + input_bmap.display(), + input_image.display() + ); + } + } else { + anyhow::ensure!( + !target_bmap.exists(), + "{} would no longer match {}; pass -b to update it, or remove it", + target_bmap.display(), + output_image.display() + ); + } + + Ok(()) +} + +fn same_file(a: &Path, b: &Path) -> Result { + Ok(resolve(a)? == resolve(b)?) +} + +/// The absolute path of `path` with symlinks resolved. A file that does not exist yet is +/// resolved through its directory. +fn resolve(path: &Path) -> Result { + if path.exists() { + return fs::canonicalize(path).context(format!("cannot resolve {}", path.display())); + } + let dir = match path.parent() { + Some(dir) if !dir.as_os_str().is_empty() => dir, + _ => Path::new("."), + }; + let name = path + .file_name() + .context(format!("{} has no file name", path.display()))?; + + Ok(fs::canonicalize(dir) + .context(format!("cannot resolve {}", dir.display()))? + .join(name)) +} + +#[cfg(test)] +mod tests { + use crate::file::output::*; + + const IMAGE: &str = "image.wic"; + const IMAGE_XZ: &str = "image.wic.xz"; + const OTHER_IMAGE: &str = "device.wic"; + const XZ_LEVEL: u32 = 9; + + fn touch(path: &Path) { + fs::write(path, []).expect("create file"); + } + + #[test] + fn output_image_follows_compression() { + let image = Path::new(IMAGE); + let image_xz = Path::new(IMAGE_XZ); + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + + assert_eq!(output_image(image, false, None, None), image); + assert_eq!(output_image(image_xz, true, None, None), image); + assert_eq!(output_image(image, false, Some(&xz), None), image_xz); + assert_eq!(output_image(image_xz, true, Some(&xz), None), image_xz); + assert_eq!( + output_image(image_xz, true, Some(&xz), Some(Path::new(OTHER_IMAGE))), + Path::new(OTHER_IMAGE) + ); + } + + #[test] + fn check_conflicts_allows_in_place_with_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + touch(&image); + touch(&image_bmap); + + check_conflicts(&image, &image, false, true, Some(&image_bmap)).expect("no conflict"); + } + + #[test] + fn check_conflicts_rejects_existing_derived_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + touch(&image_xz); + touch(&image); + + assert!(check_conflicts(&image_xz, &image, false, false, None).is_err()); + check_conflicts(&image_xz, &image, true, false, None).expect("explicit output"); + } + + #[test] + fn check_conflicts_rejects_replacing_the_input_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + touch(&image_xz); + touch(&image_bmap); + + assert!(check_conflicts(&image_xz, &image, false, true, Some(&image_bmap)).is_err()); + let other = dir.path().join(OTHER_IMAGE); + check_conflicts(&image_xz, &other, true, true, Some(&image_bmap)).expect("other output"); + } + + #[test] + fn check_conflicts_rejects_leaving_a_stale_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + touch(&bmap::bmap_path(&image)); + + assert!(check_conflicts(&image, &image, false, false, None).is_err()); + } + + #[test] + fn same_file_resolves_relative_and_missing_paths() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + let relative = dir.path().join(".").join(IMAGE); + let missing = dir.path().join(OTHER_IMAGE); + + assert!(same_file(&image, &relative).expect("same file")); + assert!(same_file(&missing, &missing).expect("same file")); + assert!(!same_file(&image, &missing).expect("same file")); + } +} diff --git a/src/lib.rs b/src/lib.rs index 18b66e7c..47ba4f31 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -75,30 +75,44 @@ where let ImageOutput { output_bmap, input_bmap, + no_input_bmap, + output_image, compress_image: target_compression, } = output; ensure_image_exists(&image_file)?; - let input_bmap = if output_bmap { + let input_bmap = if output_bmap && !no_input_bmap { bmap::find_input_bmap(&image_file, input_bmap.as_deref())? .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) .transpose()? } else { None }; - let mut dest_image_file = image_file.clone(); + let source_compression = Compression::from_file(&image_file)?; + let dest_image_file = file::output::output_image( + &image_file, + source_compression.is_some(), + target_compression.as_ref(), + output_image.as_deref(), + ); + file::output::check_conflicts( + &image_file, + &dest_image_file, + output_image.is_some(), + output_bmap, + input_bmap.as_ref().map(|(path, _)| path.as_path()), + )?; let tmp_dir = TempDirGuard::new()?; - let source_compression = Compression::from_file(&image_file)?; - if source_compression.is_some() { - dest_image_file.set_extension(""); - } let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; if let Some((path, bmap)) = &input_bmap { - bmap::verify_checksums(&tmp_image_file, bmap) - .context(format!("cannot use input bmap {}", path.display()))?; + bmap::verify_checksums(&tmp_image_file, bmap).context(format!( + "cannot use input bmap {}; pass the right one with --input-bmap, or map the \ + whole image with --no-input-bmap", + path.display() + ))?; } // run command @@ -108,7 +122,7 @@ where if output_bmap { match &input_bmap { Some((path, _)) => info!("the bmap maps the ranges of {}", path.display()), - None => info!("no input bmap found, so the bmap maps the whole image"), + None => info!("no input bmap, so the bmap maps the whole image"), } bmap::write_bmap( &tmp_image_file, @@ -120,11 +134,6 @@ where // if applicable compress image if let Some(c) = target_compression { tmp_image_file = compression::compress(&tmp_image_file, &c)?; - dest_image_file.set_file_name( - tmp_image_file - .file_name() - .context("cannot get image file name")?, - ); std::fs::copy(&tmp_image_file, &dest_image_file).context(format!( "error: std::fs::copy({:?}, {:?})", tmp_image_file, dest_image_file diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 01e2b221..4510c54e 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -6,6 +6,7 @@ use httpmock::prelude::*; use omnect_cli::{file::bmap::Bmap, ssh}; use predicates::prelude::*; use std::{ + ffi::OsStr, fs::{self, create_dir_all}, os::unix::fs::MetadataExt, path::{Path, PathBuf}, @@ -909,6 +910,10 @@ const ZERO_BLOCKS_FILE_ZEROS: usize = 64 * 1024; const OTHER_IMAGE_BMAP: &str = "40964096\ 0"; +fn bmap_path_of(image: &Path) -> PathBuf { + PathBuf::from(format!("{}.bmap", image.to_string_lossy())) +} + fn read_bmap(path: &Path) -> Bmap { Bmap::read(path).expect("read bmap") } @@ -1122,11 +1127,8 @@ fn check_file_copy_without_output_bmap_keeps_image_sparse() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_path = tr.to_pathbuf("testfiles/image.wic"); let sparse_path = image_path.with_file_name("sparse.wic"); - let bmap_path = PathBuf::from(format!("{}.bmap", sparse_path.to_string_lossy())); let in_file = tr.to_pathbuf("testfiles/boot.scr"); sparse_copy(&image_path, &sparse_path); - // without -b, a bmap next to the image is not used - fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); Command::cargo_bin("omnect-cli") .expect("omnect-cli binary") @@ -1144,10 +1146,6 @@ fn check_file_copy_without_output_bmap_keeps_image_sparse() { meta.blocks() * STAT_BLOCK_SIZE < meta.len(), "image is not sparse" ); - assert_eq!( - fs::read_to_string(&bmap_path).expect("read bmap"), - OTHER_IMAGE_BMAP - ); } #[test] @@ -1157,7 +1155,7 @@ fn check_no_output_bmap_when_image_write_back_fails() { let written_image_path = image_path.with_extension(""); let bmap_path = image_path.with_extension("bmap"); let in_file = tr.to_pathbuf("testfiles/boot.scr"); - // a directory in place of the written image makes the write back fail, also for root + // a directory as the output image makes the write back fail, also for root create_dir_all(&written_image_path).expect("create directory"); Command::cargo_bin("omnect-cli") @@ -1169,8 +1167,11 @@ fn check_no_output_bmap_when_image_write_back_fails() { .arg("-i") .arg(&image_path) .arg("-b") + .arg("-o") + .arg(&written_image_path) .assert() - .failure(); + .failure() + .stderr(predicate::str::contains("copy_image")); assert!( !bmap_path.exists(), "bmap written for an image that was not written back" @@ -1232,33 +1233,120 @@ fn check_input_bmap_of_another_image_with_the_same_size() { #[test] fn check_chained_commands_with_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_xz_path = tr.to_pathbuf("testfiles/image.wic.xz"); + let image_xz_hash = Testrunner::file_hash(&image_xz_path); + let input_bmap_path = tr.to_pathbuf("testfiles/image.wic.bmap"); + let input_bmap_hash = Testrunner::file_hash(&input_bmap_path); + let device_path = tr.pathbuf().join("device.wic"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let copy_to_image = |image: &Path, args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(image) + .args(args) + .assert() + }; + + // image.wic would be written next to image.wic.bmap without updating it + copy_to_image(&image_xz_path, &[]) + .failure() + .stderr(predicate::str::contains("pass -b to update it")); + // the bmap of image.wic would replace the bmap of image.wic.xz + copy_to_image(&image_xz_path, &["-b".as_ref()]) + .failure() + .stderr(predicate::str::contains("--output-image")); + + copy_to_image( + &image_xz_path, + &["-b".as_ref(), "-o".as_ref(), device_path.as_os_str()], + ) + .success(); + copy_to_image(&device_path, &["-b".as_ref()]).success(); + copy_to_image(&device_path, &[]) + .failure() + .stderr(predicate::str::contains("pass -b to update it")); + + assert_eq!(Testrunner::file_hash(&image_xz_path), image_xz_hash); + assert_eq!(Testrunner::file_hash(&input_bmap_path), input_bmap_hash); + assert!(maps_all_ranges_of( + &read_bmap(&bmap_path_of(&device_path)), + &read_bmap(&input_bmap_path) + )); +} + +#[test] +fn check_derived_output_image_exists() { let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); let image_xz_path = tr.to_pathbuf("testfiles/image.wic.xz"); let image_path = image_xz_path.with_extension(""); - tr.to_pathbuf("testfiles/image.wic.bmap"); let in_file = tr.to_pathbuf("testfiles/boot.scr"); - let copy_to_image = |image: &Path, output_bmap: bool| { - let mut cmd = Command::cargo_bin("omnect-cli").expect("omnect-cli binary"); - cmd.arg("file") + fs::write(&image_path, "an older image").expect("write image"); + let copy_to_image = |args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg("copy-to-image") + .arg("-f") + .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) + .arg("-i") + .arg(&image_xz_path) + .args(args) + .assert() + }; + + copy_to_image(&[]) + .failure() + .stderr(predicate::str::contains("--output-image")); + // a path given with -o is overwritten + copy_to_image(&["-o".as_ref(), image_path.as_os_str()]).success(); + assert_ne!( + fs::read(&image_path).expect("read image"), + b"an older image" + ); +} + +#[test] +fn check_no_input_bmap() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + let image_path = tr.to_pathbuf("testfiles/image.wic"); + let bmap_path = bmap_path_of(&image_path); + let other_bmap_path = tr.pathbuf().join("other.bmap"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + fs::write(&bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + fs::write(&other_bmap_path, OTHER_IMAGE_BMAP).expect("write bmap"); + let copy_to_image = |args: &[&OsStr]| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") .arg("copy-to-image") .arg("-f") .arg(format!("{},boot:/my-file", in_file.to_string_lossy())) .arg("-i") - .arg(image); - if output_bmap { - cmd.arg("-b"); - } - cmd.assert() + .arg(&image_path) + .args(args) + .assert() }; - // without -b, the bmap next to the image still has the checksums of the original image - copy_to_image(&image_xz_path, false).success(); - copy_to_image(&image_path, true) + copy_to_image(&["--no-input-bmap".as_ref()]).failure(); + copy_to_image(&[ + "-b".as_ref(), + "--no-input-bmap".as_ref(), + "--input-bmap".as_ref(), + other_bmap_path.as_os_str(), + ]) + .failure(); + copy_to_image(&["-b".as_ref()]) .failure() - .stderr(predicate::str::contains("older version of this image")); + .stderr(predicate::str::contains("--no-input-bmap")); - copy_to_image(&image_xz_path, true).success(); - copy_to_image(&image_path, true).success(); + copy_to_image(&["-b".as_ref(), "--no-input-bmap".as_ref()]).success(); + assert!(maps_whole_image(&read_bmap(&bmap_path))); } #[test] @@ -1313,11 +1401,10 @@ fn check_bmap_generation_wic() { fn check_bmap_generation_wic_xz() { let tr = Testrunner::new(function_name!().split("::").last().unwrap()); let image_path_wic_xz = tr.to_pathbuf("testfiles/image.wic.xz"); - let image_path_wic = image_path_wic_xz.with_extension(""); + let image_path_wic = tr.pathbuf().join("device.wic"); let image_path_wic_copy = image_path_wic_xz.with_extension("copy"); - // the Yocto bmap of the image; omnect-cli replaces it with the one it creates - let image_path_bmap = tr.to_pathbuf("testfiles/image.wic.bmap"); - let input_bmap = read_bmap(&image_path_bmap); + let image_path_bmap = bmap_path_of(&image_path_wic); + let input_bmap = read_bmap(&tr.to_pathbuf("testfiles/image.wic.bmap")); let in_file = tr.to_pathbuf("testfiles/boot.scr"); let in_file = in_file.to_str().unwrap(); @@ -1330,6 +1417,8 @@ fn check_bmap_generation_wic_xz() { .arg("-i") .arg(&image_path_wic_xz) .arg("-b") + .arg("-o") + .arg(&image_path_wic) .assert(); assert.success(); From f0ead2173ab39b4b0175fe40fe6ca8062172014c Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 15:21:14 +0100 Subject: [PATCH 33/46] perf(file): check the input bmap of an uncompressed image before the copy Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/lib.rs | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 47ba4f31..515a5fdf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -104,15 +104,25 @@ where input_bmap.as_ref().map(|(path, _)| path.as_path()), )?; - let tmp_dir = TempDirGuard::new()?; - let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; - - if let Some((path, bmap)) = &input_bmap { - bmap::verify_checksums(&tmp_image_file, bmap).context(format!( + let verify_input_bmap = |image: &Path| -> Result<()> { + let Some((path, bmap)) = &input_bmap else { + return Ok(()); + }; + bmap::verify_checksums(image, bmap).context(format!( "cannot use input bmap {}; pass the right one with --input-bmap, or map the \ whole image with --no-input-bmap", path.display() - ))?; + )) + }; + + // a wrong bmap of an uncompressed image fails before the copy + if source_compression.is_none() { + verify_input_bmap(&image_file)?; + } + let tmp_dir = TempDirGuard::new()?; + let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; + if source_compression.is_some() { + verify_input_bmap(&tmp_image_file)?; } // run command From be84b3386916cd9c31b9e930512dc9a05c3708ec Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:57:04 +0100 Subject: [PATCH 34/46] fix(file): read concatenated compressed streams Decompress with the multi-stream decoders, so an image made of several streams (e.g. written by pbzip2) is read completely. Decompress straight from the input image into the temp dir, without a copy of the compressed file first. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/compression.rs | 100 ++++++++++++++++++++++++++-------------- src/lib.rs | 16 +++---- 2 files changed, 71 insertions(+), 45 deletions(-) diff --git a/src/file/compression.rs b/src/file/compression.rs index 85e8fc99..261506ac 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -1,11 +1,11 @@ -use crate::file::{IMAGE_BLOCK_SIZE, is_zero}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, is_zero}; use anyhow::{Context, Result}; use filemagic::Magic; use log::debug; use std::{ env, fs::File, - io::{Seek, SeekFrom, Write}, + io::{BufWriter, IntoInnerError, Read, Seek, SeekFrom, Write}, os::unix::fs::FileExt, path::{Path, PathBuf}, str::FromStr, @@ -81,24 +81,19 @@ impl Compression { source: &mut std::fs::File, destination: &mut std::fs::File, ) -> std::io::Result { - let sparse = SparseWriter::new(destination)?; - // only `finish` detects a truncated or corrupt stream - let (bytes_read, sparse) = match &self { - Compression::bzip2 => { - let mut dec = bzip2::write::BzDecoder::new(sparse); - (std::io::copy(source, &mut dec)?, dec.finish()?) - } - Compression::gzip => { - let mut dec = flate2::write::GzDecoder::new(sparse); - (std::io::copy(source, &mut dec)?, dec.finish()?) - } - Compression::xz { .. } => { - let mut dec = xz2::write::XzDecoder::new(sparse); - (std::io::copy(source, &mut dec)?, dec.finish()?) - } + // the multi decoders also read concatenated streams, as pbzip2 writes them + let mut decoder: Box = match &self { + Compression::bzip2 => Box::new(bzip2::read::MultiBzDecoder::new(source)), + Compression::gzip => Box::new(flate2::read::MultiGzDecoder::new(source)), + Compression::xz { .. } => Box::new(xz2::read::XzDecoder::new_multi_decoder(source)), }; - sparse.finish()?; - Ok(bytes_read) + let mut sparse = BufWriter::with_capacity(COPY_CHUNK_SIZE, SparseWriter::new(destination)?); + let bytes_written = std::io::copy(&mut decoder, &mut sparse)?; + sparse + .into_inner() + .map_err(IntoInnerError::into_error)? + .finish()?; + Ok(bytes_written) } fn marker(&self) -> &'static str { @@ -208,26 +203,35 @@ impl Write for SparseWriter<'_> { } } -pub fn decompress(image_file_name: &PathBuf, compression: &Compression) -> Result { - let mut new_image_file = PathBuf::from(image_file_name); - - if new_image_file - .extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| compression.extensions().contains(&ext)) - { - new_image_file.set_extension(""); - } - // creating the destination would truncate the source +/// Decompresses `image_file_name` into `dest_dir`, named without the compression extension. +pub fn decompress( + image_file_name: &Path, + compression: &Compression, + dest_dir: &Path, +) -> Result { anyhow::ensure!( - new_image_file != *image_file_name, + image_file_name + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| compression.extensions().contains(&ext)), "decompress: {} has no {:?} file extension", image_file_name.display(), compression.extensions() ); + let new_image_file = dest_dir.join( + image_file_name + .file_stem() + .context("decompress: cannot get image file name")?, + ); - let mut destination = File::create(&new_image_file)?; - let mut source = File::open(image_file_name)?; + let mut source = File::open(image_file_name).context(format!( + "decompress: cannot open {}", + image_file_name.display() + ))?; + let mut destination = File::create(&new_image_file).context(format!( + "decompress: cannot create {}", + new_image_file.display() + ))?; debug!("decompress {image_file_name:?} to {new_image_file:?}"); let bytes_written = compression.decompress(&mut source, &mut destination)?; debug!("image::decompress: copied {} bytes.", bytes_written); @@ -363,6 +367,32 @@ mod tests { } } + #[test] + fn decompress_reads_concatenated_streams() { + let image = test_image(); + let (first, second) = image.split_at(DATA_OFFSET); + + for compression in all_compressions() { + let mut concatenated = tempfile::tempfile().expect("create concatenated"); + for part in [first, second] { + let mut compressed = compress_image(&compression, part); + concatenated + .write_all(&read_all(&mut compressed)) + .expect("write stream"); + } + concatenated.rewind().expect("rewind concatenated"); + + let mut decompressed = tempfile::tempfile().expect("create decompressed"); + compression + .decompress(&mut concatenated, &mut decompressed) + .expect("decompress"); + assert!( + read_all(&mut decompressed) == image, + "{compression:?}: content differs" + ); + } + } + #[test] fn decompress_file_strips_every_extension_of_the_compression() { let dir = tempfile::tempdir().expect("create temp dir"); @@ -374,7 +404,7 @@ mod tests { let mut compressed = compress_image(&compression, &image); fs::write(&path, read_all(&mut compressed)).expect("write compressed"); - let decompressed = decompress(&path, &compression).expect("decompress"); + let decompressed = decompress(&path, &compression, dir.path()).expect("decompress"); assert_eq!(decompressed, dir.path().join("image.wic")); assert!( fs::read(&decompressed).expect("read decompressed") == image, @@ -394,7 +424,7 @@ mod tests { let content = read_all(&mut compressed); fs::write(&path, &content).expect("write compressed"); - assert!(decompress(&path, &compression).is_err()); + assert!(decompress(&path, &compression, dir.path()).is_err()); assert!( fs::read(&path).expect("read compressed") == content, "source was changed" diff --git a/src/lib.rs b/src/lib.rs index 515a5fdf..a2ce84d7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -196,18 +196,14 @@ fn temp_image( source_compression: Option<&Compression>, tmp_dir: &Path, ) -> Result { - let tmp_image_file = tmp_dir.join( - image_file - .file_name() - .context("cannot get image file name")?, - ); - match source_compression { - Some(c) => { - std::fs::copy(image_file, &tmp_image_file)?; - compression::decompress(&tmp_image_file, c) - } + Some(c) => compression::decompress(image_file, c, tmp_dir), None => { + let tmp_image_file = tmp_dir.join( + image_file + .file_name() + .context("cannot get image file name")?, + ); copy_image(image_file, &tmp_image_file).context(format!( "error: copy_image({:?}, {:?})", image_file, tmp_image_file From 453b1245e6aec59c6b979aeca2697b9ec749c81f Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 16:57:04 +0100 Subject: [PATCH 35/46] fix(file): check every bmap that bmaptool would use Without -b, also fail when the bmap of the uncompressed image is next to a compressed output image, because bmaptool falls back to it. With -b and an output that is not the input image, also refuse to replace the bmap next to the input image when --input-bmap or --no-input-bmap is given. An input bmap that cannot be read gets the same hint as one that does not match. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 4 +-- src/file/output.rs | 62 +++++++++++++++++++++++++++++--------- src/lib.rs | 42 +++++++++++++++----------- tests/integration_tests.rs | 11 +++++-- 4 files changed, 83 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index c7e714c3..e914d157 100644 --- a/README.md +++ b/README.md @@ -191,9 +191,9 @@ The command fails instead of overwriting an existing file that is not the input ## bmap files -Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The command fails if this file is the input bmap of an image that stays, e.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`. Choose another output with `-o` then. +Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The command fails if this file is the bmap of an image that stays, e.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`, also with `--input-bmap` or `--no-input-bmap`. Choose another output with `-o` then. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if a bmap is next to its output image, because the bmap would no longer match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if `.bmap` exists, or for a compressed output image the bmap of the uncompressed image (e.g. `my-image.wic.bmap` for `my-image.wic.xz`), because `bmaptool` would use it and it would not match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/file/output.rs b/src/file/output.rs index 4db32785..ea7ffae0 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -28,7 +28,9 @@ pub fn output_image( } /// Fails if writing `output_image` and its bmap would overwrite a file the user did not name, -/// replace the bmap of an image that stays, or leave a bmap that no longer matches the image. +/// replace the bmap of an image that stays, or leave a bmap next to the output image that does +/// not match it: `.bmap`, or for a compressed output image the bmap of the +/// uncompressed image, which `bmaptool` also uses. pub fn check_conflicts( input_image: &Path, output_image: &Path, @@ -37,7 +39,6 @@ pub fn check_conflicts( input_bmap: Option<&Path>, ) -> Result<()> { let in_place = same_file(input_image, output_image)?; - let target_bmap = bmap::bmap_path(output_image); anyhow::ensure!( explicit_output || in_place || !output_image.exists(), @@ -45,20 +46,23 @@ pub fn check_conflicts( output_image.display() ); if output_bmap { - if let Some(input_bmap) = input_bmap { - anyhow::ensure!( - in_place || !same_file(input_bmap, &target_bmap)?, - "{} is the bmap of {} and would be replaced; choose the output image with \ - --output-image", - input_bmap.display(), - input_image.display() - ); + let target_bmap = bmap::bmap_path(output_image); + if !in_place { + let found_bmap = bmap::find_input_bmap(input_image, None)?; + for bmap in input_bmap.into_iter().chain(found_bmap.as_deref()) { + anyhow::ensure!( + !same_file(bmap, &target_bmap)?, + "{} is the bmap of {} and would be replaced; choose another output image \ + with --output-image", + bmap.display(), + input_image.display() + ); + } } - } else { - anyhow::ensure!( - !target_bmap.exists(), - "{} would no longer match {}; pass -b to update it, or remove it", - target_bmap.display(), + } else if let Some(stale_bmap) = bmap::find_input_bmap(output_image, None)? { + anyhow::bail!( + "{} would not match {}; pass -b to write a matching bmap", + stale_bmap.display(), output_image.display() ); } @@ -167,6 +171,34 @@ mod tests { assert!(check_conflicts(&image, &image, false, false, None).is_err()); } + #[test] + fn check_conflicts_rejects_a_stale_bmap_of_the_uncompressed_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image_bmap = bmap::bmap_path(&dir.path().join(IMAGE)); + touch(&image_xz); + touch(&image_bmap); + + // bmaptool uses image.wic.bmap for image.wic.xz + assert!(check_conflicts(&image_xz, &image_xz, false, false, None).is_err()); + check_conflicts(&image_xz, &image_xz, false, true, Some(&image_bmap)).expect("with -b"); + } + + #[test] + fn check_conflicts_rejects_replacing_the_bmap_next_to_the_input_image() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image_xz = dir.path().join(IMAGE_XZ); + let image = dir.path().join(IMAGE); + let other_bmap = dir.path().join("other.bmap"); + touch(&image_xz); + touch(&bmap::bmap_path(&image)); + touch(&other_bmap); + + // --input-bmap and --no-input-bmap + assert!(check_conflicts(&image_xz, &image, false, true, Some(&other_bmap)).is_err()); + assert!(check_conflicts(&image_xz, &image, false, true, None).is_err()); + } + #[test] fn same_file_resolves_relative_and_missing_paths() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/lib.rs b/src/lib.rs index a2ce84d7..5711540c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,8 +23,10 @@ use cli::{ }; use file::{ bmap::{self, Bmap}, - compression::Compression, + compression::{self, Compression}, functions::FileCopyToParams, + output::{check_conflicts, output_image}, + sparse, }; use log::{error, info}; use std::{ @@ -34,8 +36,6 @@ use std::{ use tokio::fs::remove_dir_all; use uuid::Uuid; -use crate::file::compression; - /// A new directory `/tmp/{uuid}`, removed on drop. struct TempDirGuard(PathBuf); @@ -76,7 +76,7 @@ where output_bmap, input_bmap, no_input_bmap, - output_image, + output_image: explicit_output, compress_image: target_compression, } = output; @@ -84,22 +84,26 @@ where let input_bmap = if output_bmap && !no_input_bmap { bmap::find_input_bmap(&image_file, input_bmap.as_deref())? - .map(|path| Bmap::read(&path).map(|bmap| (path, bmap))) + .map(|path| { + Bmap::read(&path) + .with_context(|| input_bmap_error(&path)) + .map(|bmap| (path, bmap)) + }) .transpose()? } else { None }; let source_compression = Compression::from_file(&image_file)?; - let dest_image_file = file::output::output_image( + let dest_image_file = output_image( &image_file, source_compression.is_some(), target_compression.as_ref(), - output_image.as_deref(), + explicit_output.as_deref(), ); - file::output::check_conflicts( + check_conflicts( &image_file, &dest_image_file, - output_image.is_some(), + explicit_output.is_some(), output_bmap, input_bmap.as_ref().map(|(path, _)| path.as_path()), )?; @@ -108,11 +112,7 @@ where let Some((path, bmap)) = &input_bmap else { return Ok(()); }; - bmap::verify_checksums(image, bmap).context(format!( - "cannot use input bmap {}; pass the right one with --input-bmap, or map the \ - whole image with --no-input-bmap", - path.display() - )) + bmap::verify_checksums(image, bmap).with_context(|| input_bmap_error(path)) }; // a wrong bmap of an uncompressed image fails before the copy @@ -144,7 +144,7 @@ where // if applicable compress image if let Some(c) = target_compression { tmp_image_file = compression::compress(&tmp_image_file, &c)?; - std::fs::copy(&tmp_image_file, &dest_image_file).context(format!( + fs::copy(&tmp_image_file, &dest_image_file).context(format!( "error: std::fs::copy({:?}, {:?})", tmp_image_file, dest_image_file ))?; @@ -158,7 +158,7 @@ where // copy the bmap last, so it never describes an image that was not written back if output_bmap { let target_bmap = bmap::bmap_path(&dest_image_file); - std::fs::copy(&tmp_bmap, &target_bmap).context(format!( + fs::copy(&tmp_bmap, &target_bmap).context(format!( "error: std::fs::copy({:?}, {:?})", tmp_bmap, target_bmap ))?; @@ -167,6 +167,14 @@ where Ok(()) } +fn input_bmap_error(path: &Path) -> String { + format!( + "cannot use input bmap {}; pass the right one with --input-bmap, or map the whole \ + image with --no-input-bmap", + path.display() + ) +} + /// Runs `command` on a temp copy of the image and writes nothing back. fn run_read_only_image_command(image_file: PathBuf, command: F) -> Result<()> where @@ -221,7 +229,7 @@ fn copy_image(from: &Path, to: &Path) -> Result<()> { .context("copy_image: cannot get source size")? .len(); - file::sparse::copy_to_new_file(&src, 0, to, len) + sparse::copy_to_new_file(&src, 0, to, len) } struct CertInfo { diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 4510c54e..2802a1ae 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1256,11 +1256,18 @@ fn check_chained_commands_with_input_bmap() { // image.wic would be written next to image.wic.bmap without updating it copy_to_image(&image_xz_path, &[]) .failure() - .stderr(predicate::str::contains("pass -b to update it")); + .stderr(predicate::str::contains("pass -b to write a matching bmap")); + // bmaptool would use image.wic.bmap for the changed image.wic.xz + copy_to_image(&image_xz_path, &["-p".as_ref(), "xz".as_ref()]) + .failure() + .stderr(predicate::str::contains("pass -b to write a matching bmap")); // the bmap of image.wic would replace the bmap of image.wic.xz copy_to_image(&image_xz_path, &["-b".as_ref()]) .failure() .stderr(predicate::str::contains("--output-image")); + copy_to_image(&image_xz_path, &["-b".as_ref(), "--no-input-bmap".as_ref()]) + .failure() + .stderr(predicate::str::contains("--output-image")); copy_to_image( &image_xz_path, @@ -1270,7 +1277,7 @@ fn check_chained_commands_with_input_bmap() { copy_to_image(&device_path, &["-b".as_ref()]).success(); copy_to_image(&device_path, &[]) .failure() - .stderr(predicate::str::contains("pass -b to update it")); + .stderr(predicate::str::contains("pass -b to write a matching bmap")); assert_eq!(Testrunner::file_hash(&image_xz_path), image_xz_hash); assert_eq!(Testrunner::file_hash(&input_bmap_path), input_bmap_hash); From d0800f83bcaab653af587f1e6e1172ecdf809faa Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:00:08 +0100 Subject: [PATCH 36/46] fix(file)!: name a bzip2 image .bz2 bmaptool does not read the .bzip2 extension, so it could not flash an image written with -p bzip2. BREAKING CHANGE: -p bzip2 writes .bz2 instead of .bzip2. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/compression.rs | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e914d157..62103058 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ omnect-cli file copy-to-image --help ## Output image -A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression, e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. `copy-from-image` only reads the image and writes nothing. diff --git a/src/file/compression.rs b/src/file/compression.rs index 261506ac..5f37efcb 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -111,7 +111,7 @@ impl Compression { /// The first one is used for a compressed image that omnect-cli creates. fn extensions(&self) -> &'static [&'static str] { match &self { - Compression::bzip2 => &["bzip2", "bz2"], + Compression::bzip2 => &["bz2", "bzip2"], Compression::gzip => &["gzip", "gz"], Compression::xz { .. } => &["xz"], } @@ -415,6 +415,16 @@ mod tests { } } + #[test] + fn compressed_path_has_an_extension_bmaptool_reads() { + let image = Path::new("image.wic"); + let expected = ["image.wic.bz2", "image.wic.gzip", "image.wic.xz"]; + + for (compression, expected) in all_compressions().iter().zip(expected) { + assert_eq!(compression.compressed_path(image), Path::new(expected)); + } + } + #[test] fn decompress_file_rejects_unknown_extension() { let dir = tempfile::tempdir().expect("create temp dir"); From c477f9a2d0c4c9a83f2133c69519018caab883ed Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:01:11 +0100 Subject: [PATCH 37/46] fix(file): check the extension of the --output-image name The name given with -o must have the extension of the -p compression, or no compression extension without -p, so the name matches the content. For a compressed input, strip only the extension of its compression from the derived output name. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/compression.rs | 13 +++-- src/file/output.rs | 110 ++++++++++++++++++++++++++++++++----- src/lib.rs | 4 +- tests/integration_tests.rs | 10 ++++ 5 files changed, 117 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 62103058..e8a87336 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ omnect-cli file copy-to-image --help ## Output image -A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. `copy-from-image` only reads the image and writes nothing. diff --git a/src/file/compression.rs b/src/file/compression.rs index 5f37efcb..cf3f173d 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -124,10 +124,14 @@ impl Compression { PathBuf::from(compressed) } - pub fn has_extension(path: &Path) -> bool { + pub fn matches_extension(&self, path: &Path) -> bool { path.extension() .and_then(|e| e.to_str()) - .is_some_and(|e| Compression::iter().any(|c| c.extensions().contains(&e))) + .is_some_and(|e| self.extensions().contains(&e)) + } + + pub fn has_extension(path: &Path) -> bool { + Compression::iter().any(|c| c.matches_extension(path)) } pub fn from_file(image_file_name: &PathBuf) -> Result> { @@ -210,10 +214,7 @@ pub fn decompress( dest_dir: &Path, ) -> Result { anyhow::ensure!( - image_file_name - .extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| compression.extensions().contains(&ext)), + compression.matches_extension(image_file_name), "decompress: {} has no {:?} file extension", image_file_name.display(), compression.extensions() diff --git a/src/file/output.rs b/src/file/output.rs index ea7ffae0..953173d0 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -5,26 +5,40 @@ use std::{ path::{Path, PathBuf}, }; -/// Returns `explicit`, else `image` without its last extension if it is compressed, with the -/// extension of `target` appended. +/// Returns `explicit`, else `image` without the extension of its `source` compression, with the +/// extension of `target` appended. Fails if `explicit` has no extension of `target`, or a +/// compression extension while `target` is none. pub fn output_image( image: &Path, - source_compressed: bool, + source: Option<&Compression>, target: Option<&Compression>, explicit: Option<&Path>, -) -> PathBuf { +) -> Result { if let Some(path) = explicit { - return path.to_path_buf(); + match target { + Some(c) => anyhow::ensure!( + c.matches_extension(path), + "{} has no extension of the -p compression; name it e.g. {}", + path.display(), + c.compressed_path(path).display() + ), + None => anyhow::ensure!( + !Compression::has_extension(path), + "{} has a compression extension, but the image is written uncompressed; pass -p", + path.display() + ), + } + return Ok(path.to_path_buf()); } let mut output = image.to_path_buf(); - if source_compressed { + if source.is_some_and(|c| c.matches_extension(image)) { output.set_extension(""); } - match target { + Ok(match target { Some(c) => c.compressed_path(&output), None => output, - } + }) } /// Fails if writing `output_image` and its bmap would overwrite a file the user did not name, @@ -113,15 +127,85 @@ mod tests { let xz = Compression::xz { compression_level: XZ_LEVEL, }; + let output = |image, source, target| { + output_image(image, source, target, None).expect("output image") + }; + + assert_eq!(output(image, None, None), image); + assert_eq!(output(image_xz, Some(&xz), None), image); + assert_eq!(output(image, None, Some(&xz)), image_xz); + assert_eq!(output(image_xz, Some(&xz), Some(&xz)), image_xz); + } + + #[test] + fn output_image_strips_only_the_source_compression_extension() { + let image = Path::new("image.bin"); + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; - assert_eq!(output_image(image, false, None, None), image); - assert_eq!(output_image(image_xz, true, None, None), image); - assert_eq!(output_image(image, false, Some(&xz), None), image_xz); - assert_eq!(output_image(image_xz, true, Some(&xz), None), image_xz); assert_eq!( - output_image(image_xz, true, Some(&xz), Some(Path::new(OTHER_IMAGE))), + output_image(image, Some(&xz), None, None).expect("output image"), + image + ); + } + + #[test] + fn output_image_takes_explicit_path_with_matching_extension() { + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + let other_xz = Path::new("device.wic.xz"); + + assert_eq!( + output_image( + Path::new(IMAGE_XZ), + Some(&xz), + None, + Some(Path::new(OTHER_IMAGE)) + ) + .expect("uncompressed"), Path::new(OTHER_IMAGE) ); + assert_eq!( + output_image(Path::new(IMAGE), None, Some(&xz), Some(other_xz)).expect("compressed"), + other_xz + ); + } + + #[test] + fn output_image_rejects_explicit_path_with_wrong_extension() { + let xz = Compression::xz { + compression_level: XZ_LEVEL, + }; + + assert!( + output_image( + Path::new(IMAGE), + None, + Some(&xz), + Some(Path::new(OTHER_IMAGE)) + ) + .is_err() + ); + assert!( + output_image( + Path::new(IMAGE), + None, + Some(&xz), + Some(Path::new("device.wic.gz")) + ) + .is_err() + ); + assert!( + output_image( + Path::new(IMAGE_XZ), + Some(&xz), + None, + Some(Path::new("device.wic.xz")) + ) + .is_err() + ); } #[test] diff --git a/src/lib.rs b/src/lib.rs index 5711540c..f4eb0168 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -96,10 +96,10 @@ where let source_compression = Compression::from_file(&image_file)?; let dest_image_file = output_image( &image_file, - source_compression.is_some(), + source_compression.as_ref(), target_compression.as_ref(), explicit_output.as_deref(), - ); + )?; check_conflicts( &image_file, &dest_image_file, diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 2802a1ae..9aeb30a9 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1310,6 +1310,16 @@ fn check_derived_output_image_exists() { copy_to_image(&[]) .failure() .stderr(predicate::str::contains("--output-image")); + copy_to_image(&[ + "-o".as_ref(), + image_path.as_os_str(), + "-p".as_ref(), + "xz".as_ref(), + ]) + .failure() + .stderr(predicate::str::contains( + "has no extension of the -p compression", + )); // a path given with -o is overwritten copy_to_image(&["-o".as_ref(), image_path.as_os_str()]).success(); assert_ne!( From 7ab73f8ddf7a13ef3255196e16f3fef244e04966 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:02:09 +0100 Subject: [PATCH 38/46] fix(file): write the output image through a rename The write-back truncated the output image first, so a failed copy, e.g. on a full disk, destroyed an image that was changed in place. Write the image and the bmap to a new file in the same directory and rename it over the old one. A symlink is followed, and the old file keeps its permissions. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/output.rs | 76 ++++++++++++++++++++++++++++++++++++++ src/lib.rs | 29 ++++++++------- tests/integration_tests.rs | 2 +- 4 files changed, 93 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e8a87336..bc366430 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ omnect-cli file copy-to-image --help A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. -The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. `copy-from-image` only reads the image and writes nothing. +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs space for one more image. `copy-from-image` only reads the image and writes nothing. ## bmap files diff --git a/src/file/output.rs b/src/file/output.rs index 953173d0..f1349b7c 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -5,6 +5,8 @@ use std::{ path::{Path, PathBuf}, }; +const TMP_DIR_PREFIX: &str = ".omnect-cli-"; + /// Returns `explicit`, else `image` without the extension of its `source` compression, with the /// extension of `target` appended. Fails if `explicit` has no extension of `target`, or a /// compression extension while `target` is none. @@ -84,6 +86,34 @@ pub fn check_conflicts( Ok(()) } +/// Writes `dest` as a new file in its directory and renames it, so `dest` stays unchanged if +/// `write` fails. A symlink is followed, and an existing `dest` keeps its permissions. +pub fn replace(dest: &Path, write: impl FnOnce(&Path) -> Result<()>) -> Result<()> { + let dest = resolve(dest)?; + let dir = dest + .parent() + .context(format!("{} has no directory", dest.display()))?; + let tmp_dir = tempfile::Builder::new() + .prefix(TMP_DIR_PREFIX) + .tempdir_in(dir) + .context(format!("cannot create a temp dir in {}", dir.display()))?; + let tmp_file = tmp_dir.path().join( + dest.file_name() + .context(format!("{} has no file name", dest.display()))?, + ); + + write(&tmp_file)?; + if let Ok(metadata) = fs::metadata(&dest) { + fs::set_permissions(&tmp_file, metadata.permissions()) + .context(format!("cannot set permissions of {}", tmp_file.display()))?; + } + fs::rename(&tmp_file, &dest).context(format!( + "cannot rename {} to {}", + tmp_file.display(), + dest.display() + )) +} + fn same_file(a: &Path, b: &Path) -> Result { Ok(resolve(a)? == resolve(b)?) } @@ -110,11 +140,16 @@ fn resolve(path: &Path) -> Result { #[cfg(test)] mod tests { use crate::file::output::*; + use std::os::unix::fs::{PermissionsExt, symlink}; const IMAGE: &str = "image.wic"; const IMAGE_XZ: &str = "image.wic.xz"; const OTHER_IMAGE: &str = "device.wic"; const XZ_LEVEL: u32 = 9; + const OLD_CONTENT: &[u8] = b"old image"; + const NEW_CONTENT: &[u8] = b"new image"; + const IMAGE_MODE: u32 = 0o644; + const MODE_MASK: u32 = 0o777; fn touch(path: &Path) { fs::write(path, []).expect("create file"); @@ -283,6 +318,47 @@ mod tests { assert!(check_conflicts(&image_xz, &image, false, true, None).is_err()); } + #[test] + fn replace_keeps_dest_when_write_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + fs::write(&image, OLD_CONTENT).expect("write image"); + + assert!( + replace(&image, |tmp| { + fs::write(tmp, NEW_CONTENT).expect("write tmp"); + anyhow::bail!("write failed") + }) + .is_err() + ); + assert_eq!(fs::read(&image).expect("read image"), OLD_CONTENT); + assert_eq!( + fs::read_dir(dir.path()).expect("read dir").count(), + 1, + "temp dir was left" + ); + } + + #[test] + fn replace_keeps_permissions_and_symlink() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let link = dir.path().join(OTHER_IMAGE); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::set_permissions(&image, fs::Permissions::from_mode(IMAGE_MODE)) + .expect("set permissions"); + symlink(&image, &link).expect("create symlink"); + + replace(&link, |tmp| Ok(fs::write(tmp, NEW_CONTENT)?)).expect("replace"); + + assert!(link.is_symlink()); + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + assert_eq!( + fs::metadata(&image).expect("metadata").permissions().mode() & MODE_MASK, + IMAGE_MODE + ); + } + #[test] fn same_file_resolves_relative_and_missing_paths() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/lib.rs b/src/lib.rs index f4eb0168..3f32e2f9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,7 +25,7 @@ use file::{ bmap::{self, Bmap}, compression::{self, Compression}, functions::FileCopyToParams, - output::{check_conflicts, output_image}, + output::{check_conflicts, output_image, replace}, sparse, }; use log::{error, info}; @@ -144,24 +144,25 @@ where // if applicable compress image if let Some(c) = target_compression { tmp_image_file = compression::compress(&tmp_image_file, &c)?; - fs::copy(&tmp_image_file, &dest_image_file).context(format!( - "error: std::fs::copy({:?}, {:?})", - tmp_image_file, dest_image_file - ))?; + replace(&dest_image_file, |dest| { + fs::copy(&tmp_image_file, dest) + .map(drop) + .context(format!("cannot copy {tmp_image_file:?} to {dest:?}")) + })?; } else { - copy_image(&tmp_image_file, &dest_image_file).context(format!( - "error: copy_image({:?}, {:?})", - tmp_image_file, dest_image_file - ))?; + replace(&dest_image_file, |dest| { + copy_image(&tmp_image_file, dest) + .context(format!("cannot copy {tmp_image_file:?} to {dest:?}")) + })?; } // copy the bmap last, so it never describes an image that was not written back if output_bmap { - let target_bmap = bmap::bmap_path(&dest_image_file); - fs::copy(&tmp_bmap, &target_bmap).context(format!( - "error: std::fs::copy({:?}, {:?})", - tmp_bmap, target_bmap - ))?; + replace(&bmap::bmap_path(&dest_image_file), |dest| { + fs::copy(&tmp_bmap, dest) + .map(drop) + .context(format!("cannot copy {tmp_bmap:?} to {dest:?}")) + })?; } Ok(()) diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 9aeb30a9..32616a39 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1171,7 +1171,7 @@ fn check_no_output_bmap_when_image_write_back_fails() { .arg(&written_image_path) .assert() .failure() - .stderr(predicate::str::contains("copy_image")); + .stderr(predicate::str::contains("cannot rename")); assert!( !bmap_path.exists(), "bmap written for an image that was not written back" From 20c114b2692b5214374b1b6ed821d6f31288219a Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:02:49 +0100 Subject: [PATCH 39/46] refactor(file): use tempfile::TempDir for the temp image The temp dir now follows TMPDIR and no longer needs its own tokio runtime to be removed. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/lib.rs | 44 +++++--------------------------------------- 1 file changed, 5 insertions(+), 39 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 3f32e2f9..85ab18d9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,45 +28,11 @@ use file::{ output::{check_conflicts, output_image, replace}, sparse, }; -use log::{error, info}; +use log::info; use std::{ fs, path::{Path, PathBuf}, }; -use tokio::fs::remove_dir_all; -use uuid::Uuid; - -/// A new directory `/tmp/{uuid}`, removed on drop. -struct TempDirGuard(PathBuf); - -impl TempDirGuard { - fn new() -> Result { - let tmp_dir = PathBuf::from(format!("/tmp/{}", Uuid::new_v4())); - fs::create_dir_all(&tmp_dir).context(format!( - "TempDirGuard::new: cannot create {}", - tmp_dir.display() - ))?; - Ok(Self(tmp_dir)) - } -} - -impl Drop for TempDirGuard { - fn drop(&mut self) { - let Ok(rt) = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - else { - error!("cannot create tokio runtime"); - return; - }; - - rt.block_on(async { - if let Err(e) = remove_dir_all(self.0.clone()).await { - error!("cannot remove tmp dir: {e}") - } - }) - } -} fn run_image_command(image_file: PathBuf, output: ImageOutput, command: F) -> Result<()> where @@ -119,8 +85,8 @@ where if source_compression.is_none() { verify_input_bmap(&image_file)?; } - let tmp_dir = TempDirGuard::new()?; - let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; + let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; + let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; if source_compression.is_some() { verify_input_bmap(&tmp_image_file)?; } @@ -183,9 +149,9 @@ where { ensure_image_exists(&image_file)?; - let tmp_dir = TempDirGuard::new()?; + let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; let source_compression = Compression::from_file(&image_file)?; - let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), &tmp_dir.0)?; + let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; command(&tmp_image_file) } From 342bea1bb1c5cdcaff9cdc16a4c19c1683489ba6 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:03:57 +0100 Subject: [PATCH 40/46] fix(file): keep the partition files in their own temp dir The partition files were named .img in the directory of the image, so an image named e.g. 1.img was overwritten by its own partition 1. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/functions.rs | 22 ++++++----------- tests/integration_tests.rs | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/src/file/functions.rs b/src/file/functions.rs index 9c9ffecc..5e4a2794 100644 --- a/src/file/functions.rs +++ b/src/file/functions.rs @@ -189,12 +189,8 @@ macro_rules! try_exec_cmd { } pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) -> Result<()> { - // we use the folder the image is located in - // the caller is responsible to create a /tmp/ directory if needed - let working_dir = image_file - .parent() - .context("copy_to_image: cannot get directory of image")? - .to_path_buf(); + // the partition files get their own dir, so they never collide with the image + let working_dir = tempfile::tempdir().context("copy_to_image: cannot create a temp dir")?; let image_file = image_file.to_str().unwrap(); let mut partition_map: HashMap<&Partition, Vec<(&PathBuf, &PathBuf)>> = HashMap::new(); @@ -209,7 +205,7 @@ pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) - // 1. for each involved partition for partition in partition_map.keys() { - let mut partition_file = working_dir.clone(); + let mut partition_file = working_dir.path().to_path_buf(); let partition_info = get_partition_info(image_file, partition)?; partition_file.push(Path::new(&format!("{}.img", partition_info.num))); @@ -271,16 +267,12 @@ pub fn copy_to_image(file_copy_params: &[FileCopyToParams], image_file: &Path) - } pub fn copy_from_image(file_copy_params: &[FileCopyFromParams], image_file: &Path) -> Result<()> { - // we use the folder the image is located in - // the caller is responsible to create a /tmp/ directory if needed - let working_dir = image_file - .parent() - .context("copy_to_image: cannot get directory of image")? - .to_path_buf(); + // the partition files get their own dir, so they never collide with the image + let working_dir = tempfile::tempdir().context("copy_from_image: cannot create a temp dir")?; let image_file = image_file.to_str().unwrap(); for param in file_copy_params.iter() { - let mut partition_file = working_dir.clone(); + let mut partition_file = working_dir.path().to_path_buf(); let partition_info = get_partition_info(image_file, ¶m.partition)?; let in_file = param.in_file.to_str().unwrap(); @@ -302,7 +294,7 @@ pub fn copy_from_image(file_copy_params: &[FileCopyFromParams], image_file: &Pat // copy if param.partition == Partition::boot { - let mut tmp_out_file = working_dir.clone(); + let mut tmp_out_file = working_dir.path().to_path_buf(); // mcopy deadlocks when target file is not residing in workingdir so we copy to a temp file tmp_out_file.push(format!( "{}-{}", diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 32616a39..d0b358e5 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -1618,6 +1618,55 @@ fn check_copy_from_image_writes_no_image() { assert_eq!(Testrunner::file_hash(&image_path_wic_xz), image_hash); } +#[test] +fn check_partition_files_stay_out_of_the_image_dir() { + let tr = Testrunner::new(function_name!().split("::").last().expect("test name")); + // the name of the file that holds partition 1 + let image_path = tr.pathbuf().join("1.img"); + fs::rename(tr.to_pathbuf("testfiles/image.wic"), &image_path).expect("rename image"); + let in_file = tr.to_pathbuf("testfiles/boot.scr"); + let out_file = tr.pathbuf().join("my-file"); + let file_image = |command: &str, files: String| { + Command::cargo_bin("omnect-cli") + .expect("omnect-cli binary") + .arg("file") + .arg(command) + .arg("-f") + .arg(files) + .arg("-i") + .arg(&image_path) + .assert() + .success(); + }; + let dir_entries = || { + let mut entries = fs::read_dir(tr.pathbuf()) + .expect("read test dir") + .map(|entry| entry.expect("dir entry").file_name()) + .collect::>(); + entries.sort(); + entries + }; + + file_image( + "copy-to-image", + format!("{},boot:/my-file", in_file.to_string_lossy()), + ); + let entries = dir_entries(); + file_image( + "copy-from-image", + format!("boot:/my-file,{}", out_file.to_string_lossy()), + ); + + assert_eq!( + fs::read(&out_file).expect("read out file"), + fs::read(&in_file).expect("read in file") + ); + let mut expected = entries; + expected.push(out_file.file_name().expect("file name").to_owned()); + expected.sort(); + assert_eq!(dir_entries(), expected); +} + #[tokio::test] async fn check_ssh_tunnel_setup() { let tr = Testrunner::new("check_ssh_tunnel_setup"); From 53672ec37f3da9c5a9b09de792ba9e03d2821075 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:03:57 +0100 Subject: [PATCH 41/46] perf(file): run copy-from-image on an uncompressed image directly copy-from-image only reads the image, and the partition files no longer go next to it, so a temp copy is needed only to decompress the image. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/lib.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 85ab18d9..a6b6eb27 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -142,18 +142,20 @@ fn input_bmap_error(path: &Path) -> String { ) } -/// Runs `command` on a temp copy of the image and writes nothing back. +/// Runs `command` on the image, or on a decompressed temp copy of a compressed image. fn run_read_only_image_command(image_file: PathBuf, command: F) -> Result<()> where F: FnOnce(&PathBuf) -> Result<()>, { ensure_image_exists(&image_file)?; - let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; - let source_compression = Compression::from_file(&image_file)?; - let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; - - command(&tmp_image_file) + match Compression::from_file(&image_file)? { + Some(c) => { + let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; + command(&compression::decompress(&image_file, &c, tmp_dir.path())?) + } + None => command(&image_file), + } } fn ensure_image_exists(image_file: &Path) -> Result<()> { From b328b451d7b5e5c788f10b47e1bbe22beed9fc99 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:04:39 +0100 Subject: [PATCH 42/46] fix(file): read partitions of a GPT with 4096-byte sectors gptman finds a GPT with 512 or 4096-byte sectors, but the partition offsets were always counted in 512-byte sectors. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/partition.rs | 43 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/src/file/partition.rs b/src/file/partition.rs index 7a617fcd..747c8f4f 100644 --- a/src/file/partition.rs +++ b/src/file/partition.rs @@ -31,10 +31,12 @@ pub fn get_partition_data>(path: P, partition_num: u32) -> Result entry.ending_lba >= entry.starting_lba, "GPT partition {partition_num} has invalid LBA range (ending_lba < starting_lba)" ); + // the callers count in SECTOR_SIZE units + let scale = gpt.sector_size / SECTOR_SIZE; return Ok(PartitionData { num: partition_num, - start: entry.starting_lba, - count: entry.ending_lba - entry.starting_lba + 1, + start: entry.starting_lba * scale, + count: (entry.ending_lba - entry.starting_lba + 1) * scale, }); } Err(e) => e, @@ -61,3 +63,40 @@ pub fn is_gpt>(path: P) -> Result { let mut file = File::open(path.as_ref()).context("is_gpt: failed to open image")?; Ok(gptman::GPT::find_from(&mut file).is_ok()) } + +#[cfg(test)] +mod tests { + use crate::file::partition::*; + use gptman::{GPT, GPTPartitionEntry}; + + const GPT_SECTOR_SIZE: u64 = 4096; + const DISK_SECTORS: u64 = 64; + const PART_NUM: u32 = 1; + const PART_SECTORS: u64 = 8; + const GUID: [u8; 16] = [0xff; 16]; + + #[test] + fn get_partition_data_converts_4k_gpt_sectors() { + let dir = tempfile::tempdir().expect("create temp dir"); + let path = dir.path().join("image.wic"); + let mut file = File::create(&path).expect("create image"); + file.set_len(DISK_SECTORS * GPT_SECTOR_SIZE) + .expect("set image size"); + let mut gpt = GPT::new_from(&mut file, GPT_SECTOR_SIZE, GUID).expect("create gpt"); + let start = gpt.header.first_usable_lba; + gpt[PART_NUM] = GPTPartitionEntry { + partition_type_guid: GUID, + unique_partition_guid: GUID, + starting_lba: start, + ending_lba: start + PART_SECTORS - 1, + attribute_bits: 0, + partition_name: "part".into(), + }; + gpt.write_into(&mut file).expect("write gpt"); + + let data = get_partition_data(&path, PART_NUM).expect("partition data"); + let scale = GPT_SECTOR_SIZE / SECTOR_SIZE; + assert_eq!(data.start, start * scale); + assert_eq!(data.count, PART_SECTORS * scale); + } +} From ee181b42b2a605aa7ac72337aa22b1c9860c0423 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:07:35 +0100 Subject: [PATCH 43/46] docs(file): describe the --output-image extension rule and the rename Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/cli.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bc366430..61afc38d 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ omnect-cli file copy-to-image --help A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. -The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs space for one more image. `copy-from-image` only reads the image and writes nothing. +The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. ## bmap files diff --git a/src/cli.rs b/src/cli.rs index 7b824c6f..3c2e77de 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -23,7 +23,7 @@ pub struct ImageOutput { conflicts_with = "input_bmap" )] pub no_input_bmap: bool, - /// optional: write the output image to this path; default: the input image, without the compression extension, plus the --pack-image extension + /// optional: write the output image to this path, with the --pack-image extension or no compression extension; default: the input image, without the compression extension, plus the --pack-image extension #[arg(short = 'o', long = "output-image")] pub output_image: Option, /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=') From 961f95b16bc50f4ce0017aea77cfbdc81f43adb2 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:42:21 +0100 Subject: [PATCH 44/46] fix(file): keep the name of an image with any extension of the -p compression `-i x.wic.gz -p gzip` wrote `x.wic.gzip`, and `-i x.wic.bzip2 -p bzip2` wrote `x.wic.bz2`, so a second run failed. gzip images now get `.gz`. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/file/compression.rs | 4 ++-- src/file/output.rs | 26 +++++++++++++++++++++++--- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 61afc38d..6d7ef853 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ omnect-cli file copy-to-image --help ## Output image -A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gzip`), e.g. `my-image.wic.xz` for `my-image.wic`. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. +A command that changes an image writes the result to the input image. For a compressed image it writes the uncompressed image next to it, e.g. `my-image.wic` for `my-image.wic.xz`. With `-p` (`--pack-image`) it adds the extension of the compression (`.xz`, `.bz2` or `.gz`), e.g. `my-image.wic.xz` for `my-image.wic`. An image that already has an extension of that compression keeps its name. `-o` (`--output-image`) writes the result to the given path instead, and the input image stays unchanged. Its name must have the extension of the `-p` compression, or no compression extension without `-p`. The command fails instead of overwriting an existing file that is not the input image. To overwrite it, name it with `-o`. The output image is written to a new file in its directory and then renamed, so the directory needs write access and space for one more image. The new file belongs to the user who runs `omnect-cli`. `copy-from-image` only reads the image and writes nothing. diff --git a/src/file/compression.rs b/src/file/compression.rs index cf3f173d..6d9b5ed4 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -112,7 +112,7 @@ impl Compression { fn extensions(&self) -> &'static [&'static str] { match &self { Compression::bzip2 => &["bz2", "bzip2"], - Compression::gzip => &["gzip", "gz"], + Compression::gzip => &["gz", "gzip"], Compression::xz { .. } => &["xz"], } } @@ -419,7 +419,7 @@ mod tests { #[test] fn compressed_path_has_an_extension_bmaptool_reads() { let image = Path::new("image.wic"); - let expected = ["image.wic.bz2", "image.wic.gzip", "image.wic.xz"]; + let expected = ["image.wic.bz2", "image.wic.gz", "image.wic.xz"]; for (compression, expected) in all_compressions().iter().zip(expected) { assert_eq!(compression.compressed_path(image), Path::new(expected)); diff --git a/src/file/output.rs b/src/file/output.rs index f1349b7c..389b3601 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -7,9 +7,9 @@ use std::{ const TMP_DIR_PREFIX: &str = ".omnect-cli-"; -/// Returns `explicit`, else `image` without the extension of its `source` compression, with the -/// extension of `target` appended. Fails if `explicit` has no extension of `target`, or a -/// compression extension while `target` is none. +/// Returns `explicit`, else `image` if it has an extension of `target`, else `image` without the +/// extension of its `source` compression, with the extension of `target` appended. Fails if +/// `explicit` has no extension of `target`, or a compression extension while `target` is none. pub fn output_image( image: &Path, source: Option<&Compression>, @@ -32,6 +32,9 @@ pub fn output_image( } return Ok(path.to_path_buf()); } + if target.is_some_and(|c| c.matches_extension(image)) { + return Ok(image.to_path_buf()); + } let mut output = image.to_path_buf(); if source.is_some_and(|c| c.matches_extension(image)) { @@ -172,6 +175,23 @@ mod tests { assert_eq!(output(image_xz, Some(&xz), Some(&xz)), image_xz); } + #[test] + fn output_image_keeps_every_extension_of_the_same_compression() { + for (image, compression) in [ + ("image.wic.gz", Compression::gzip), + ("image.wic.gzip", Compression::gzip), + ("image.wic.bz2", Compression::bzip2), + ("image.wic.bzip2", Compression::bzip2), + ] { + let image = Path::new(image); + assert_eq!( + output_image(image, Some(&compression), Some(&compression), None) + .expect("output image"), + image + ); + } + } + #[test] fn output_image_strips_only_the_source_compression_extension() { let image = Path::new("image.bin"); From 3504b5a9e50eace9faaa86e0a1b71f81051f8417 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:43:26 +0100 Subject: [PATCH 45/46] fix(file): find the input bmap the way bmaptool does bmaptool removes one extension of the image at a time until it finds a bmap, so it also uses e.g. `x.bmap` for `x.wic`. The input bmap and the stale bmap check now follow the same rule. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- README.md | 2 +- src/cli.rs | 2 +- src/file/bmap.rs | 35 +++++++++++++++++++++++++++-------- src/file/output.rs | 15 ++++++++++++--- 4 files changed, 41 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 6d7ef853..a6ebc073 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ The command fails instead of overwriting an existing file that is not the input Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`. The command fails if this file is the bmap of an image that stays, e.g. `my-image.wic.bmap` when `-i my-image.wic.xz` writes `my-image.wic`, also with `--input-bmap` or `--no-input-bmap`. Choose another output with `-o` then. -The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if `.bmap` exists, or for a compressed output image the bmap of the uncompressed image (e.g. `my-image.wic.bmap` for `my-image.wic.xz`), because `bmaptool` would use it and it would not match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. +The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else the bmap next to the image that `bmaptool` would use: `.bmap`, else the same with one extension of the image removed at a time, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` fails if such a bmap exists for the output image (e.g. `my-image.wic.bmap` for `my-image.wic.xz`), because `bmaptool` would use it and it would not match the image. So when you run several commands on one image, use `-b` on each of them. `--no-input-bmap` skips a bmap next to the image that does not match, and the created bmap then maps the whole image. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse. Without an input bmap, the created bmap maps the whole image, and a flash writes every block. diff --git a/src/cli.rs b/src/cli.rs index 3c2e77de..dd20908c 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -13,7 +13,7 @@ pub struct ImageOutput { /// optional: create a bmap of the output image #[arg(short = 'b', long = "output-bmap")] pub output_bmap: bool, - /// optional: bmap of the input image; default: .bmap or .bmap next to the image + /// optional: bmap of the input image; default: the bmap next to the image that bmaptool would use #[arg(long = "input-bmap", requires = "output_bmap")] pub input_bmap: Option, /// optional: do not use a bmap next to the image, so the bmap maps the whole image diff --git a/src/file/bmap.rs b/src/file/bmap.rs index 96b13b5d..aacc6dfc 100644 --- a/src/file/bmap.rs +++ b/src/file/bmap.rs @@ -1,4 +1,4 @@ -use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, compression::Compression, sparse}; +use crate::file::{COPY_CHUNK_SIZE, IMAGE_BLOCK_SIZE, sparse}; use anyhow::{Context, Result}; use sha2::{Digest, Sha256}; use std::{ @@ -90,8 +90,8 @@ fn parse_range(text: &str) -> Result> { Ok(first..=last) } -/// Returns `explicit`, else `.bmap`, else `.bmap`, -/// whichever exists first. +/// Returns `explicit`, else the bmap `bmaptool` uses for `image`: `.bmap`, else the same +/// with one extension of `image` removed at a time, whichever exists first. pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result> { if let Some(path) = explicit { anyhow::ensure!( @@ -102,12 +102,17 @@ pub fn find_input_bmap(image: &Path, explicit: Option<&Path>) -> Result PathBuf { @@ -420,6 +425,20 @@ mod tests { ); } + #[test] + fn find_input_bmap_strips_extensions_like_bmaptool() { + let dir = tempfile::tempdir().expect("create temp dir"); + let bmap = dir.path().join("image.bmap"); + fs::write(&bmap, "").expect("write bmap"); + + for image in ["image.wic", "image.wic.xz"] { + assert_eq!( + find_input_bmap(&dir.path().join(image), None).expect("find bmap"), + Some(bmap.clone()) + ); + } + } + #[test] fn write_bmap_maps_zero_ranges_of_the_input_bmap() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/file/output.rs b/src/file/output.rs index 389b3601..fc213713 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -47,9 +47,8 @@ pub fn output_image( } /// Fails if writing `output_image` and its bmap would overwrite a file the user did not name, -/// replace the bmap of an image that stays, or leave a bmap next to the output image that does -/// not match it: `.bmap`, or for a compressed output image the bmap of the -/// uncompressed image, which `bmaptool` also uses. +/// replace the bmap of an image that stays, or leave a bmap that `bmaptool` would use for the +/// output image and that does not match it. pub fn check_conflicts( input_image: &Path, output_image: &Path, @@ -310,6 +309,16 @@ mod tests { assert!(check_conflicts(&image, &image, false, false, None).is_err()); } + #[test] + fn check_conflicts_rejects_a_stale_bmap_bmaptool_finds() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + touch(&image); + touch(&dir.path().join("image.bmap")); + + assert!(check_conflicts(&image, &image, false, false, None).is_err()); + } + #[test] fn check_conflicts_rejects_a_stale_bmap_of_the_uncompressed_image() { let dir = tempfile::tempdir().expect("create temp dir"); From 3da5a4d43dc48d03aa5b02b82a5a5251b4947070 Mon Sep 17 00:00:00 2001 From: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> Date: Thu, 29 Oct 2026 17:46:08 +0100 Subject: [PATCH 46/46] fix(file): sync the output image and remove the old bmap before the rename The output image and its bmap are now staged first: written, synced on the handle that wrote them, and given their permissions. Then the old bmap is removed, the image and the bmap are renamed, and their directories are synced. So a power loss cannot leave an empty or partial image, and a failed bmap rename leaves no bmap that does not match the new image. A compressed image is written into its staged file directly, without a copy in TMPDIR. Signed-off-by: Jan Zachmann <50990105+JanZachmann@users.noreply.github.com> --- src/file/compression.rs | 19 +++-- src/file/output.rs | 167 ++++++++++++++++++++++++++++++++-------- src/file/sparse.rs | 11 ++- src/lib.rs | 56 ++++++-------- 4 files changed, 182 insertions(+), 71 deletions(-) diff --git a/src/file/compression.rs b/src/file/compression.rs index 6d9b5ed4..d15c1c30 100644 --- a/src/file/compression.rs +++ b/src/file/compression.rs @@ -239,14 +239,19 @@ pub fn decompress( Ok(new_image_file) } -pub fn compress(image_file_name: &PathBuf, compression: &Compression) -> Result { - let new_image_file = compression.compressed_path(image_file_name); - let mut destination = File::create(&new_image_file)?; - let mut source = File::open(image_file_name)?; - debug!("compress {image_file_name:?} to {new_image_file:?}"); - let bytes_written = compression.compress(&mut source, &mut destination)?; +pub fn compress( + image_file_name: &Path, + compression: &Compression, + destination: &mut File, +) -> Result<()> { + let mut source = File::open(image_file_name).context(format!( + "compress: cannot open {}", + image_file_name.display() + ))?; + debug!("compress {image_file_name:?}"); + let bytes_written = compression.compress(&mut source, destination)?; debug!("image::compress: copied {} bytes.", bytes_written); - Ok(new_image_file) + Ok(()) } #[cfg(test)] diff --git a/src/file/output.rs b/src/file/output.rs index fc213713..d52dd02a 100644 --- a/src/file/output.rs +++ b/src/file/output.rs @@ -1,9 +1,14 @@ use crate::file::{bmap, compression::Compression}; use anyhow::{Context, Result}; +use log::warn; use std::{ - fs, + collections::BTreeSet, + fs::{self, File}, + io::ErrorKind, + iter, path::{Path, PathBuf}, }; +use tempfile::TempDir; const TMP_DIR_PREFIX: &str = ".omnect-cli-"; @@ -88,32 +93,89 @@ pub fn check_conflicts( Ok(()) } -/// Writes `dest` as a new file in its directory and renames it, so `dest` stays unchanged if -/// `write` fails. A symlink is followed, and an existing `dest` keeps its permissions. -pub fn replace(dest: &Path, write: impl FnOnce(&Path) -> Result<()>) -> Result<()> { - let dest = resolve(dest)?; - let dir = dest - .parent() - .context(format!("{} has no directory", dest.display()))?; - let tmp_dir = tempfile::Builder::new() - .prefix(TMP_DIR_PREFIX) - .tempdir_in(dir) - .context(format!("cannot create a temp dir in {}", dir.display()))?; - let tmp_file = tmp_dir.path().join( - dest.file_name() - .context(format!("{} has no file name", dest.display()))?, - ); +/// A new file in the directory of its destination, which `commit` renames to the destination. +/// Until then the destination stays unchanged. +pub struct Staged { + _tmp_dir: TempDir, + file: PathBuf, + dest: PathBuf, +} + +impl Staged { + /// Writes and syncs the new file for `dest`. A symlink is followed, and an existing `dest` + /// keeps its permissions. + pub fn write(dest: &Path, write: impl FnOnce(&mut File) -> Result<()>) -> Result { + let dest = resolve(dest)?; + let dir = dest + .parent() + .context(format!("{} has no directory", dest.display()))?; + let tmp_dir = tempfile::Builder::new() + .prefix(TMP_DIR_PREFIX) + .tempdir_in(dir) + .context(format!("cannot create a temp dir in {}", dir.display()))?; + let file = tmp_dir.path().join( + dest.file_name() + .context(format!("{} has no file name", dest.display()))?, + ); + + let mut handle = + File::create(&file).context(format!("cannot create {}", file.display()))?; + write(&mut handle)?; + if let Ok(metadata) = fs::metadata(&dest) { + handle + .set_permissions(metadata.permissions()) + .context(format!("cannot set permissions of {}", file.display()))?; + } + // the handle that wrote the data also gets the write errors + handle + .sync_all() + .context(format!("cannot sync {}", file.display()))?; + + Ok(Self { + _tmp_dir: tmp_dir, + file, + dest, + }) + } + + fn rename(&self) -> Result<()> { + fs::rename(&self.file, &self.dest).context(format!( + "cannot rename {} to {}", + self.file.display(), + self.dest.display() + )) + } +} - write(&tmp_file)?; - if let Ok(metadata) = fs::metadata(&dest) { - fs::set_permissions(&tmp_file, metadata.permissions()) - .context(format!("cannot set permissions of {}", tmp_file.display()))?; +/// Renames the staged `image` and its `bmap`, then syncs their directories, so the renames are +/// not lost on a power loss. The old bmap is removed before the image is renamed, so if renaming +/// the bmap fails, no bmap is left that does not match the new image. +pub fn commit(image: Staged, bmap: Option) -> Result<()> { + if let Some(bmap) = &bmap { + match fs::remove_file(&bmap.dest) { + Err(e) if e.kind() != ErrorKind::NotFound => { + return Err(e).context(format!("cannot remove {}", bmap.dest.display())); + } + _ => {} + } + } + image.rename()?; + if let Some(bmap) = &bmap { + bmap.rename()?; + } + + let dirs: BTreeSet<_> = iter::once(&image) + .chain(&bmap) + .filter_map(|staged| staged.dest.parent()) + .collect(); + for dir in dirs { + // both files are complete already, only a power loss could undo the rename + if let Err(e) = File::open(dir).and_then(|dir| dir.sync_all()) { + warn!("cannot sync {}: {e}", dir.display()); + } } - fs::rename(&tmp_file, &dest).context(format!( - "cannot rename {} to {}", - tmp_file.display(), - dest.display() - )) + + Ok(()) } fn same_file(a: &Path, b: &Path) -> Result { @@ -142,7 +204,10 @@ fn resolve(path: &Path) -> Result { #[cfg(test)] mod tests { use crate::file::output::*; - use std::os::unix::fs::{PermissionsExt, symlink}; + use std::{ + io::Write, + os::unix::fs::{PermissionsExt, symlink}, + }; const IMAGE: &str = "image.wic"; const IMAGE_XZ: &str = "image.wic.xz"; @@ -347,15 +412,19 @@ mod tests { assert!(check_conflicts(&image_xz, &image, false, true, None).is_err()); } + fn stage(dest: &Path, content: &'static [u8]) -> Staged { + Staged::write(dest, |file| Ok(file.write_all(content)?)).expect("stage") + } + #[test] - fn replace_keeps_dest_when_write_fails() { + fn staged_write_keeps_dest_when_write_fails() { let dir = tempfile::tempdir().expect("create temp dir"); let image = dir.path().join(IMAGE); fs::write(&image, OLD_CONTENT).expect("write image"); assert!( - replace(&image, |tmp| { - fs::write(tmp, NEW_CONTENT).expect("write tmp"); + Staged::write(&image, |file| { + file.write_all(NEW_CONTENT).expect("write file"); anyhow::bail!("write failed") }) .is_err() @@ -369,7 +438,7 @@ mod tests { } #[test] - fn replace_keeps_permissions_and_symlink() { + fn staged_commit_keeps_permissions_and_symlink() { let dir = tempfile::tempdir().expect("create temp dir"); let image = dir.path().join(IMAGE); let link = dir.path().join(OTHER_IMAGE); @@ -378,7 +447,7 @@ mod tests { .expect("set permissions"); symlink(&image, &link).expect("create symlink"); - replace(&link, |tmp| Ok(fs::write(tmp, NEW_CONTENT)?)).expect("replace"); + commit(stage(&link, NEW_CONTENT), None).expect("commit"); assert!(link.is_symlink()); assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); @@ -388,6 +457,42 @@ mod tests { ); } + #[test] + fn commit_leaves_no_old_bmap_when_the_bmap_rename_fails() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::write(&image_bmap, OLD_CONTENT).expect("write bmap"); + + let staged_bmap = stage(&image_bmap, NEW_CONTENT); + fs::remove_file(&staged_bmap.file).expect("remove staged bmap"); + + assert!(commit(stage(&image, NEW_CONTENT), Some(staged_bmap)).is_err()); + assert_eq!(fs::read(&image).expect("read image"), NEW_CONTENT); + assert!(!image_bmap.exists()); + } + + #[test] + fn commit_replaces_the_target_of_a_symlinked_bmap() { + let dir = tempfile::tempdir().expect("create temp dir"); + let image = dir.path().join(IMAGE); + let image_bmap = bmap::bmap_path(&image); + let target_bmap = dir.path().join("target.bmap"); + fs::write(&image, OLD_CONTENT).expect("write image"); + fs::write(&target_bmap, OLD_CONTENT).expect("write bmap"); + symlink(&target_bmap, &image_bmap).expect("create symlink"); + + commit( + stage(&image, NEW_CONTENT), + Some(stage(&image_bmap, NEW_CONTENT)), + ) + .expect("commit"); + + assert!(image_bmap.is_symlink()); + assert_eq!(fs::read(&target_bmap).expect("read bmap"), NEW_CONTENT); + } + #[test] fn same_file_resolves_relative_and_missing_paths() { let dir = tempfile::tempdir().expect("create temp dir"); diff --git a/src/file/sparse.rs b/src/file/sparse.rs index 3f5a4005..7953b8e1 100644 --- a/src/file/sparse.rs +++ b/src/file/sparse.rs @@ -87,10 +87,17 @@ pub fn copy_range( pub fn copy_to_new_file(src: &File, src_offset: u64, dst: &Path, len: u64) -> Result<()> { let dst = File::create(dst).context(format!("copy_to_new_file: cannot create {}", dst.display()))?; + + copy_to_empty_file(src, src_offset, &dst, len) +} + +/// Sets the size of the empty `dst` to `len` and copies `len` bytes of `src` at `src_offset` +/// into it. Holes of `src` stay holes in `dst`. +pub fn copy_to_empty_file(src: &File, src_offset: u64, dst: &File, len: u64) -> Result<()> { dst.set_len(len) - .context("copy_to_new_file: cannot set destination size")?; + .context("copy_to_empty_file: cannot set destination size")?; - copy_range(src, src_offset, &dst, 0, len) + copy_range(src, src_offset, dst, 0, len) } #[cfg(test)] diff --git a/src/lib.rs b/src/lib.rs index a6b6eb27..95ba6e53 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -25,12 +25,12 @@ use file::{ bmap::{self, Bmap}, compression::{self, Compression}, functions::FileCopyToParams, - output::{check_conflicts, output_image, replace}, + output::{self, Staged, check_conflicts, output_image}, sparse, }; use log::info; use std::{ - fs, + fs, io, path::{Path, PathBuf}, }; @@ -86,7 +86,7 @@ where verify_input_bmap(&image_file)?; } let tmp_dir = tempfile::tempdir().context("cannot create a temp dir")?; - let mut tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; + let tmp_image_file = temp_image(&image_file, source_compression.as_ref(), tmp_dir.path())?; if source_compression.is_some() { verify_input_bmap(&tmp_image_file)?; } @@ -107,31 +107,23 @@ where )?; } - // if applicable compress image - if let Some(c) = target_compression { - tmp_image_file = compression::compress(&tmp_image_file, &c)?; - replace(&dest_image_file, |dest| { - fs::copy(&tmp_image_file, dest) - .map(drop) - .context(format!("cannot copy {tmp_image_file:?} to {dest:?}")) - })?; - } else { - replace(&dest_image_file, |dest| { - copy_image(&tmp_image_file, dest) - .context(format!("cannot copy {tmp_image_file:?} to {dest:?}")) - })?; - } - - // copy the bmap last, so it never describes an image that was not written back - if output_bmap { - replace(&bmap::bmap_path(&dest_image_file), |dest| { - fs::copy(&tmp_bmap, dest) - .map(drop) - .context(format!("cannot copy {tmp_bmap:?} to {dest:?}")) - })?; - } + let image = Staged::write(&dest_image_file, |dest| match &target_compression { + Some(c) => compression::compress(&tmp_image_file, c, dest), + None => copy_image(&tmp_image_file, dest), + })?; + let bmap = output_bmap + .then(|| { + Staged::write(&bmap::bmap_path(&dest_image_file), |dest| { + let mut src = fs::File::open(&tmp_bmap) + .context(format!("cannot open {}", tmp_bmap.display()))?; + io::copy(&mut src, dest) + .map(drop) + .context(format!("cannot copy {}", tmp_bmap.display())) + }) + }) + .transpose()?; - Ok(()) + output::commit(image, bmap) } fn input_bmap_error(path: &Path) -> String { @@ -181,7 +173,9 @@ fn temp_image( .file_name() .context("cannot get image file name")?, ); - copy_image(image_file, &tmp_image_file).context(format!( + let tmp_image = fs::File::create(&tmp_image_file) + .context(format!("cannot create {}", tmp_image_file.display()))?; + copy_image(image_file, &tmp_image).context(format!( "error: copy_image({:?}, {:?})", image_file, tmp_image_file ))?; @@ -190,15 +184,15 @@ fn temp_image( } } -/// Copies `from` to `to` and keeps the holes of a sparse `from`. -fn copy_image(from: &Path, to: &Path) -> Result<()> { +/// Copies `from` to the empty `to` and keeps the holes of a sparse `from`. +fn copy_image(from: &Path, to: &fs::File) -> Result<()> { let src = fs::File::open(from).context("copy_image: cannot open source")?; let len = src .metadata() .context("copy_image: cannot get source size")? .len(); - sparse::copy_to_new_file(&src, 0, to, len) + sparse::copy_to_empty_file(&src, 0, to, len) } struct CertInfo {