fix(file)!: keep image holes, map the input bmap ranges, rename bmap options

Cargo.toml

@@ -7,7 +7,7 @@ license = "MIT OR Apache-2.0"
 name = "omnect-cli"
 readme = "README.md"
 repository = "https://github.com/omnect/omnect-cli"
-version = "0.28.5"
+version = "0.29.0"
 
 [dependencies]
 actix-web = { version = "4.17", default-features = false, features = ["macros"] }
@@ -31,7 +31,6 @@ hmac = { version = "0.13", default-features = false }
 mbrman = { version = "0.6", default-features = false }
 omnect-crypto = { git = "https://github.com/omnect/omnect-crypto.git", tag = "0.5.0" }
 keyring-core = { version = "1.0", default-features = false }
-libfs = { version = "0.9", default-features = false }
 log = { version = "0.4", default-features = false }
 num_cpus = { version = "1.17", default-features = false }
 oauth2 = { version = "5.0", default-features = false, features = [
@@ -44,6 +43,8 @@ reqwest = { version = "0.13", default-features = false, features = [
     "json",
     "native-tls",
 ] }
+roxmltree = { version = "0.21", default-features = false, features = ["std"] }
+rustix = { version = "1.1", default-features = false, features = ["fs", "std"] }
 serde = { version = "1.0", default-features = false, features = ["derive"] }
 serde_json = { version = "1.0", default-features = false }
 serde_path_to_error = { version = "0.1", default-features = false }
@@ -81,5 +82,5 @@ tar = "0.4"
 
 # metadata for building with cargo-deb (https://crates.io/crates/cargo-deb)
 [package.metadata.deb]
-depends = "bmap-tools, e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools"
+depends = "e2tools, keychain, libc6 (>= 2.39), libmagic1, libssl3 (>= 3.0.0), mtools"
 revision = ""

Dockerfile

@@ -13,7 +13,6 @@ COPY ${debian_dir}/omnect-cli_${omnect_cli_version}_amd64.deb omnect-cli_${omnec
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-    bmap-tools \
     ca-certificates \
     e2tools \
     fdisk \
@@ -34,14 +33,11 @@ RUN <<EOT
     mkdir -p /copy/status.d /copy/usr/lib
 
     executables=(
-        /usr/bin/dd \
         /usr/bin/e2cp \
         /usr/bin/e2mkdir \
-        /usr/bin/fallocate \
         /usr/bin/mcopy \
         /usr/bin/omnect-cli \
         /usr/bin/ssh-keygen \
-        /usr/bin/sync \
         /usr/sbin/fdisk \
     )
 

README.md

@@ -48,9 +48,8 @@ docker run --rm -it \
   omnect/omnect-cli:latest file copy-to-image --files /source/my-source-file,boot:/my-dest-file -i /source/my-image.wic
   ```
 
-  **Note1**: `-b` option to create bmap file is not supported by omnect-cli docker image.<br>
-  **Note2**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.<br>
-  **Note3**: The docker inject command is not supported by omnect-cli docker image.<br>.
+  **Note1**: The ssh tunnel option requires some additional settings. See [here](Usage-with-docker) for more details.<br>
+  **Note2**: The docker inject command is not supported by omnect-cli docker image.<br>.
 
 # Build from sources
 
@@ -184,6 +183,14 @@ omnect-cli file copy-to-image --help
 - File permissions: inject `systemd-tmpfiles.d`
 - Wifi: inject `wpa_supplicant-wlan0.conf`
 
+## bmap files
+
+Commands that change an image create a bmap file with `-b` (`--output-bmap`). It is written next to the output image as `<output image>.bmap`, e.g. `my-image.wic.bmap` for `my-image.wic` and `my-image.wic.xz.bmap` for `my-image.wic.xz`, and replaces a file with this name. `bmaptool` finds it there without `--bmap`.
+
+The bmap of the input image is the base for it: `omnect-cli` takes the file given with `--input-bmap`, else `<image>.bmap` or the bmap of the uncompressed image next to the image, e.g. `my-image.wic.bmap` for `my-image.wic.xz` as the Yocto build creates it. The input bmap must have the size of the image and sha256 range checksums that match it, otherwise the command fails. A command without `-b` changes the image but not the bmap next to it, so when you run several commands on one image, use `-b` on each of them, or pass the matching bmap with `--input-bmap`. The created bmap maps every range of the input bmap, also when it holds only zeros, so a bmap-based flash overwrites what an earlier installation left there (partition table entries, FAT, ext4 journals). It also maps every block that the image file stores as data, which includes the blocks `omnect-cli` changed. The image file itself stays sparse.
+
+Without an input bmap, the created bmap maps the whole image, and a flash writes every block.
+
 ## ssh tunnel
 
 ### Inject ssh tunnel credentials

project-context.md

@@ -50,15 +50,17 @@
 - `src/docker.rs` — `docker pull --platform` + `docker save` for multi-arch images
 - `src/image.rs` — firmware image architecture detection (ARM32/ARM64/x86_64)
 - `src/file/mod.rs` — high-level image operations: identity config, certs, hostname patching
-- `src/file/functions.rs` — partition read/modify/write via dd + e2cp/mcopy
+- `src/file/functions.rs` — sparse partition read, modify via e2cp/mcopy, write back of changed blocks only
+- `src/file/bmap.rs` — input bmap lookup and checksum check; writes the output bmap: input ranges (or the whole image) plus data blocks
+- `src/file/sparse.rs` — data/hole ranges via `SEEK_DATA`/`SEEK_HOLE`, sparse range copy
 - `src/file/partition.rs` — GPT/MBR partition table parsing
 - `src/file/compression.rs` — xz/bzip2/gzip compress/decompress with auto-detection
 - `src/validators/` — validation for identity config (TOML), device-update config (JSON), SSH keys
 - `conf/*.template` — 11 config templates (identity, device-update, WiFi)
 
 ## 4. Repository-Specific Constraints
 
-- External tools `e2cp`, `e2mkdir`, `mcopy`, `mmd`, `dd`, `fallocate`, `ssh-keygen`, `fdisk` must be available at runtime (Dockerfile copies them explicitly).
+- External tools `e2cp`, `e2mkdir`, `mcopy`, `mmd`, `ssh-keygen`, `fdisk` must be available at runtime (Dockerfile copies them explicitly).
 - Partition enum maps partition names to numbers differently for GPT vs MBR — see `file/functions.rs`.
 - OAuth2 callback binds to `127.0.0.1:4000` and `[::1]:4000`; container mode overrides to `0.0.0.0`.
 - `conf/` directory uses `.gitignore` to track only `*.template` files — actual configs are generated, never committed.

src/cli.rs

@@ -2,12 +2,25 @@ use crate::file::{
     compression::Compression,
     functions::{FileCopyFromParams, FileCopyToParams, Partition},
 };
-use clap::Parser;
+use clap::{Args, Parser};
 use std::path::PathBuf;
 use url::Url;
 
 const COPYRIGHT: &str = "Copyright © 2021 by conplement AG";
 
+#[derive(Args, Debug, Default)]
+pub struct ImageOutput {
+    /// optional: create a bmap of the output image
+    #[arg(short = 'b', long = "output-bmap")]
+    pub output_bmap: bool,
+    /// optional: bmap of the input image; default: <image>.bmap or <uncompressed image>.bmap next to the image
+    #[arg(long = "input-bmap", requires = "output_bmap")]
+    pub input_bmap: Option<PathBuf>,
+    /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
+    #[arg(short = 'p', long = "pack-image", value_enum)]
+    pub compress_image: Option<Compression>,
+}
+
 // ToDo: command completion
 #[derive(Parser, Debug)]
 #[command(after_help = COPYRIGHT)]
@@ -34,12 +47,8 @@ pub enum Docker {
         /// destination path of the docker image in the firmware image (must end in ".tar.gz")
         #[clap(short = 'e', long = "dest")]
         dest: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
 }
 
@@ -55,12 +64,8 @@ pub enum File {
         /// path to wic image file (optionally compressed with xz, bzip2 or gzip)
         #[arg(short = 'i', long = "image")]
         image: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// copy files from image
     CopyFromImage {
@@ -88,12 +93,8 @@ pub enum IdentityConfig {
         /// path to wic image file (optionally compressed with xz, bzip2 or gzip)
         #[arg(short = 'i', long = "image")]
         image: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// EXPERIMENTAL: set transparent gateway config.toml file and additional certificates and keys
     SetIotedgeGatewayConfig {
@@ -112,12 +113,8 @@ pub enum IdentityConfig {
         /// path to device identity certificate key file
         #[arg(short = 'k', long = "device_identity_key")]
         device_identity_key: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// EXPERIMENTAL: set leaf device config.toml file and additional certificate
     SetIotLeafSasConfig {
@@ -130,12 +127,8 @@ pub enum IdentityConfig {
         /// path to root ca certificate file
         #[arg(short = 'r', long = "root_ca")]
         root_ca: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// set certificates in order to support X.509 based DPS provisioning and certificate renewal via EST
     SetDeviceCertificate {
@@ -154,12 +147,8 @@ pub enum IdentityConfig {
         /// period of validity in days
         #[arg(short = 'D', long = "days")]
         days: u32,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// set certificates in order to support X.509 based DPS provisioning WITHOUT certificate renewal via EST
     SetDeviceCertificateNoEst {
@@ -172,12 +161,8 @@ pub enum IdentityConfig {
         /// path to wic image file (optionally compressed with xz, bzip2 or gzip)
         #[arg(short = 'i', long = "image")]
         image: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// generate and set bootstrap certificate for edge ca issuance/renewal.
     SetEdgeCaCertificate {
@@ -196,12 +181,8 @@ pub enum IdentityConfig {
         /// period of validity in days
         #[arg(short = 'D', long = "days")]
         days: u32,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
 }
 
@@ -217,12 +198,8 @@ pub enum IotHubDeviceUpdate {
         /// path to wic image file (optionally compressed with xz, bzip2 or gzip)
         #[arg(short = 'i', long = "image")]
         image: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
     /// import update to azure iot-hub
     ImportUpdate {
@@ -336,12 +313,8 @@ pub enum SshConfig {
         /// path to public key of the ssh root ca
         #[arg(short = 'r', long = "root_ca")]
         root_ca: PathBuf,
-        /// optional: generate bmap file (currently not working in docker image)
-        #[arg(short = 'b', long = "generate-bmap-file")]
-        generate_bmap: bool,
-        /// optional: pack image [xz, bzip2, gzip] (for xz default level '9' is used, which can be overwritten by setting 'XZ_COMPRESSION_LEVEL=')
-        #[arg(short = 'p', long = "pack-image", value_enum)]
-        compress_image: Option<Compression>,
+        #[command(flatten)]
+        output: ImageOutput,
     },
 
     /// set ssh connection parameters (see