From c54cf6da0df90d76ecf2b7bc99686f8664aabd1b Mon Sep 17 00:00:00 2001 From: Arun Babu Neelicattu Date: Fri, 23 Oct 2026 22:44:32 +0200 Subject: [PATCH] feat(oci): backfill image deltas from digests --- ARCHITECTURE.md | 3 +- README.md | 15 +++++ cmd/deltas.go | 92 ++++++++++++++++++++++++++++ pkg/oci/backfill.go | 128 +++++++++++++++++++++++++++++++++++++++ pkg/oci/backfill_test.go | 103 +++++++++++++++++++++++++++++++ pkg/oci/delta.go | 8 ++- pkg/oci/oci.go | 2 +- 7 files changed, 346 insertions(+), 5 deletions(-) create mode 100644 pkg/oci/backfill.go create mode 100644 pkg/oci/backfill_test.go diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 460b999..0be883c 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -73,7 +73,8 @@ Plumbing commands are designed to be run programmatically (e.g., inside CI matri * **`aetherpak build-site`**: Reads a tree of cell execution records, fetches the active landing page index, merges new records, and writes static Pages assets. * **`aetherpak plan`**: Analyzes repository git differences since a base commit and expands the `aetherpak.yaml` configuration into a parallelizable build/publish matrix. * **`aetherpak inspect-repo`**: Resolves information (commits, metadata, refs) directly from a local OSTree repository. -* **`aetherpak deltas prune`**: Deletes `_delta-*` and `_deltasrc-*` tags whose manifest returns a definitive 404 and rebuilds `_deltaindex` from the remaining tags. Transient errors leave tags untouched. +* **`aetherpak deltas prune`**: Deletes `_delta-*` tags whose target image is gone and `_deltasrc-*` tags no surviving delta or application tag references, then rebuilds `_deltaindex`. Transient errors leave tags untouched. +* **`aetherpak deltas backfill`**: Generates delta layers between existing image digests and republishes the delta manifest and index entry without pushing the images again. --- diff --git a/README.md b/README.md index 8b69a6c..e2f3162 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,7 @@ How it behaves: * Current limitation: only single-layer images are supported. Multi-layer images are skipped. Extension refs (for example `.Locale`) are not listed as apps and therefore inherit `defaults`/global rather than the base app's value. * `_deltaindex` is eventually consistent during a parallel release. If a delta publish is interrupted, the delta can be missing for that release and is repaired by a later push; the image and its record are written before any delta work starts. * Prune stale tags with `aetherpak deltas prune --dry-run`, then without `--dry-run`. It removes `_delta-*` tags whose target image is gone, and `_deltasrc-*` tags no surviving delta or application tag references, then rebuilds `_deltaindex` from the remaining tags. Auth, network, and other transient errors leave tags untouched. It needs registry write credentials (`--dry-run` only needs read access) and should run as a serialized maintenance step, not from parallel push jobs. +* Repair a missing or lost delta with `aetherpak deltas backfill --to-digest --from-digest ` (repeat `--from-digest` for multiple sources). It generates the layers from existing images and republishes the delta manifest and index entry without pushing the images again. Requirements: a Flatpak build with OCI remote support and zstd (validated with Flatpak 1.18.x). Older clients, or clients on releases older than the configured `delta_back` window, simply download the full layer. @@ -562,6 +563,20 @@ Options: Run it as a serialized maintenance step with registry write credentials. It only removes tags whose manifest is definitively gone or whose source is no longer referenced; transient errors leave them in place. +#### `deltas backfill` +Generates and publishes deltas for an existing target image from existing sources: +```bash +aetherpak deltas backfill --registry ghcr.io --oci-repository my-org/my-app \ + --to-digest sha256: --from-digest sha256: +``` +Options: +* `--registry `, `--oci-repository `: Target registry and repository. Default to configuration or the SCM convention. +* `--to-digest `: Target image manifest digest. Required. +* `--from-digest `: Source image manifest digest. Repeat for multiple sources. +* `--insecure`: Allow an HTTP registry connection. +* `--dry-run`: Generate the delta but do not publish it or retain source tags. +* `--output-file `: Write `target-digest`, `layers`, `smallest-bytes`, and `published` as dotenv output. + #### `build-site` Downloads old static index, merges recent cell records, and regenerates index listings: ```bash diff --git a/cmd/deltas.go b/cmd/deltas.go index a1a966a..2c501fa 100644 --- a/cmd/deltas.go +++ b/cmd/deltas.go @@ -17,6 +17,14 @@ var ( deltasPruneInsecure bool deltasPruneDryRun bool deltasPruneOutputFile string + + deltasBackfillRegistry string + deltasBackfillOCIRepository string + deltasBackfillInsecure bool + deltasBackfillDryRun bool + deltasBackfillOutputFile string + deltasBackfillToDigest string + deltasBackfillFromDigests []string ) var deltasCmd = &cobra.Command{ @@ -87,13 +95,97 @@ var deltasPruneCmd = &cobra.Command{ }, } +var deltasBackfillCmd = &cobra.Command{ + Use: "backfill", + Short: "Generate and publish deltas for an existing target image", + Long: `Generates tar-diff layers from one or more existing source images to an existing target image and publishes the delta manifest and index entry. Use it to repair a missing or lost delta without pushing the images again.`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := LoadConfig() + if err != nil { + return NewCmdErrorf(2, "Configuration error: %w", err) + } + + registry := deltasBackfillRegistry + if registry == "" { + registry = cfg.Registry + } + if registry == "" { + registry = scm.Registry() + } + if registry == "" { + return NewCmdError(2, fmt.Errorf("registry is required")) + } + + ociRepository := deltasBackfillOCIRepository + if ociRepository == "" { + ociRepository = cfg.OCIRepository + } + if ociRepository == "" { + ociRepository = scm.OCIRepository() + } + if ociRepository == "" { + return NewCmdError(2, fmt.Errorf("oci repository is required")) + } + + if deltasBackfillToDigest == "" { + return NewCmdError(2, fmt.Errorf("--to-digest is required")) + } + if len(deltasBackfillFromDigests) == 0 { + return NewCmdError(2, fmt.Errorf("at least one --from-digest is required")) + } + + res, err := oci.BackfillDeltas(oci.BackfillOptions{ + Registry: registry, + OCIRepository: ociRepository, + Insecure: deltasBackfillInsecure, + OCIUsername: viper.GetString("oci_username"), + OCIPassword: viper.GetString("oci_password"), + ToDigest: deltasBackfillToDigest, + FromDigests: deltasBackfillFromDigests, + WorkDir: ".", + DryRun: deltasBackfillDryRun, + }) + if err != nil { + return NewCmdError(1, err) + } + + if err := ciout.Emit(deltasBackfillOutputFile, []ciout.KV{ + {Key: "target-digest", Value: res.TargetDigest}, + {Key: "layers", Value: fmt.Sprintf("%d", res.Layers)}, + {Key: "smallest-bytes", Value: fmt.Sprintf("%d", res.SmallestBytes)}, + {Key: "published", Value: fmt.Sprintf("%t", res.Published)}, + }); err != nil { + return NewCmdError(1, err) + } + + if res.Layers == 0 { + logger.SuccessBanner("Delta Backfill Completed", fmt.Sprintf("No smaller delta was needed for %s.", res.TargetDigest)) + } else if deltasBackfillDryRun { + logger.SuccessBanner("Delta Backfill Completed", fmt.Sprintf("Would publish %d layer(s) for %s.", res.Layers, res.TargetDigest)) + } else { + logger.SuccessBanner("Delta Backfill Completed", fmt.Sprintf("Published %d layer(s) for %s.", res.Layers, res.TargetDigest)) + } + return nil + }, +} + func init() { RootCmd.AddCommand(deltasCmd) deltasCmd.AddCommand(deltasPruneCmd) + deltasCmd.AddCommand(deltasBackfillCmd) deltasPruneCmd.Flags().StringVar(&deltasPruneRegistry, "registry", "", "target OCI registry host") deltasPruneCmd.Flags().StringVar(&deltasPruneOCIRepository, "oci-repository", "", "target repository path/name") deltasPruneCmd.Flags().BoolVar(&deltasPruneInsecure, "insecure", false, "allow connection to an insecure OCI registry (HTTP)") deltasPruneCmd.Flags().BoolVar(&deltasPruneDryRun, "dry-run", false, "report what would be removed without deleting anything") deltasPruneCmd.Flags().StringVar(&deltasPruneOutputFile, "output-file", "", "write resolved outputs as dotenv KEY=VALUE (- or empty = stdout)") + + deltasBackfillCmd.Flags().StringVar(&deltasBackfillRegistry, "registry", "", "target OCI registry host") + deltasBackfillCmd.Flags().StringVar(&deltasBackfillOCIRepository, "oci-repository", "", "target repository path/name") + deltasBackfillCmd.Flags().BoolVar(&deltasBackfillInsecure, "insecure", false, "allow connection to an insecure OCI registry (HTTP)") + deltasBackfillCmd.Flags().BoolVar(&deltasBackfillDryRun, "dry-run", false, "generate the delta but do not publish it") + deltasBackfillCmd.Flags().StringVar(&deltasBackfillToDigest, "to-digest", "", "target image manifest digest") + deltasBackfillCmd.Flags().StringSliceVar(&deltasBackfillFromDigests, "from-digest", nil, "source image manifest digest (repeatable)") + deltasBackfillCmd.Flags().StringVar(&deltasBackfillOutputFile, "output-file", "", "write resolved outputs as dotenv KEY=VALUE (- or empty = stdout)") } diff --git a/pkg/oci/backfill.go b/pkg/oci/backfill.go new file mode 100644 index 0000000..67136f1 --- /dev/null +++ b/pkg/oci/backfill.go @@ -0,0 +1,128 @@ +package oci + +import ( + "fmt" + "strings" + + "github.com/aetherpak/aetherpak/pkg/logger" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// BackfillOptions configures a delta backfill or repair from explicit image +// manifest digests. +type BackfillOptions struct { + Registry string + OCIRepository string + Insecure bool + OCIUsername string + OCIPassword string + ToDigest string + FromDigests []string + WorkDir string + DryRun bool +} + +// BackfillResult reports what a backfill pass generated. +type BackfillResult struct { + TargetDigest string + Layers int + SmallestBytes int64 + Published bool +} + +// BackfillDeltas generates and publishes deltas for an existing target image +// from one or more existing source images. It does not push the images again +// and is meant to repair a missing or lost delta manifest. +func BackfillDeltas(opts BackfillOptions) (BackfillResult, error) { + var result BackfillResult + if opts.ToDigest == "" { + return result, fmt.Errorf("to digest is required") + } + if len(opts.FromDigests) == 0 { + return result, fmt.Errorf("at least one from digest is required") + } + + var repoOpts []name.Option + if opts.Insecure { + repoOpts = append(repoOpts, name.Insecure) + } + repoName := opts.Registry + "/" + opts.OCIRepository + repoRef, err := name.NewRepository(repoName, repoOpts...) + if err != nil { + return result, fmt.Errorf("invalid repository path %q: %w", repoName, err) + } + authOpt := resolveAuth(opts.OCIUsername, opts.OCIPassword) + + toHash, err := parseDigest(opts.ToDigest) + if err != nil { + return result, fmt.Errorf("invalid to digest: %w", err) + } + result.TargetDigest = toHash.String() + toRef := repoRef.Digest(toHash.String()) + newImg, err := remote.Image(toRef, authOpt) + if err != nil { + return result, fmt.Errorf("failed to read target image %s: %w", toHash, err) + } + cfg, err := newImg.ConfigFile() + if err != nil { + return result, fmt.Errorf("failed to read target image config: %w", err) + } + + sources := make([]deltaSource, 0, len(opts.FromDigests)) + seen := map[v1.Hash]bool{} + for _, value := range opts.FromDigests { + fromHash, err := parseDigest(value) + if err != nil { + return result, fmt.Errorf("invalid from digest %q: %w", value, err) + } + if fromHash == toHash || seen[fromHash] { + continue + } + seen[fromHash] = true + sources = append(sources, deltaSource{ + ref: repoRef.Digest(fromHash.String()), + desc: &v1.Descriptor{MediaType: types.OCIManifestSchema1, Digest: fromHash}, + }) + } + if len(sources) == 0 { + return result, fmt.Errorf("no usable from digests remain after de-duplication") + } + + delta, err := prepareImageDeltas(repoRef, sources, cfg.Architecture, newImg, toHash, authOpt, opts.WorkDir, !opts.DryRun) + if err != nil { + return result, err + } + if delta == nil { + logger.Info("No smaller delta was generated for %s.", toHash) + return result, nil + } + defer delta.Close() + + result.Layers = len(delta.layers) + if len(delta.layers) > 0 { + result.SmallestBytes = delta.layers[0].size + } + if opts.DryRun { + logger.Info("[DRY RUN] Would publish %d delta layer(s) for %s.", result.Layers, toHash) + return result, nil + } + if err := publishImageDelta(repoRef, authOpt, delta); err != nil { + return result, err + } + result.Published = true + return result, nil +} + +func parseDigest(value string) (v1.Hash, error) { + value = strings.TrimSpace(value) + if value == "" { + return v1.Hash{}, fmt.Errorf("empty digest") + } + if !strings.Contains(value, ":") { + value = "sha256:" + value + } + return v1.NewHash(value) +} diff --git a/pkg/oci/backfill_test.go b/pkg/oci/backfill_test.go new file mode 100644 index 0000000..a15ec2c --- /dev/null +++ b/pkg/oci/backfill_test.go @@ -0,0 +1,103 @@ +package oci + +import ( + "net/http/httptest" + "strings" + "testing" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/remote" +) + +func TestBackfillDeltasPublishesExistingImages(t *testing.T) { + regServer := httptest.NewServer(registry.New()) + defer regServer.Close() + host := strings.TrimPrefix(regServer.URL, "http://") + recordsDir := t.TempDir() + + res1 := pushTestImageBack(t, host, "org.example.app", recordsDir, "version one", false, true, 1) + res2 := pushTestImageBack(t, host, "org.example.app", recordsDir, "version two", false, true, 1) + + res, err := BackfillDeltas(BackfillOptions{ + Registry: host, + OCIRepository: "org.example.app", + Insecure: true, + ToDigest: res2.Digest, + FromDigests: []string{res1.Digest}, + WorkDir: t.TempDir(), + }) + if err != nil { + t.Fatalf("backfill failed: %v", err) + } + if !res.Published || res.Layers != 1 { + t.Fatalf("backfill result = %+v, want one published layer", res) + } + + repoRef, err := name.NewRepository(host+"/org.example.app", name.Insecure) + if err != nil { + t.Fatal(err) + } + auth := remote.WithAuth(authn.Anonymous) + + manifest := fetchDeltaManifest(t, repoRef, res2.Digest, auth) + if len(manifest.Layers) != 1 { + t.Fatalf("expected one delta layer, got %d", len(manifest.Layers)) + } + if got := manifest.Layers[0].Annotations[deltaFromImageAnnotation]; got != res1.Digest { + t.Fatalf("from-image = %q, want %q", got, res1.Digest) + } + if got := manifest.Annotations[deltaTargetAnnotation]; got != res2.Digest { + t.Fatalf("target = %q, want %q", got, res2.Digest) + } + + index, err := remote.Index(repoRef.Tag(deltaIndexTag), auth) + if err != nil { + t.Fatal(err) + } + im, err := index.IndexManifest() + if err != nil { + t.Fatal(err) + } + if len(im.Manifests) != 1 || im.Manifests[0].Annotations[deltaTargetAnnotation] != res2.Digest { + t.Fatalf("unexpected delta index: %+v", im.Manifests) + } +} + +func TestBackfillDeltasDryRunDoesNotPublish(t *testing.T) { + regServer := httptest.NewServer(registry.New()) + defer regServer.Close() + host := strings.TrimPrefix(regServer.URL, "http://") + recordsDir := t.TempDir() + + res1 := pushTestImageBack(t, host, "org.example.app", recordsDir, "version one", false, true, 1) + res2 := pushTestImageBack(t, host, "org.example.app", recordsDir, "version two", false, true, 1) + + res, err := BackfillDeltas(BackfillOptions{ + Registry: host, + OCIRepository: "org.example.app", + Insecure: true, + ToDigest: res2.Digest, + FromDigests: []string{res1.Digest}, + WorkDir: t.TempDir(), + DryRun: true, + }) + if err != nil { + t.Fatal(err) + } + if res.Published || res.Layers != 1 { + t.Fatalf("dry-run result = %+v, want one generated layer and no publish", res) + } + + repoRef, err := name.NewRepository(host+"/org.example.app", name.Insecure) + if err != nil { + t.Fatal(err) + } + if _, err := remote.Head(repoRef.Tag(deltaManifestTag(mustHash(t, res2.Digest))), remote.WithAuth(authn.Anonymous)); err == nil { + t.Fatalf("dry run published a delta manifest") + } + if _, err := remote.Head(repoRef.Tag(deltaSourceTag(mustHash(t, res1.Digest))), remote.WithAuth(authn.Anonymous)); err == nil { + t.Fatalf("dry run retained a source tag") + } +} diff --git a/pkg/oci/delta.go b/pkg/oci/delta.go index 4a4290e..5b12689 100644 --- a/pkg/oci/delta.go +++ b/pkg/oci/delta.go @@ -267,7 +267,7 @@ func containsHash(values []v1.Hash, want v1.Hash) bool { // prepareImageDeltas generates one delta layer per source. It returns nil when // no delta is smaller than the full layer. -func prepareImageDeltas(repoRef name.Repository, sources []deltaSource, arch string, newImg v1.Image, newDigest v1.Hash, authOpt remote.Option, workDir string) (*imageDelta, error) { +func prepareImageDeltas(repoRef name.Repository, sources []deltaSource, arch string, newImg v1.Image, newDigest v1.Hash, authOpt remote.Option, workDir string, retainSources bool) (*imageDelta, error) { newCfg, err := newImg.ConfigFile() if err != nil { return nil, fmt.Errorf("failed to read target image config: %w", err) @@ -333,8 +333,10 @@ func prepareImageDeltas(repoRef name.Repository, sources []deltaSource, arch str logger.Warn("WARNING: failed to read delta source %s: %v", source.desc.Digest, err) continue } - if err := retainDeltaSource(repoRef, oldImg, source.desc.Digest, authOpt); err != nil { - logger.Warn("WARNING: could not retain delta source %s: %v", source.desc.Digest, err) + if retainSources { + if err := retainDeltaSource(repoRef, oldImg, source.desc.Digest, authOpt); err != nil { + logger.Warn("WARNING: could not retain delta source %s: %v", source.desc.Digest, err) + } } oldCfg, err := oldImg.ConfigFile() if err != nil { diff --git a/pkg/oci/oci.go b/pkg/oci/oci.go index c2f4d80..3266298 100644 --- a/pkg/oci/oci.go +++ b/pkg/oci/oci.go @@ -324,7 +324,7 @@ func Push(opts PushOptions) (PushResult, error) { // 6. Publish the optional delta only after the image and its record are // durable. A slow or failed delta must never block the release contract. if opts.ImageDeltas && len(deltaSources) > 0 { - delta, derr := prepareImageDeltas(repoRef, deltaSources, configFile.Architecture, img, digest, authOpt, opts.RecordsDir) + delta, derr := prepareImageDeltas(repoRef, deltaSources, configFile.Architecture, img, digest, authOpt, opts.RecordsDir, true) if derr != nil { logger.Warn("WARNING: failed to prepare image delta: %v (clients will download the full layer).", derr) } else if delta != nil {