feat(oci): backfill image deltas from digests

ARCHITECTURE.md

@@ -73,7 +73,8 @@ Plumbing commands are designed to be run programmatically (e.g., inside CI matri
 * **`aetherpak build-site`**: Reads a tree of cell execution records, fetches the active landing page index, merges new records, and writes static Pages assets.
 * **`aetherpak plan`**: Analyzes repository git differences since a base commit and expands the `aetherpak.yaml` configuration into a parallelizable build/publish matrix.
 * **`aetherpak inspect-repo`**: Resolves information (commits, metadata, refs) directly from a local OSTree repository.
-* **`aetherpak deltas prune`**: Deletes `_delta-*` and `_deltasrc-*` tags whose manifest returns a definitive 404 and rebuilds `_deltaindex` from the remaining tags. Transient errors leave tags untouched.
+* **`aetherpak deltas prune`**: Deletes `_delta-*` tags whose target image is gone and `_deltasrc-*` tags no surviving delta or application tag references, then rebuilds `_deltaindex`. Transient errors leave tags untouched.
+* **`aetherpak deltas backfill`**: Generates delta layers between existing image digests and republishes the delta manifest and index entry without pushing the images again.
 
 ---
 

README.md

@@ -229,6 +229,7 @@ How it behaves:
 * Current limitation: only single-layer images are supported. Multi-layer images are skipped. Extension refs (for example `.Locale`) are not listed as apps and therefore inherit `defaults`/global rather than the base app's value.
 * `_deltaindex` is eventually consistent during a parallel release. If a delta publish is interrupted, the delta can be missing for that release and is repaired by a later push; the image and its record are written before any delta work starts.
 * Prune stale tags with `aetherpak deltas prune --dry-run`, then without `--dry-run`. It removes `_delta-*` tags whose target image is gone, and `_deltasrc-*` tags no surviving delta or application tag references, then rebuilds `_deltaindex` from the remaining tags. Auth, network, and other transient errors leave tags untouched. It needs registry write credentials (`--dry-run` only needs read access) and should run as a serialized maintenance step, not from parallel push jobs.
+* Repair a missing or lost delta with `aetherpak deltas backfill --to-digest <target> --from-digest <source>` (repeat `--from-digest` for multiple sources). It generates the layers from existing images and republishes the delta manifest and index entry without pushing the images again.
 
 Requirements: a Flatpak build with OCI remote support and zstd (validated with Flatpak 1.18.x). Older clients, or clients on releases older than the configured `delta_back` window, simply download the full layer.
 
@@ -562,6 +563,20 @@ Options:
 
 Run it as a serialized maintenance step with registry write credentials. It only removes tags whose manifest is definitively gone or whose source is no longer referenced; transient errors leave them in place.
 
+#### `deltas backfill`
+Generates and publishes deltas for an existing target image from existing sources:
+```bash
+aetherpak deltas backfill --registry ghcr.io --oci-repository my-org/my-app \
+  --to-digest sha256:<target> --from-digest sha256:<source>
+```
+Options:
+* `--registry <host>`, `--oci-repository <path>`: Target registry and repository. Default to configuration or the SCM convention.
+* `--to-digest <digest>`: Target image manifest digest. Required.
+* `--from-digest <digest>`: Source image manifest digest. Repeat for multiple sources.
+* `--insecure`: Allow an HTTP registry connection.
+* `--dry-run`: Generate the delta but do not publish it or retain source tags.
+* `--output-file <path>`: Write `target-digest`, `layers`, `smallest-bytes`, and `published` as dotenv output.
+
 #### `build-site`
 Downloads old static index, merges recent cell records, and regenerates index listings:
 ```bash

cmd/deltas.go

@@ -17,6 +17,14 @@ var (
 	deltasPruneInsecure      bool
 	deltasPruneDryRun        bool
 	deltasPruneOutputFile    string
+
+	deltasBackfillRegistry      string
+	deltasBackfillOCIRepository string
+	deltasBackfillInsecure      bool
+	deltasBackfillDryRun        bool
+	deltasBackfillOutputFile    string
+	deltasBackfillToDigest      string
+	deltasBackfillFromDigests   []string
 )
 
 var deltasCmd = &cobra.Command{
@@ -87,13 +95,97 @@ var deltasPruneCmd = &cobra.Command{
 	},
 }
 
+var deltasBackfillCmd = &cobra.Command{
+	Use:   "backfill",
+	Short: "Generate and publish deltas for an existing target image",
+	Long:  `Generates tar-diff layers from one or more existing source images to an existing target image and publishes the delta manifest and index entry. Use it to repair a missing or lost delta without pushing the images again.`,
+	Args:  cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		cfg, err := LoadConfig()
+		if err != nil {
+			return NewCmdErrorf(2, "Configuration error: %w", err)
+		}
+
+		registry := deltasBackfillRegistry
+		if registry == "" {
+			registry = cfg.Registry
+		}
+		if registry == "" {
+			registry = scm.Registry()
+		}
+		if registry == "" {
+			return NewCmdError(2, fmt.Errorf("registry is required"))
+		}
+
+		ociRepository := deltasBackfillOCIRepository
+		if ociRepository == "" {
+			ociRepository = cfg.OCIRepository
+		}
+		if ociRepository == "" {
+			ociRepository = scm.OCIRepository()
+		}
+		if ociRepository == "" {
+			return NewCmdError(2, fmt.Errorf("oci repository is required"))
+		}
+
+		if deltasBackfillToDigest == "" {
+			return NewCmdError(2, fmt.Errorf("--to-digest is required"))
+		}
+		if len(deltasBackfillFromDigests) == 0 {
+			return NewCmdError(2, fmt.Errorf("at least one --from-digest is required"))
+		}
+
+		res, err := oci.BackfillDeltas(oci.BackfillOptions{
+			Registry:      registry,
+			OCIRepository: ociRepository,
+			Insecure:      deltasBackfillInsecure,
+			OCIUsername:   viper.GetString("oci_username"),
+			OCIPassword:   viper.GetString("oci_password"),
+			ToDigest:      deltasBackfillToDigest,
+			FromDigests:   deltasBackfillFromDigests,
+			WorkDir:       ".",
+			DryRun:        deltasBackfillDryRun,
+		})
+		if err != nil {
+			return NewCmdError(1, err)
+		}
+
+		if err := ciout.Emit(deltasBackfillOutputFile, []ciout.KV{
+			{Key: "target-digest", Value: res.TargetDigest},
+			{Key: "layers", Value: fmt.Sprintf("%d", res.Layers)},
+			{Key: "smallest-bytes", Value: fmt.Sprintf("%d", res.SmallestBytes)},
+			{Key: "published", Value: fmt.Sprintf("%t", res.Published)},
+		}); err != nil {
+			return NewCmdError(1, err)
+		}
+
+		if res.Layers == 0 {
+			logger.SuccessBanner("Delta Backfill Completed", fmt.Sprintf("No smaller delta was needed for %s.", res.TargetDigest))
+		} else if deltasBackfillDryRun {
+			logger.SuccessBanner("Delta Backfill Completed", fmt.Sprintf("Would publish %d layer(s) for %s.", res.Layers, res.TargetDigest))
+		} else {
+			logger.SuccessBanner("Delta Backfill Completed", fmt.Sprintf("Published %d layer(s) for %s.", res.Layers, res.TargetDigest))
+		}
+		return nil
+	},
+}
+
 func init() {
 	RootCmd.AddCommand(deltasCmd)
 	deltasCmd.AddCommand(deltasPruneCmd)
+	deltasCmd.AddCommand(deltasBackfillCmd)
 
 	deltasPruneCmd.Flags().StringVar(&deltasPruneRegistry, "registry", "", "target OCI registry host")
 	deltasPruneCmd.Flags().StringVar(&deltasPruneOCIRepository, "oci-repository", "", "target repository path/name")
 	deltasPruneCmd.Flags().BoolVar(&deltasPruneInsecure, "insecure", false, "allow connection to an insecure OCI registry (HTTP)")
 	deltasPruneCmd.Flags().BoolVar(&deltasPruneDryRun, "dry-run", false, "report what would be removed without deleting anything")
 	deltasPruneCmd.Flags().StringVar(&deltasPruneOutputFile, "output-file", "", "write resolved outputs as dotenv KEY=VALUE (- or empty = stdout)")
+
+	deltasBackfillCmd.Flags().StringVar(&deltasBackfillRegistry, "registry", "", "target OCI registry host")
+	deltasBackfillCmd.Flags().StringVar(&deltasBackfillOCIRepository, "oci-repository", "", "target repository path/name")
+	deltasBackfillCmd.Flags().BoolVar(&deltasBackfillInsecure, "insecure", false, "allow connection to an insecure OCI registry (HTTP)")
+	deltasBackfillCmd.Flags().BoolVar(&deltasBackfillDryRun, "dry-run", false, "generate the delta but do not publish it")
+	deltasBackfillCmd.Flags().StringVar(&deltasBackfillToDigest, "to-digest", "", "target image manifest digest")
+	deltasBackfillCmd.Flags().StringSliceVar(&deltasBackfillFromDigests, "from-digest", nil, "source image manifest digest (repeatable)")
+	deltasBackfillCmd.Flags().StringVar(&deltasBackfillOutputFile, "output-file", "", "write resolved outputs as dotenv KEY=VALUE (- or empty = stdout)")
 }

pkg/oci/backfill.go

@@ -0,0 +1,128 @@
+package oci
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/aetherpak/aetherpak/pkg/logger"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+	"github.com/google/go-containerregistry/pkg/v1/types"
+)
+
+// BackfillOptions configures a delta backfill or repair from explicit image
+// manifest digests.
+type BackfillOptions struct {
+	Registry      string
+	OCIRepository string
+	Insecure      bool
+	OCIUsername   string
+	OCIPassword   string
+	ToDigest      string
+	FromDigests   []string
+	WorkDir       string
+	DryRun        bool
+}
+
+// BackfillResult reports what a backfill pass generated.
+type BackfillResult struct {
+	TargetDigest  string
+	Layers        int
+	SmallestBytes int64
+	Published     bool
+}
+
+// BackfillDeltas generates and publishes deltas for an existing target image
+// from one or more existing source images. It does not push the images again
+// and is meant to repair a missing or lost delta manifest.
+func BackfillDeltas(opts BackfillOptions) (BackfillResult, error) {
+	var result BackfillResult
+	if opts.ToDigest == "" {
+		return result, fmt.Errorf("to digest is required")
+	}
+	if len(opts.FromDigests) == 0 {
+		return result, fmt.Errorf("at least one from digest is required")
+	}
+
+	var repoOpts []name.Option
+	if opts.Insecure {
+		repoOpts = append(repoOpts, name.Insecure)
+	}
+	repoName := opts.Registry + "/" + opts.OCIRepository
+	repoRef, err := name.NewRepository(repoName, repoOpts...)
+	if err != nil {
+		return result, fmt.Errorf("invalid repository path %q: %w", repoName, err)
+	}
+	authOpt := resolveAuth(opts.OCIUsername, opts.OCIPassword)
+
+	toHash, err := parseDigest(opts.ToDigest)
+	if err != nil {
+		return result, fmt.Errorf("invalid to digest: %w", err)
+	}
+	result.TargetDigest = toHash.String()
+	toRef := repoRef.Digest(toHash.String())
+	newImg, err := remote.Image(toRef, authOpt)
+	if err != nil {
+		return result, fmt.Errorf("failed to read target image %s: %w", toHash, err)
+	}
+	cfg, err := newImg.ConfigFile()
+	if err != nil {
+		return result, fmt.Errorf("failed to read target image config: %w", err)
+	}
+
+	sources := make([]deltaSource, 0, len(opts.FromDigests))
+	seen := map[v1.Hash]bool{}
+	for _, value := range opts.FromDigests {
+		fromHash, err := parseDigest(value)
+		if err != nil {
+			return result, fmt.Errorf("invalid from digest %q: %w", value, err)
+		}
+		if fromHash == toHash || seen[fromHash] {
+			continue
+		}
+		seen[fromHash] = true
+		sources = append(sources, deltaSource{
+			ref:  repoRef.Digest(fromHash.String()),
+			desc: &v1.Descriptor{MediaType: types.OCIManifestSchema1, Digest: fromHash},
+		})
+	}
+	if len(sources) == 0 {
+		return result, fmt.Errorf("no usable from digests remain after de-duplication")
+	}
+
+	delta, err := prepareImageDeltas(repoRef, sources, cfg.Architecture, newImg, toHash, authOpt, opts.WorkDir, !opts.DryRun)
+	if err != nil {
+		return result, err
+	}
+	if delta == nil {
+		logger.Info("No smaller delta was generated for %s.", toHash)
+		return result, nil
+	}
+	defer delta.Close()
+
+	result.Layers = len(delta.layers)
+	if len(delta.layers) > 0 {
+		result.SmallestBytes = delta.layers[0].size
+	}
+	if opts.DryRun {
+		logger.Info("[DRY RUN] Would publish %d delta layer(s) for %s.", result.Layers, toHash)
+		return result, nil
+	}
+	if err := publishImageDelta(repoRef, authOpt, delta); err != nil {
+		return result, err
+	}
+	result.Published = true
+	return result, nil
+}
+
+func parseDigest(value string) (v1.Hash, error) {
+	value = strings.TrimSpace(value)
+	if value == "" {
+		return v1.Hash{}, fmt.Errorf("empty digest")
+	}
+	if !strings.Contains(value, ":") {
+		value = "sha256:" + value
+	}
+	return v1.NewHash(value)
+}

pkg/oci/backfill_test.go

@@ -0,0 +1,103 @@
+package oci
+
+import (
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/registry"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+)
+
+func TestBackfillDeltasPublishesExistingImages(t *testing.T) {
+	regServer := httptest.NewServer(registry.New())
+	defer regServer.Close()
+	host := strings.TrimPrefix(regServer.URL, "http://")
+	recordsDir := t.TempDir()
+
+	res1 := pushTestImageBack(t, host, "org.example.app", recordsDir, "version one", false, true, 1)
+	res2 := pushTestImageBack(t, host, "org.example.app", recordsDir, "version two", false, true, 1)
+
+	res, err := BackfillDeltas(BackfillOptions{
+		Registry:      host,
+		OCIRepository: "org.example.app",
+		Insecure:      true,
+		ToDigest:      res2.Digest,
+		FromDigests:   []string{res1.Digest},
+		WorkDir:       t.TempDir(),
+	})
+	if err != nil {
+		t.Fatalf("backfill failed: %v", err)
+	}
+	if !res.Published || res.Layers != 1 {
+		t.Fatalf("backfill result = %+v, want one published layer", res)
+	}
+
+	repoRef, err := name.NewRepository(host+"/org.example.app", name.Insecure)
+	if err != nil {
+		t.Fatal(err)
+	}
+	auth := remote.WithAuth(authn.Anonymous)
+
+	manifest := fetchDeltaManifest(t, repoRef, res2.Digest, auth)
+	if len(manifest.Layers) != 1 {
+		t.Fatalf("expected one delta layer, got %d", len(manifest.Layers))
+	}
+	if got := manifest.Layers[0].Annotations[deltaFromImageAnnotation]; got != res1.Digest {
+		t.Fatalf("from-image = %q, want %q", got, res1.Digest)
+	}
+	if got := manifest.Annotations[deltaTargetAnnotation]; got != res2.Digest {
+		t.Fatalf("target = %q, want %q", got, res2.Digest)
+	}
+
+	index, err := remote.Index(repoRef.Tag(deltaIndexTag), auth)
+	if err != nil {
+		t.Fatal(err)
+	}
+	im, err := index.IndexManifest()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(im.Manifests) != 1 || im.Manifests[0].Annotations[deltaTargetAnnotation] != res2.Digest {
+		t.Fatalf("unexpected delta index: %+v", im.Manifests)
+	}
+}
+
+func TestBackfillDeltasDryRunDoesNotPublish(t *testing.T) {
+	regServer := httptest.NewServer(registry.New())
+	defer regServer.Close()
+	host := strings.TrimPrefix(regServer.URL, "http://")
+	recordsDir := t.TempDir()
+
+	res1 := pushTestImageBack(t, host, "org.example.app", recordsDir, "version one", false, true, 1)
+	res2 := pushTestImageBack(t, host, "org.example.app", recordsDir, "version two", false, true, 1)
+
+	res, err := BackfillDeltas(BackfillOptions{
+		Registry:      host,
+		OCIRepository: "org.example.app",
+		Insecure:      true,
+		ToDigest:      res2.Digest,
+		FromDigests:   []string{res1.Digest},
+		WorkDir:       t.TempDir(),
+		DryRun:        true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.Published || res.Layers != 1 {
+		t.Fatalf("dry-run result = %+v, want one generated layer and no publish", res)
+	}
+
+	repoRef, err := name.NewRepository(host+"/org.example.app", name.Insecure)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := remote.Head(repoRef.Tag(deltaManifestTag(mustHash(t, res2.Digest))), remote.WithAuth(authn.Anonymous)); err == nil {
+		t.Fatalf("dry run published a delta manifest")
+	}
+	if _, err := remote.Head(repoRef.Tag(deltaSourceTag(mustHash(t, res1.Digest))), remote.WithAuth(authn.Anonymous)); err == nil {
+		t.Fatalf("dry run retained a source tag")
+	}
+}