fix: a run of many files no longer runs out of memory while it saves its manifest

CHANGELOG.md

@@ -43,6 +43,13 @@ because it turns other people's test suites red.
 
 ### Fixed
 
+- **A run of many files no longer runs out of memory while it saves its
+  manifest.** The manifest was put together in memory in one piece, about
+  ten times its own size, after the last file. In a container with 1 GB a
+  run of 200 000 text files was killed at that moment and left its files
+  with no manifest, so `tfg cleanup` could not remove them. The manifest is
+  now written entry by entry, with the same bytes, and that run finishes.
+  Planning a large run also takes less memory.
 - **Ctrl+C stops a run that draws its sizes from a range.** With
   `--size-range`, every file was tried at its drawn size before anything else,
   and that part did not look at Ctrl+C - three thousand JPG files went on for

internal/engine/engine.go

@@ -450,6 +450,11 @@ func PlanContext(ctx context.Context, targets []Target, opt Options) ([]PlannedF
 			Remedy: core.Says("engine.NoOutDirFix", "Name a directory, for example ./fixtures")}
 	}
 
+	// Worked out after the budget took its reference point, so the reserved
+	// list is weighed as part of the plan - which is what it is. Taken at the
+	// first file that passes its checks (plantarget.go), not here.
+	pl.reserve = planCapacity(targets, pl.budget)
+
 	for i := range targets {
 		t := &targets[i]
 

internal/engine/planmemory.go

@@ -1,12 +1,47 @@
 package engine
 
 import (
+	"reflect"
 	"runtime"
 	"runtime/metrics"
 
 	"github.com/donislawdev/TestingFilesGenerator/internal/core"
 )
 
+// planCapacity is how long a list to reserve for the plan. It is worked out
+// before the first target is walked and taken at the first file that passes
+// its checks (plantarget.go says why then).
+//
+// Reserved rather than grown since 2026-11-02. Grown by append, a plan of a
+// million text files took 1.9-2.2 GB from the system for 0.94 GB of plan,
+// because every doubling leaves the last array to the collector. Reserved, it
+// took 1.2 GB and five collections instead of fourteen - the plansize probe,
+// two runs each, interleaved (docs/FILE-CEILING-2026-11-02.md section 2).
+//
+// Never more than the run may hold. A count past the file ceiling is not
+// reserved at all: the loop refuses it at the target that crosses the ceiling,
+// and reserving it first would be the very allocation the ceiling is there to
+// prevent. Under a memory setting the reservation stops where the bare list
+// would fill the plan's ceiling, so the reservation alone is never what takes
+// a run past the memory its person allowed - a plan that needs more than that
+// is refused by account as the list grows, as it always was.
+func planCapacity(targets []Target, budget *planMemory) int {
+	n := 0
+	for i := range targets {
+		n += len(targets[i].Sizes)
+		if n > core.MaxFilesPerRun {
+			return 0
+		}
+	}
+	if budget == nil {
+		return n
+	}
+	if fit := budget.ceiling / uint64(reflect.TypeFor[PlannedFile]().Size()); uint64(n) > fit {
+		return int(fit)
+	}
+	return n
+}
+
 // The ceiling on what a plan is allowed to cost.
 //
 // core.MaxFilesPerRun counts files, and the number it settled on was justified

internal/engine/plantarget.go

@@ -25,6 +25,9 @@ type planning struct {
 	names      map[string]nameOwner
 	totalBytes int64
 	budget     *planMemory
+	// reserve is how long a list the plan takes at its first file - see
+	// planCapacity for how long, and startList for when.
+	reserve int
 }
 
 // files plans every file of one target and adds them to the run.
@@ -119,6 +122,11 @@ func (pl *planning) files(ctx context.Context, t *Target, desc format.Descriptor
 				Because: core.SaidOf(err)}
 		}
 
+		// Here, after every check above, and not before the first target -
+		// see startList for why.
+		if err := pl.startList(ctx); err != nil {
+			return err
+		}
 		pl.out = append(pl.out, PlannedFile{
 			ID:     fmt.Sprintf("f_%04d", len(pl.out)+1),
 			Target: t,
@@ -142,3 +150,26 @@ func (pl *planning) files(ctx context.Context, t *Target, desc format.Descriptor
 	}
 	return nil
 }
+
+// startList takes the list of the plan at its first file, as long as
+// planCapacity said, and does nothing at every file after that.
+//
+// Not before the first target. A run refused at its first file - a size its
+// format cannot make - is refused in tens of megabytes, and reserving first
+// took a million such files to 450 MB before saying no (review of #212,
+// measured 2026-11-02: 27-30 MB against 441-450 MB, two runs each). Ctrl+C is
+// asked first for the same reason: a run that is being stopped is not given
+// the largest allocation it would make.
+//
+// A method rather than a block inside files, because there it was the
+// fiftieth function nested three deep, and that ceiling is a ratchet.
+func (pl *planning) startList(ctx context.Context) error {
+	if pl.out != nil {
+		return nil
+	}
+	if err := ctx.Err(); err != nil {
+		return err
+	}
+	pl.out = make([]PlannedFile, 0, max(1, pl.reserve))
+	return nil
+}

internal/guard/manifestreach_test.go

@@ -70,17 +70,11 @@ func TestAnOrdinaryRunSaysNothingAboutTheManifestCeiling(t *testing.T) {
 	}
 }
 
-// The length worked out for a manifest is the length Encode writes, to the
-// byte.
-//
-// This replaces a guard that held an estimate to "a quarter either way", and
-// a quarter was wide enough to let a constant 6% low for text and half of what
-// an e-mail entry costs stay green while a run of 20 500 files went unwarned.
-// The answer is counted entry by entry now, so it is held to the byte: a
-// document of no entries, one entry, and many unlike each other - notes, a
-// failure, words outside ASCII, characters JSON escapes, properties nested
-// several deep.
-func TestTheManifestSizeIsTheLengthOfTheManifest(t *testing.T) {
+// manifestSamples are the documents both guards of the manifest's layout ask
+// about - its counted length and its streamed bytes - so neither keeps a
+// second copy of the cases: none, one, and many entries with text that
+// encoding/json escapes, nested properties, notes and damage.
+func manifestSamples() map[string]*manifest.Manifest {
 	documents := map[string]*manifest.Manifest{"no entries": newDocument()}
 
 	one := newDocument()
@@ -99,8 +93,21 @@ func TestTheManifestSizeIsTheLengthOfTheManifest(t *testing.T) {
 	many.Add(manifest.File{ID: "failed", Path: "failed.png", Name: "failed.png", Format: "png", Failed: true,
 		Error: "a generator stopped", Damage: []manifest.Damage{{Type: "zero-head", Settings: map[string]string{"bytes": "4"}}}})
 	documents["many entries"] = many
+	return documents
+}
 
-	for name, m := range documents {
+// The length worked out for a manifest is the length Encode writes, to the
+// byte.
+//
+// This replaces a guard that held an estimate to "a quarter either way", and
+// a quarter was wide enough to let a constant 6% low for text and half of what
+// an e-mail entry costs stay green while a run of 20 500 files went unwarned.
+// The answer is counted entry by entry now, so it is held to the byte: a
+// document of no entries, one entry, and many unlike each other - notes, a
+// failure, words outside ASCII, characters JSON escapes, properties nested
+// several deep.
+func TestTheManifestSizeIsTheLengthOfTheManifest(t *testing.T) {
+	for name, m := range manifestSamples() {
 		counted, err := m.EncodedBytes()
 		if err != nil {
 			t.Fatalf("%s: counting: %v", name, err)

internal/guard/manifeststream_test.go

@@ -0,0 +1,129 @@
+package guard
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/donislawdev/TestingFilesGenerator/internal/manifest"
+)
+
+// The manifest is written an entry at a time, and the bytes are still the ones
+// encoding/json writes.
+//
+// Until 2026-11-02 Encode was one call into encoding/json, which builds the
+// whole document in memory before its first write and indents it into a second
+// copy while every entry is still held. At 100 000 text files saving the
+// manifest peaked at 9.3 KB a file against 4.0 KB for writing every file, and a
+// run of 200 000 in a container of 1 GB was killed after its last file with no
+// manifest written (docs/FILE-CEILING-2026-11-02.md section 8). Encode now
+// streams. Two things can go wrong with that and each half of this guard is
+// about one of them.
+//
+// The bytes. verify, cleanup, the exports and every reader somebody wrote
+// against a manifest read these bytes, and the streamed layout is assembled by
+// hand. So it is compared with encoding/json's own rendering of the same value
+// - not with size.go, which counts the same hand made layout and would agree
+// with a mistake both made the same way.
+func TestTheManifestIsWrittenAsEncodingJSONWritesIt(t *testing.T) {
+	documents := manifestSamples()
+	documents["a list that was never made"] = neverListed(t)
+	documents["two thousand entries"] = manyEntries(2000)
+	for name, m := range documents {
+		var streamed bytes.Buffer
+		if err := m.Encode(&streamed); err != nil {
+			t.Fatalf("%s: Encode: %v", name, err)
+		}
+		if want := encodingJSON(t, m); !bytes.Equal(streamed.Bytes(), want) {
+			at := firstByteApart(streamed.Bytes(), want)
+			t.Errorf("%s: Encode wrote %d B and encoding/json writes %d B, apart from byte %d:\n got: %q\nwant: %q",
+				name, streamed.Len(), len(want), at, bytesAround(streamed.Bytes(), at), bytesAround(want, at))
+		}
+	}
+}
+
+// And the document is never handed over whole. This is the half that says the
+// streaming is there at all: going back to json.NewEncoder writes the same
+// bytes in ONE write, the half above stays green, and this one turns red.
+func TestTheManifestIsNotHeldWholeToBeWritten(t *testing.T) {
+	m := manyEntries(2000)
+	var w writeSizes
+	if err := m.Encode(&w); err != nil {
+		t.Fatalf("Encode: %v", err)
+	}
+	// The question only means something for a document many writes long, so
+	// that is asserted rather than assumed.
+	if w.total < 512<<10 {
+		t.Fatalf("the document of two thousand entries is only %d B - too short to tell a whole write from a streamed one", w.total)
+	}
+	if w.largest*8 > w.total {
+		t.Errorf("a manifest of %d B reached its writer in %d writes, the largest %d B. "+
+			"Written whole, the document sits in memory beside every entry it was made from", w.total, w.writes, w.largest)
+	}
+}
+
+// writeSizes counts what a writer was handed and the largest single piece.
+type writeSizes struct{ total, writes, largest int }
+
+func (w *writeSizes) Write(p []byte) (int, error) {
+	w.total += len(p)
+	w.writes++
+	w.largest = max(w.largest, len(p))
+	return len(p), nil
+}
+
+// encodingJSON is the manifest as encoding/json renders it in one piece, with
+// the indent Encode has always used.
+func encodingJSON(t *testing.T, m *manifest.Manifest) []byte {
+	t.Helper()
+	var b bytes.Buffer
+	enc := json.NewEncoder(&b)
+	enc.SetIndent("", "  ")
+	if err := enc.Encode(m); err != nil {
+		t.Fatalf("encoding/json: %v", err)
+	}
+	return b.Bytes()
+}
+
+// neverListed is a manifest read from a document with no list of files at
+// all, so its list is nil - encoded as null rather than [], and a streamed
+// layout that assumed a list would write the wrong one.
+func neverListed(t *testing.T) *manifest.Manifest {
+	t.Helper()
+	var m manifest.Manifest
+	if err := json.Unmarshal([]byte(`{"manifest_version":"1.0"}`), &m); err != nil {
+		t.Fatalf("decoding: %v", err)
+	}
+	if m.Files != nil {
+		t.Fatalf("the list came back as %v rather than nil, so this case is not the one it names", m.Files)
+	}
+	return &m
+}
+
+// manyEntries is a manifest long enough that a whole write and a streamed one
+// cannot be confused, with entries that carry escaped text and nested values.
+func manyEntries(n int) *manifest.Manifest {
+	m := newDocument()
+	for i := range n {
+		name := fmt.Sprintf("files_%04d <%d> & \"x\".txt", i+1, i)
+		m.Add(manifest.File{ID: fmt.Sprintf("f_%04d", i+1), Path: name, Name: name, Format: "txt", Bytes: int64(i),
+			Hashes:     manifest.Hashes{SHA256: strings.Repeat("cd", 32)},
+			Properties: map[string]any{"line": "a\u2028b", "depth": i % 3, "nested": map[string]any{"k": []any{1, "two"}}}})
+	}
+	return m
+}
+
+func firstByteApart(a, b []byte) int {
+	for i := range min(len(a), len(b)) {
+		if a[i] != b[i] {
+			return i
+		}
+	}
+	return min(len(a), len(b))
+}
+
+func bytesAround(b []byte, at int) []byte {
+	return b[max(0, at-40):min(len(b), at+40)]
+}

internal/guard/mutation/entries.py

@@ -15876,8 +15876,8 @@
         # the source sees it, which is why that walk exists.
         "the engine makes a context from nothing in a place nobody listed",
         "internal/engine/plantarget.go",
-        "if err := ctx.Err(); err != nil {",
-        "if err := context.Background().Err(); err != nil {",
+        "// thousand pictures is the case this is for.\n\t\tif err := ctx.Err(); err != nil {",
+        "// thousand pictures is the case this is for.\n\t\tif err := context.Background().Err(); err != nil {",
         "TestAContextIsMadeFromNothingOnlyWhereItIsMeantToStart",
     ),
     (
@@ -16276,4 +16276,47 @@
         "map[string]any{\"Floor\": HumanBytes(core.MaxPlanBytes * 2)}",
         "TestTheMemorySettingIsChosenOnPreferencesAndKept",
     ),
+    # --- the manifest is written an entry at a time, the plan list reserved (2026-11-02) ---
+    (
+        "the manifest is handed to encoding/json whole again, every entry and the document held at once",
+        "internal/manifest/encode.go",
+        "\tif len(m.Files) == 0 {\n\t\treturn encodeWhole(w, m)\n",
+        "\tif len(m.Files) >= 0 {\n\t\treturn encodeWhole(w, m)\n",
+        "TestTheManifestIsNotHeldWholeToBeWritten",
+    ),
+    (
+        "a manifest that never made its list is cut as if it had one, and writes an empty list instead of null",
+        "internal/manifest/encode.go",
+        "\tif len(m.Files) == 0 {\n\t\treturn encodeWhole(w, m)\n\t}\n\thead, tail, err := m.shell()\n",
+        "\thead, tail, err := m.shell()\n",
+        "TestTheManifestIsWrittenAsEncodingJSONWritesIt",
+    ),
+    (
+        "the streamed entries are joined without the line break encoding/json puts between them",
+        "internal/manifest/encode.go",
+        '\t\tbetween := []byte(",\\n")\n',
+        '\t\tbetween := []byte(",")\n',
+        "TestTheManifestIsWrittenAsEncodingJSONWritesIt",
+    ),
+    (
+        "the streamed list of files closes without the indent of its level",
+        "internal/manifest/encode.go",
+        '\tif err := put(out, []byte("\\n  "), tail); err != nil {\n',
+        '\tif err := put(out, []byte("\\n"), tail); err != nil {\n',
+        "TestTheManifestIsWrittenAsEncodingJSONWritesIt",
+    ),
+    (
+        "the plan is grown a file at a time again instead of reserved",
+        "internal/engine/plantarget.go",
+        "\tpl.out = make([]PlannedFile, 0, max(1, pl.reserve))\n",
+        "\tpl.out = make([]PlannedFile, 0, 1)\n",
+        "TestAPlanIsReservedRatherThanGrown",
+    ),
+    (
+        "the plan list is reserved before any file passed its checks, so a run refused at its first file takes the whole list first",
+        "internal/engine/engine.go",
+        "\tpl.reserve = planCapacity(targets, pl.budget)\n",
+        "\tpl.reserve = planCapacity(targets, pl.budget)\n\tpl.out = make([]PlannedFile, 0, pl.reserve)\n",
+        "TestARunRefusedAtItsFirstFileReservesNothing",
+    ),
 ]