engine: the memory setting holds the whole run, not only its plan

CHANGELOG.md

@@ -18,11 +18,12 @@ because it turns other people's test suites red.
 
 - **How much memory a run may use is yours to set.** `--memory` on
   `generate`, `validate` and `preset show`, and Memory on the window's
-  Preferences screen: `auto` (half of what this machine or container gives
-  the program), a size such as `8gb`, or `off`. Pictures and films wait for
-  room rather than all being held at once. A plan is refused at this
-  setting rather than at a fixed 2 GiB, and `auto` never goes below that.
-  Files come out byte for byte the same.
+  Preferences screen: `auto` (half of this machine or container), a size
+  such as `8gb`, or `off`. Near that amount the program frees memory more
+  often and runs slower, but a run that needs more still gets it. A plan
+  above it is refused, and under `auto` never below 2 GiB. In a container of
+  1 GB, 500 000 text files are written where 400 000 were killed. Files come
+  out byte for byte the same.
 
 - **A Docker image.** `ghcr.io/donislawdev/tfg` holds the Linux command line
   of a release and nothing else, for x64 and Arm, and runs with no network. It

README.md

@@ -345,7 +345,7 @@ tfg generate --format txt --size 1mb    settings come from the flags
 | `--clean` | turn off the self describing label written inside each file |
 | `--dry-run` | count and show, write nothing at all |
 | `--json` | write the manifest to standard output |
-| `--memory <m>` | how much memory the run may use: `auto` (the default, half of what this machine gives the program), a size such as `8gb`, or `off` for no limit. Pictures wait for room rather than all being held at once, and a plan larger than this is refused before anything is written. Under `auto`, a plan of up to 2.0 GB is never refused. Also on `validate` and `preset show` |
+| `--memory <m>` | how much memory the run may use: `auto` (the default, half of what this machine gives the program), a size such as `8gb`, or `off` for no limit. Pictures wait for room rather than all being held at once, and a plan larger than this is refused before anything is written. When a run gets close to this amount, the program frees memory more often, which makes the run slower. A run that needs more memory than this still gets it. Under `auto`, a plan of up to 2.0 GB is never refused. Also on `validate` and `preset show` |
 
 ### `tfg validate`
 

cmd/tfg-gui/main.go

@@ -8,9 +8,14 @@ package main
 import (
 	"os"
 
+	"github.com/donislawdev/TestingFilesGenerator/internal/engine"
 	"github.com/donislawdev/TestingFilesGenerator/internal/gui"
 )
 
 func main() {
+	// This process is the window's own, so the memory setting holds each
+	// whole run and not only its plan - engine/processmemory.go says why only
+	// the two main functions say so.
+	engine.OwnProcess()
 	os.Exit(gui.Run(os.Args[1:], os.Stderr))
 }

cmd/tfg/main.go

@@ -14,6 +14,7 @@ import (
 	"syscall"
 
 	"github.com/donislawdev/TestingFilesGenerator/internal/cli"
+	"github.com/donislawdev/TestingFilesGenerator/internal/engine"
 )
 
 func main() {
@@ -29,6 +30,11 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	// This process is the command's own, so --memory holds the whole run and
+	// not only its plan. Here and in the window's main, and nowhere a test
+	// reaches - engine/processmemory.go says why.
+	engine.OwnProcess()
+
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, os.Interrupt, syscall.SIGTERM)
 	defer signal.Stop(signals)

internal/cli/memoryflag.go

@@ -23,6 +23,8 @@ import (
 var memoryHelp = "how much memory this run may use: auto (half of what this machine gives the program), " +
 	"a size such as 8gb, or off for no limit. Pictures wait for room rather than all being held at once, " +
 	"and a plan larger than this is refused before anything is written. " +
+	"When a run gets close to this amount, the program frees memory more often, which makes the run slower. " +
+	"A run that needs more memory than this still gets it. " +
 	"Under auto, a plan of up to " + core.HumanBytes(core.MaxPlanBytes) + " is never refused"
 
 // memoryValue is the flag's value, auto until somebody writes one.

internal/core/memory.go

@@ -199,3 +199,27 @@ func (m Memory) On(machine func() (int64, error)) MemoryLimits {
 	half := total / 2
 	return MemoryLimits{Run: half, Plan: max(half, MaxPlanBytes)}
 }
+
+// ForRuntime is the memory limit a process that is the program's own hands the
+// Go runtime for a run under these limits. start is the limit the process
+// started with - what GOMEMLIMIT said, or no limit at all.
+//
+// The runtime's limit is soft. Close to it the collector runs more often, a
+// run that needs more still gets it, and the collector takes no more than
+// about half the processor, so a run held under it is at most about twice as
+// slow (go.dev/doc/gc-guide, the memory limit section). Well under it nothing
+// changes. Without it a run takes about two and a half times what it holds,
+// and in a container of 1 GB that was the difference between 400 000 files
+// killed by the system and 500 000 written (docs/FILE-CEILING-2026-11-02.md
+// section 12).
+//
+// Never above start, because a person who set GOMEMLIMIT lower chose that on
+// purpose. Off hands start back, and so does a machine that could not be
+// asked: its Run is MaxPlanBytes, a number about no machine, and on a large
+// machine whose memory could not be read it would slow a run for nothing.
+func (l MemoryLimits) ForRuntime(start int64) int64 {
+	if l.Run <= 0 || l.Unknown != nil {
+		return start
+	}
+	return min(l.Run, start)
+}

internal/engine/engine.go

@@ -401,7 +401,9 @@ func PlanContext(ctx context.Context, targets []Target, opt Options) ([]PlannedF
 	// Watches what the plan costs while it is being built. Constructed here
 	// rather than at the first file because constructing it is what takes the
 	// reference point, and that has to happen before any of the plan exists.
-	pl.budget = newPlanMemory(opt.Memory, opt.memoryLimits())
+	limits := opt.memoryLimits()
+	pl.budget = newPlanMemory(opt.Memory, limits)
+	handToRuntime(limits)
 
 	// The manifest lands beside the files, so its name is a name too. A path
 	// here would leave a manifest outside the directory the run was pointed

internal/engine/processmemory.go

@@ -0,0 +1,67 @@
+package engine
+
+import (
+	"runtime/debug"
+
+	"github.com/donislawdev/TestingFilesGenerator/internal/core"
+)
+
+// The memory setting handed to the Go runtime, in a process that is the
+// program's own.
+//
+// The setting said how much memory a run may take and held only the plan to
+// it. The rest of a run - the plan, the manifest entries and what the writers
+// throw away - was left to the collector's default, which lets the heap grow
+// to about twice what is live before it collects, and a run took about two and
+// a half times what it held. In a container of 1 GB that killed a run of
+// 400 000 text files under auto, and one of three runs of 300 000
+// (docs/FILE-CEILING-2026-11-02.md section 12).
+//
+// Plain variables rather than a lock, because both are written once, by the
+// main function, before it starts anything that plans - and a goroutine
+// started after a write sees it. A lock here would be concurrency the guard of
+// the concurrent files has to be told about, defending against a caller that
+// does not exist.
+var (
+	owned bool
+	// startLimit is the runtime's limit when the process said it was the
+	// program's own.
+	startLimit int64
+)
+
+// OwnProcess says this process is the program's own - the command line or the
+// window, with nothing beside it that a limit on its memory would hold back -
+// so every plan from now on hands its memory setting to the Go runtime as the
+// limit of the whole process. Only the two main functions call it.
+//
+// Not done for every caller of the engine, because the limit is the process's
+// and not the run's. The guards plan hundreds of runs side by side in one test
+// binary, some under a setting of a few megabytes, and handing that to the
+// runtime would hold every guard beside them to it.
+//
+// Never undone. The command line ends with its one run, and the window hands
+// over the setting of each run it plans, so a changed setting is in force from
+// the next run and off gives the runtime back the limit it started with.
+//
+// Called first, before anything that plans starts. A second call changes
+// nothing, so the limit it started with is never taken from a run.
+func OwnProcess() {
+	if owned {
+		return
+	}
+	startLimit = debug.SetMemoryLimit(-1)
+	owned = true
+}
+
+// handToRuntime gives the runtime the limit for a run under limits, in a
+// process that is the program's own, and does nothing anywhere else.
+//
+// Called when a plan starts rather than when the writing does, because the
+// plan is the first thing both surfaces do for a run and the manifest is saved
+// after the writing has returned - a limit set and taken back inside Run would
+// leave both outside it.
+func handToRuntime(limits core.MemoryLimits) {
+	if owned {
+		debug.SetMemoryLimit(limits.ForRuntime(startLimit))
+	}
+}

internal/guard/mutation/entries.py

@@ -16038,7 +16038,7 @@
         # the ceiling that could not be moved, which is R02.
         "planning ignores the memory setting and works to the old ceiling",
         "internal/engine/engine.go",
-        "pl.budget = newPlanMemory(opt.Memory, opt.memoryLimits())",
+        "pl.budget = newPlanMemory(opt.Memory, limits)",
         "pl.budget = newPlanMemory(opt.Memory, core.MemoryLimits{Plan: core.MaxPlanBytes})",
         "TestThePlanCeilingIsTheMemorySetting",
     ),
@@ -16319,4 +16319,46 @@
         "\tpl.reserve = planCapacity(targets, pl.budget)\n\tpl.out = make([]PlannedFile, 0, pl.reserve)\n",
         "TestARunRefusedAtItsFirstFileReservesNothing",
     ),
+    (
+        "the runtime is handed the setting even above the limit the process started with",
+        "internal/core/memory.go",
+        "\treturn min(l.Run, start)\n",
+        "\treturn l.Run\n",
+        "TestTheMemorySettingIsTheLimitTheRuntimeIsHanded",
+    ),
+    (
+        "a machine that could not be asked hands the runtime a number about no machine",
+        "internal/core/memory.go",
+        "\tif l.Run <= 0 || l.Unknown != nil {\n\t\treturn start\n",
+        "\tif l.Run <= 0 {\n\t\treturn start\n",
+        "TestTheMemorySettingIsTheLimitTheRuntimeIsHanded",
+    ),
+    (
+        "every plan hands its setting to the runtime, also in a process that is not the program's own",
+        "internal/engine/processmemory.go",
+        "\tif owned {\n\t\tdebug.SetMemoryLimit(",
+        "\tif true {\n\t\tdebug.SetMemoryLimit(",
+        "TestAPlanOutsideTheProgramsOwnProcessLeavesTheRuntimeAlone",
+    ),
+    (
+        "a plan no longer hands its memory setting to the runtime",
+        "internal/engine/engine.go",
+        "\thandToRuntime(limits)\n",
+        "\t_ = limits\n",
+        "TestTheCommandLineHoldsTheWholeRunToItsMemorySetting",
+    ),
+    (
+        "the command line does not say its process is the program's own",
+        "cmd/tfg/main.go",
+        "\tengine.OwnProcess()\n",
+        "\t_ = engine.OwnProcess\n",
+        "TestTheCommandLineHoldsTheWholeRunToItsMemorySetting",
+    ),
+    (
+        "the window does not say its process is the program's own",
+        "cmd/tfg-gui/main.go",
+        "\tengine.OwnProcess()\n",
+        "\t_ = engine.OwnProcess\n",
+        "TestTheWindowSaysItsProcessIsItsOwnBeforeItOpens",
+    ),
 ]