From ee8e0d543b5df48f90d7a01c0995e2683aaa60d5 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 12 Nov 2026 00:03:07 +0000 Subject: [PATCH] Move the panel rule and the Baseline condition into the harness The panel (the top-K models by Epoch's Capabilities Index we can run, one seat per model family) and the Baseline condition (no system prompt, search offered and never required, top reasoning level) are now shared by more than one dashboard, so they live beside the ECI reader in packages/fri-forecaster: - fri_forecaster/panel.py: the table of runnable models, the selection rule, the reasoning levels, and a CLI (python -m fri_forecaster.panel). - fri_forecaster/baseline.py: the Baseline loop's settings and a run() helper. AIRO reads both from there. redlines/registry.py keeps what is AIRO's own (colors, chart roles, ForecastBench names) joined onto the shared rows; redlines/llm.py, redlines/pi_loop.py and code/run_unified.py import the shared constants. Nothing AIRO asks or publishes changes: registry.MODELS, the panel for every snapshot and the baseline prompt are identical to before, and tests/test_shared_panel.py pins that AIRO and the harness agree. Co-Authored-By: Claude Fable 5.2 --- README.md | 3 +- code/run_unified.py | 14 +- packages/fri-forecaster/README.md | 14 ++ .../fri-forecaster/examples/round.env.example | 2 + .../fri-forecaster/fri_forecaster/baseline.py | 73 ++++++ .../fri-forecaster/fri_forecaster/panel.py | 231 ++++++++++++++++++ packages/fri-forecaster/tests/test_panel.py | 118 +++++++++ redlines/llm.py | 5 +- redlines/pi_loop.py | 10 +- redlines/registry.py | 144 +++++------ tests/test_shared_panel.py | 32 +++ 11 files changed, 547 insertions(+), 99 deletions(-) create mode 100644 packages/fri-forecaster/fri_forecaster/baseline.py create mode 100644 packages/fri-forecaster/fri_forecaster/panel.py create mode 100644 packages/fri-forecaster/tests/test_panel.py create mode 100644 tests/test_shared_panel.py diff --git a/README.md b/README.md index 050fc6a..f4ac625 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,8 @@ packages/fri-forecaster/ the FRI Forecaster harness -- pi, the research tools redlines/ the pipeline package (stdlib only) __main__.py CLI: build / assemble / export-fb / paper config.py paths (env-overridable), round pins, thresholds - registry.py the single model table (ids, labels, ECI, colors) + panel() + registry.py AIRO's model table (colors, chart roles, pinned ECI) over the shared rows and + panel rule in packages/fri-forecaster (fri_forecaster.panel) eci.py the Epoch Capabilities Index: fri_forecaster.eci (one implementation, shared with the ForecastBench rounds) bound to data/ and the pinned vintages questions.py schema-checked loaders for the Auto-ARC question set diff --git a/code/run_unified.py b/code/run_unified.py index 7b32e7e..992a653 100644 --- a/code/run_unified.py +++ b/code/run_unified.py @@ -103,6 +103,7 @@ from redlines.config import UNBATCHED # noqa: E402 from redlines.questions import (horizon_label, horizon_sort_key, # noqa: E402 resolves_on, severity_label) +from fri_forecaster import baseline as _baseline # noqa: E402 (on the path through redlines.eci) LADDER = os.path.join(ROOT, "data", "autoarc_ladder.json") CROSS = os.path.join(ROOT, "data", "autoarc_crosscutting.json") @@ -388,8 +389,12 @@ def apply_variant(prompt, variant): # far above any call seen (64 turns); it also keeps agent-box's nudge on, so a # reply cut at the output limit with no submission (Opus 5.5, 2026-10-19) # does not end the call. -MINIMAL_SYSTEM = " " -MINIMAL_MAX_TURNS = 100 +# +# Since 2026-11-11 this is the Baseline condition every dashboard shares +# (fri_forecaster.baseline, packages/fri-forecaster): the system prompt, the +# turn limit and the web_search description are read from there. +MINIMAL_SYSTEM = _baseline.SYSTEM +MINIMAL_MAX_TURNS = _baseline.MAX_TURNS MINIMAL_JOINT_HEAD = ("forecasting questions, grouped by cause of catastrophe. Forecast\n" "all of them together, in one pass -- and forecast every") MINIMAL_JOINT_RESEARCH = re.compile(r"Research before answering, in rounds:.*?applied to the same evidence\.\n", re.S) @@ -397,10 +402,7 @@ def apply_variant(prompt, variant): "you relied on (only pages you read), and any cells not yet delivered.") DEBRIEF_SENTENCE = re.compile(r" ?After it is accepted you will be asked one more thing.*?" r"research with those questions in mind\.", re.S) -MINIMAL_TOOLS = { - "web_search": "Search the web. Returns titled snippets with URLs and publication dates. " - "recent_days limits results to the last N days.", -} +MINIMAL_TOOLS = {"web_search": _baseline.WEB_SEARCH} def minimal_joint_prompt(prompt, variant=DEFAULT_VARIANT): diff --git a/packages/fri-forecaster/README.md b/packages/fri-forecaster/README.md index abaf0fd..b70be65 100644 --- a/packages/fri-forecaster/README.md +++ b/packages/fri-forecaster/README.md @@ -157,6 +157,20 @@ The result `harness.run` returns, folded from the outbox: Only the answers asked for (`allowed_keys`) are kept; anything else the model sends is dropped. +## The panel and the Baseline condition + +What FRI's dashboards share when they ask a standing panel of models (AIRO, EEAI): + +- `fri_forecaster.eci` fetches and ranks Epoch's Capabilities Index. +- `fri_forecaster.panel` picks the panel from it: the top 4 models we can run, one + seat per model family. The table of runnable models is in that file; a box may add + rows on its own in `auto_models.json` beside the snapshots (AIRO's `redlines.autoreg`). + `python -m fri_forecaster.panel --dir DIR [K]` prints it. +- `fri_forecaster.baseline` is how the panel is asked: no system prompt, web search + offered and never required, reasoning at the top level, no debrief. + `baseline.run(model, prompt, ...)` makes one such call with the caller's own answer + schema. AIRO's published `none` condition reads its constants from here. + ## Requirements - Python ≥ 3.11, stdlib only. diff --git a/packages/fri-forecaster/examples/round.env.example b/packages/fri-forecaster/examples/round.env.example index 9769123..9e2999d 100644 --- a/packages/fri-forecaster/examples/round.env.example +++ b/packages/fri-forecaster/examples/round.env.example @@ -15,5 +15,7 @@ ROUNDS_DIR="$HOME/forecastbench/rounds" # default: /rounds # place when its members are not the panel's top N. On AIRO's box the panel is AIRO's, # refreshed from Epoch's index first (code/refresh_panel.sh, which pages a change): # panel_members() { (cd "$HOME/Projects/airo" && { code/refresh_panel.sh >&2; "$PY" -m redlines.registry 6; }); } +# Or, with no AIRO checkout, the shared panel over a directory of snapshots: +# panel_members() { "$PY" -m fri_forecaster.panel --dir "$HOME/.cache/fri-forecaster/eci" 6; } # Without this, the round ranks Epoch's published index by name # (`python -m fri_forecaster.eci --fetch --names`) and can only warn when the top N moved. diff --git a/packages/fri-forecaster/fri_forecaster/baseline.py b/packages/fri-forecaster/fri_forecaster/baseline.py new file mode 100644 index 0000000..6212c8b --- /dev/null +++ b/packages/fri-forecaster/fri_forecaster/baseline.py @@ -0,0 +1,73 @@ +"""The Baseline condition: how a dashboard asks the panel. + +One call per panel member (fri_forecaster.panel), with nothing in it but the +dashboard's own prompt: + + no system prompt one space, because pi puts its own coding-assistant + prompt in place of an empty one. + research is offered, web_search and read_page (Tavily, tools.py), described + never required by what they do; no research floor, no coverage rule. + no debrief the call ends with the answer. + reasoning at the top panel.reasoning_for. + a turn limit of 100 a runaway guard far above any call seen (64 turns); it + also keeps the harness's nudge on, so a reply cut at + the output limit with no submission does not end the + call. + +This is AIRO's published condition `none` (its minimal instrument, +combined-v10, 2026-10-19: the prompt-variant experiments of 2026-10-14..19 +found the instrument's guidance moved no headline forecast), labelled +"Baseline" on its dashboard. AIRO's runner reads the constants below, so the +two cannot drift; a dashboard that asks something other than probabilities +calls run() with its own answer schema. +""" +import copy +import os +import tempfile + +from . import panel +from .tools import READ_PAGE_TOOL, WEB_SEARCH_TOOL + +SYSTEM = " " +MAX_TURNS = 100 +# A whole call, research included. +TIMEOUT = 3 * 3600 +WEB_SEARCH = ("Search the web. Returns titled snippets with URLs and publication dates. " + "recent_days limits results to the last N days.") +LOOP = {"min_research": 0, "floor_counts": "returned", "withhold_final": True, + "warn_missing_keys": False, "max_turns": MAX_TURNS} + + +def research_tools(): + """web_search and read_page as the condition offers them: {name, + description, parameters}. web_search loses `cause`, which serves only a + coverage rule.""" + search = {k: copy.deepcopy(WEB_SEARCH_TOOL[k]) for k in ("name", "description", "parameters")} + search["description"] = WEB_SEARCH + search["parameters"]["properties"].pop("cause", None) + read = {k: copy.deepcopy(READ_PAGE_TOOL[k]) for k in ("name", "description", "parameters")} + return [search, read] + + +def run(model, prompt, *, workdir, cell_schema, key_fields, allowed_keys, submit_cells, submit_forecast, + env=None, timeout=TIMEOUT, cwd=None, retries=0, harness_run=None): + """One Baseline call. `model` is a panel row's `model`; `submit_cells` and + `submit_forecast` are the dashboard's two answer tools as {description, + parameters} (the pieces along the way, and the final answer); the cells + they carry are `cell_schema`, keyed by `key_fields`, and only + `allowed_keys` are kept. -> harness.run's result.""" + if harness_run is None: + from . import harness + harness_run = harness.run + research = research_tools() + specs = {**{t["name"]: t for t in research}, "submit_cells": submit_cells, "submit_forecast": submit_forecast} + # pi writes its working directory into its system prompt: run it from an + # empty directory whose name says nothing about the project. + cwd = cwd or os.path.join(tempfile.gettempdir(), "w") + os.makedirs(cwd, exist_ok=True) + return harness_run(model=panel.pi_model(model), system=SYSTEM, prompt=prompt, workdir=workdir, cwd=cwd, + cell_schema=cell_schema, key_fields=list(key_fields), allowed_keys=list(allowed_keys), + tool_descriptions={n: s["description"] for n, s in specs.items()}, + tool_parameters={n: s["parameters"] for n, s in specs.items()}, + research_tools=tuple(t["name"] for t in research), cells_tool=True, + thinking=panel.reasoning_for(model), timeout=timeout, env=env, retries=retries, **LOOP) diff --git a/packages/fri-forecaster/fri_forecaster/panel.py b/packages/fri-forecaster/fri_forecaster/panel.py new file mode 100644 index 0000000..fffb18a --- /dev/null +++ b/packages/fri-forecaster/fri_forecaster/panel.py @@ -0,0 +1,231 @@ +"""The model panel: the top k of Epoch's index that we can run, one seat per family. + +Every FRI dashboard that asks a standing panel of frontier models asks THIS +panel (AIRO since 2026-08-28; EEAI; the ForecastBench rounds through `names +sync`). It is not a hand-typed list: it is the PANEL_K highest-ECI models in +the newest snapshot of Epoch's index (fri_forecaster.eci, beside this module) +that MODELS below can run, re-ranked every time it is asked for. Until +2026-11-11 the rule and the table lived in AIRO's redlines/registry.py while +the index reader lived here; they are one thing and now sit together. + + k = 4 averaging the top 3-4 by ECI beat the single best-ranked + model and closed the gap to the hindsight-best one; at 5 the + marginal model is already below the frontier (AIRO's + docs/eci-top4-arm.md). + one seat per a model whose family already holds a seat is passed over + family and the next runnable model takes it: two versions of one + model are one opinion twice (2026-09-08). + runnable a model is runnable when it has a row: a hand row in MODELS, + or one a dashboard's box added on its own after probing the + model (AIRO's redlines.autoreg writes auto_models.json + beside the snapshots). A hand row wins. + +Everything the rule does out of the ordinary is a warning, returned and +printed once: a stale snapshot, a model passed over for its family, a model +above the cut with no row, a tie at the cut, a short panel. + + python -m fri_forecaster.panel [--dir DIR] [K] + # the panel's top K (default PANEL_K), best first, one per line as + # ` TAB TAB `: what `names sync` reads +""" +import json +import os +import sys +from pathlib import Path + +from . import eci + +PANEL_K = 4 +PANEL_SET = "eci_topk" + +# ── The runnable models ────────────────────────────────────────────────────── +# key a stable slug, what a dashboard keys its own columns on (AIRO's +# colors and chart roles are in redlines/registry.py, by this key). +# family the line whose versions supersede one another, for the one-seat +# rule. By hand, and a judgement: Claude's tiers (Fable, Opus, +# Sonnet, Haiku) are separate lines; OpenAI's plain GPT-n are one +# line, the Pro tier another, and the code-named GPT-5.6 Sol / +# GPT-6 Astra each their own -- so the 2026-09-08 panel seats Astra +# beside GPT-5.5 Pro. Fold two families into one here if that ever +# reads wrong; nothing else changes. +# label the display name. +# model `provider/id`. OpenAI ids that name the Responses API by a path +# segment (`openai/responses/...`) are what AIRO's rows have always +# recorded; pi_model() gives pi's spelling. +# epoch_name Epoch's name for the model, exactly: the join to the index. +# +# Rows are never removed: a past run's panel is re-derived from its snapshot. +_FIELDS = ("key", "family", "label", "model", "epoch_name") +_ROWS = [ + ("gpt-3.5-turbo", "gpt", "GPT-3.5", "openai/gpt-3.5-turbo-0125", "GPT-3.5 Turbo (Jan 2024)"), + ("llama-3.3-70b", "llama", "Llama-3.3-70B", "deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo", "Llama 3.3 70B"), + ("gpt-4o", "gpt", "GPT-4o", "openai/gpt-4o", "GPT-4o (Nov 2024)"), + ("deepseek-v3", "deepseek", "DeepSeek-V3", "deepinfra/deepseek-ai/DeepSeek-V3", "DeepSeek-V3"), + ("claude-sonnet-4-5", "claude-sonnet", "Sonnet 4.5", "anthropic/claude-sonnet-4-5-20250929", "Claude Sonnet 4.5"), + ("gpt-5", "gpt", "GPT-5", "openai/gpt-5-2025-08-07", "GPT-5"), + ("claude-opus-4-6", "claude-opus", "Opus 4.6", "anthropic/claude-opus-4-6", "Claude Opus 4.6"), + ("claude-opus-4-8", "claude-opus", "Opus 4.8", "anthropic/claude-opus-4-8", "Claude Opus 4.8"), + ("claude-fable-5", "claude-fable", "Fable 5", "anthropic/claude-fable-5", "Claude Fable 5"), + ("gpt-4.1", "gpt", "GPT-4.1", "openai/gpt-4.1-2025-04-14", "GPT-4.1"), + ("claude-haiku-4-5", "claude-haiku", "Haiku 4.5", "anthropic/claude-haiku-4-5-20251001", "Claude Haiku 4.5"), + ("gpt-5.5-pro", "gpt-pro", "GPT-5.5 Pro", "openai/gpt-5.5-pro", "GPT-5.5 Pro"), + ("gpt-5.5", "gpt", "GPT-5.5", "openai/gpt-5.5", "GPT-5.5"), + ("gemini-3.1-pro", "gemini-pro", "Gemini 3.1 Pro", "gemini/gemini-3.1-pro-preview", "Gemini 3.1 Pro"), + ("grok-4.20", "grok", "Grok 4.20", "xai/grok-4.20", "Grok 4.20"), + # Added 2026-08-28 with the ECI top-4 arm. + ("claude-opus-5", "claude-opus", "Opus 5", "anthropic/claude-opus-5", "Claude Opus 5"), + ("gpt-5.6-sol", "gpt-sol", "GPT-5.6 Sol", "openai/gpt-5.6-sol", "GPT-5.6 Sol"), + # Added 2026-09-08: the index's new top two. GPT-6 Astra takes the + # Responses API: on /v1/chat/completions OpenAI refuses function tools for + # it under any reasoning effort, and the research tools are the point. + ("claude-fable-5-1", "claude-fable", "Fable 5.1", "anthropic/claude-fable-5-1", "Claude Fable 5.1"), + ("gpt-6-astra", "gpt-astra", "GPT-6 Astra", "openai/responses/gpt-6-astra", "GPT-6 Astra"), + # Added 2026-10-01: the index ranks them first and third, so they take the + # claude-opus seat from Opus 5 and a claude-sonnet seat of their own. + ("claude-opus-5-5", "claude-opus", "Opus 5.5", "anthropic/claude-opus-5-5", "Claude Opus 5.5"), + ("claude-sonnet-5-5", "claude-sonnet", "Sonnet 5.5", "anthropic/claude-sonnet-5-5", "Claude Sonnet 5.5"), +] +MODELS = [dict(zip(_FIELDS, row)) for row in _ROWS] + +AUTO_MODELS = "auto_models.json" + +# Reasoning level per provider, pi's `--thinking`: the top rung, which is +# spelled differently per provider (AIRO's project lead, 2026-09-02: reasoning +# at its maximum; Anthropic's effort also governs how many tool calls the model +# makes, which is the point). +REASONING = {"anthropic": "max", "openai": "xhigh"} +REASONING_DEFAULT = "high" + + +def reasoning_for(model): + """pi's `--thinking` for `model` (`provider/id`).""" + return REASONING.get(model.split("/", 1)[0], REASONING_DEFAULT) + + +def pi_model(model): + """A row's `model` -> pi's id for it. pi picks the API itself, so the + Responses-API path segment is dropped.""" + return model.replace("openai/responses/", "openai/", 1) + + +def auto_models_path(snapshot_dir=None): + """Where a box's automatic rows are: $FORECASTER_AUTO_MODELS, else + auto_models.json beside the snapshots.""" + explicit = os.environ.get("FORECASTER_AUTO_MODELS") + return Path(explicit) if explicit else eci._dir(snapshot_dir) / AUTO_MODELS + + +def load_auto(path): + """The automatic rows at `path` ([] if there is no file), in this module's + fields; an entry written by AIRO's autoreg spells `model` as `litellm_id`.""" + path = Path(path) + if not path.exists(): + return [] + out = [] + for e in json.loads(path.read_text(encoding="utf-8")): + row = {f: e.get(f) for f in _FIELDS} + row["model"] = row["model"] or e.get("litellm_id") + out.append(row) + return out + + +def models(auto=None, snapshot_dir=None): + """Every runnable model: the hand rows, then the automatic ones (from + `auto`, a path, or auto_models_path) that no hand row already covers.""" + out = [dict(m) for m in MODELS] + hand = {m["epoch_name"] for m in out} | {m["key"] for m in out} + for row in load_auto(auto or auto_models_path(snapshot_dir)): + if row["epoch_name"] not in hand and row["key"] not in hand: + out.append({**row, "auto": True}) + return out + + +def select(ranked, table, k=PANEL_K, snapshot_name="the snapshot", stale=None): + """THE RULE. `ranked` is the index best first (eci.ranked); `table` the + runnable rows, each with at least `epoch_name` and `family` (a dashboard + may pass its own rows, with its own extra columns, and gets them back). + -> (members, warnings): members are (row, index record) pairs, best + first; `stale` is a staleness warning to lead with, if any.""" + by_epoch = {m["epoch_name"]: m for m in table} + chosen, unrunnable, passed, warnings = [], [], [], [] + if stale: + warnings.append(stale) + seated = {} + for r in ranked: + m = by_epoch.get(r["model"]) + if m is None: + unrunnable.append(r) + continue + if m["family"] in seated: + passed.append((m, r, seated[m["family"]])) + continue + chosen.append((m, r)) + seated[m["family"]] = m + if len(chosen) == k: + break + for m, r, holder in passed: + warnings.append(f"passed over {r['model']} ({r['eci']:g}): its family " + f"({m['family']}) already holds a seat through {holder['epoch_name']}; " + f"the next runnable model took the seat") + if len(chosen) < k: + warnings.append(f"only {len(chosen)} of the top {k} in {snapshot_name} are models the " + f"registry can run; the panel is short") + if chosen: + cut = chosen[-1][1] + above = [r["model"] for r in unrunnable if (r["eci"], -r["rank"]) > (cut["eci"], -cut["rank"])] + if above: + warnings.append(f"the index ranks {', '.join(above)} above the panel's cut but the " + f"registry cannot run them (no row in fri_forecaster/panel.py, and no " + f"automatic one -- the autoreg log line says why); " + f"skipped, the next runnable model took the seat") + tied = [r["model"] for r in ranked if r["eci"] == cut["eci"] + and r["model"] not in {m["epoch_name"] for m, _ in chosen}] + if tied: + warnings.append(f"tie at the cut: {cut['model']} ({cut['eci']:g}) is in the panel " + f"and {', '.join(tied)} ({cut['eci']:g}) out, on Epoch's rank order") + return chosen, warnings + + +def panel(k=PANEL_K, snapshot=None, snapshot_dir=None, auto=None, warn=True): + """The panel for a dashboard with no table of its own: the k highest-ECI + runnable models of `snapshot` ((date, path); the newest in `snapshot_dir` + by default), best first. -> (members, provenance). A member is its row + plus `pi_model`, `reasoning`, and the index's `eci`, `rank` and + `organization`; `provenance` is what a run stamps on its rows (which + snapshot chose the panel, how old it was, the warnings).""" + day, path = snapshot or eci.latest(snapshot_dir) + stale = None if snapshot else eci.stale_message(snapshot_dir=snapshot_dir) + chosen, warnings = select(eci.ranked(path), models(auto, snapshot_dir), k, Path(path).name, stale) + if warn: + for w in warnings: + print(f"WARN: {w}", file=sys.stderr) + members = [{**m, "pi_model": pi_model(m["model"]), "reasoning": reasoning_for(m["model"]), + "eci": r["eci"], "rank": r["rank"], "organization": r.get("organization")} + for m, r in chosen] + provenance = {"set": PANEL_SET, "k": k, "snapshot": day.isoformat(), + "age_days": eci.age_days(snapshot_dir=snapshot_dir) if not snapshot else None, + "members": [{"label": m["label"], "eci": m["eci"], "rank": m["rank"]} for m in members], + "warnings": warnings} + return members, provenance + + +def main(argv=None): + args = list(sys.argv[1:] if argv is None else argv) + snapshot_dir = None + if "--dir" in args: + i = args.index("--dir") + snapshot_dir = args[i + 1] + del args[i:i + 2] + k = int(args[0]) if args else PANEL_K + try: + members, _ = panel(k, snapshot_dir=snapshot_dir) + except FileNotFoundError as e: + print(e, file=sys.stderr) + return 1 + for m in members: + print(m["epoch_name"], m["pi_model"], m["organization"] or "", sep="\t") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/fri-forecaster/tests/test_panel.py b/packages/fri-forecaster/tests/test_panel.py new file mode 100644 index 0000000..8eff207 --- /dev/null +++ b/packages/fri-forecaster/tests/test_panel.py @@ -0,0 +1,118 @@ +"""The shared panel (top k by ECI, one seat per family) and the Baseline condition.""" +import csv +import json +import tempfile +import unittest +from datetime import date +from pathlib import Path + +from fri_forecaster import baseline, eci, panel + + +def _snapshot(tmp, day, rows): + """A rank-format snapshot: rows are (model, eci).""" + p = Path(tmp) / f"{eci.SNAPSHOT_PREFIX}{day}.csv" + with open(p, "w", newline="") as f: + w = csv.writer(f) + w.writerow(["rank", "model", "eci", "ci_low", "ci_high", "retrieved"]) + for i, (m, e) in enumerate(rows, 1): + w.writerow([i, m, e, e - 3, e + 3, day]) + return date.fromisoformat(day), p + + +class TestRule(unittest.TestCase): + TABLE = [{"epoch_name": "A 2", "family": "a"}, {"epoch_name": "A 1", "family": "a"}, + {"epoch_name": "B 1", "family": "b"}, {"epoch_name": "C 1", "family": "c"}] + + def _ranked(self, rows): + return [{"model": m, "eci": e, "rank": i} for i, (m, e) in enumerate(rows, 1)] + + def test_top_k_one_seat_per_family_and_what_it_says(self): + ranked = self._ranked([("A 2", 170), ("X 9", 169), ("A 1", 168), ("B 1", 160), ("C 1", 150)]) + members, warnings = panel.select(ranked, self.TABLE, 2, "snap.csv") + self.assertEqual([m["epoch_name"] for m, _ in members], ["A 2", "B 1"]) + self.assertTrue(any("passed over A 1 (168)" in w and "A 2" in w for w in warnings), warnings) + self.assertTrue(any("X 9" in w and "cannot run" in w for w in warnings), warnings) + + def test_a_tie_at_the_cut_and_a_short_panel(self): + ranked = self._ranked([("B 1", 160), ("C 1", 160)]) + members, warnings = panel.select(ranked, self.TABLE, 1, "snap.csv") + self.assertEqual(members[0][0]["epoch_name"], "B 1") + self.assertTrue(any(w.startswith("tie at the cut") and "C 1" in w for w in warnings), warnings) + _, warnings = panel.select(ranked, self.TABLE, 4, "snap.csv", stale="old") + self.assertEqual(warnings[0], "old") + self.assertTrue(any("only 2 of the top 4 in snap.csv" in w and "panel is short" in w for w in warnings)) + + def test_rows_come_back_as_given(self): + table = [dict(r, color="#123456") for r in self.TABLE] + members, _ = panel.select(self._ranked([("C 1", 150)]), table, 1) + self.assertEqual(members[0][0]["color"], "#123456") + + +class TestPanel(unittest.TestCase): + def test_table_is_well_formed(self): + keys = [m["key"] for m in panel.MODELS] + self.assertEqual(len(keys), len(set(keys))) + self.assertEqual(len({m["epoch_name"] for m in panel.MODELS}), len(panel.MODELS)) + for m in panel.MODELS: + self.assertTrue(all(m[f] for f in panel._FIELDS), m) + self.assertIn("/", m["model"]) + + def test_panel_from_a_snapshot_directory_with_an_automatic_row(self): + with tempfile.TemporaryDirectory() as tmp: + snap = _snapshot(tmp, "2026-11-01", [("Nova 9", 180), ("Claude Opus 5.5", 167), ("Claude Opus 5", 163), + ("GPT-6 Astra", 166), ("Claude Fable 5.1", 164)]) + members, prov = panel.panel(3, snapshot=snap, snapshot_dir=tmp, warn=False) + self.assertEqual([m["key"] for m in members], ["claude-opus-5-5", "gpt-6-astra", "claude-fable-5-1"]) + self.assertTrue(any("Nova 9" in w for w in prov["warnings"])) + # A box's automatic row (AIRO's autoreg spells `model` litellm_id) makes Nova runnable. + (Path(tmp) / panel.AUTO_MODELS).write_text(json.dumps([ + {"key": "nova-9", "family": "nova", "label": "Nova 9", "litellm_id": "openai/responses/nova-9", + "epoch_name": "Nova 9", "color": "#000000"}, + {"key": "claude-opus-5-5", "family": "x", "label": "x", "litellm_id": "x/x", "epoch_name": "Claude Opus 5.5"}])) + members, prov = panel.panel(3, snapshot=snap, snapshot_dir=tmp, warn=False) + self.assertEqual([m["key"] for m in members], ["nova-9", "claude-opus-5-5", "gpt-6-astra"]) + self.assertEqual(members[0]["pi_model"], "openai/nova-9") + self.assertEqual(members[0]["reasoning"], "xhigh") + self.assertEqual(members[1]["family"], "claude-opus") # the hand row wins + self.assertEqual(prov["snapshot"], "2026-11-01") + self.assertEqual([m["label"] for m in prov["members"]], ["Nova 9", "Opus 5.5", "GPT-6 Astra"]) + + def test_ids_and_reasoning(self): + self.assertEqual(panel.pi_model("openai/responses/gpt-6-astra"), "openai/gpt-6-astra") + self.assertEqual(panel.pi_model("anthropic/claude-opus-5-5"), "anthropic/claude-opus-5-5") + self.assertEqual(panel.reasoning_for("anthropic/claude-opus-5-5"), "max") + self.assertEqual(panel.reasoning_for("gemini/gemini-3.1-pro-preview"), "high") + + +class TestBaseline(unittest.TestCase): + def test_research_is_offered_plainly(self): + search, read = baseline.research_tools() + self.assertEqual((search["name"], read["name"]), ("web_search", "read_page")) + self.assertEqual(search["description"], baseline.WEB_SEARCH) + self.assertNotIn("cause", search["parameters"]["properties"]) + self.assertEqual(search["parameters"]["required"], ["query"]) + + def test_run_is_the_minimal_loop(self): + seen = {} + + def fake(**kw): + seen.update(kw) + return {"cells": {}} + tool = {"description": "d", "parameters": {"type": "object"}} + with tempfile.TemporaryDirectory() as tmp: + baseline.run("openai/responses/gpt-6-astra", "P", workdir=tmp, cell_schema={"type": "object"}, + key_fields=["question_id"], allowed_keys=["a"], submit_cells=tool, submit_forecast=tool, + cwd=tmp, harness_run=fake) + self.assertEqual(seen["model"], "openai/gpt-6-astra") + self.assertEqual(seen["system"], " ") + self.assertEqual(seen["thinking"], "xhigh") + self.assertEqual(seen["research_tools"], ("web_search", "read_page")) + self.assertEqual({k: seen[k] for k in baseline.LOOP}, + {"min_research": 0, "floor_counts": "returned", "withhold_final": True, + "warn_missing_keys": False, "max_turns": 100}) + self.assertEqual(set(seen["tool_parameters"]), {"web_search", "read_page", "submit_cells", "submit_forecast"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/redlines/llm.py b/redlines/llm.py index 0b5d7d7..68398d3 100644 --- a/redlines/llm.py +++ b/redlines/llm.py @@ -91,8 +91,9 @@ def load_keys(): # effort also governs how many tool calls the model makes, which is the # point. REDLINES_REASONING overrides for an experiment ("none" turns # thinking off). -REASONING = {"anthropic": "max", "openai": "xhigh"} -REASONING_DEFAULT = "high" +# The table is the shared panel's (fri_forecaster.panel, since 2026-11-11). +from . import eci as _eci # noqa: E402,F401 (puts packages/fri-forecaster on the path in a bare checkout) +from fri_forecaster.panel import REASONING, REASONING_DEFAULT # noqa: E402 def reasoning_for(model): diff --git a/redlines/pi_loop.py b/redlines/pi_loop.py index dd17967..df099f4 100644 --- a/redlines/pi_loop.py +++ b/redlines/pi_loop.py @@ -28,12 +28,10 @@ DEFAULT_TIMEOUT = 3 * 3600 -def pi_model(model): - """The registry's id -> pi's. The registry's ids name OpenAI's Responses - API by a path segment (`openai/responses/...`, the route the litellm loop - took, and what rows and data/auto_models.json record); pi picks the API - itself.""" - return model.replace("openai/responses/", "openai/", 1) +# The registry's id -> pi's: the registry's ids name OpenAI's Responses API by a +# path segment (`openai/responses/...`, what rows and data/auto_models.json +# record); pi picks the API itself. Shared with every dashboard's panel. +from fri_forecaster.panel import pi_model # noqa: E402,F401 def call_tools(model, prompt, tools, final_tool, *, max_iters=6, system=None, diff --git a/redlines/registry.py b/redlines/registry.py index 818f017..24884c9 100644 --- a/redlines/registry.py +++ b/redlines/registry.py @@ -23,6 +23,12 @@ 3-4 by ECI; the specific list goes stale fast, so never inherit one. The hand-typed frontier five survive as the named set `frontier5`, the panel the series ran on until then. + +Since 2026-11-11 the rule (top k, one seat per family) and the rows every +dashboard shares (key, family, label, id, Epoch name) are +fri_forecaster.panel, beside the index reader; this module adds AIRO's +columns (colors, chart roles, the pinned ECI vintage) and binds the rule to +AIRO's snapshots under data/. """ from datetime import date import functools @@ -31,7 +37,8 @@ from pathlib import Path from . import autoreg as _autoreg -from . import eci as _eci +from . import eci as _eci # also puts packages/fri-forecaster on the path +from fri_forecaster import panel as _panel REPO_ROOT = Path(__file__).resolve().parent.parent # Graph 4's vintage. Models already in this file keep its values, so the @@ -98,57 +105,63 @@ def load_epoch_index(csv_path: Path = EPOCH_CSV, supplement: Path | None = None) # into the panel, the views can draw it -- none of those has been drawn yet, # and they are chosen only to sit apart from the eight above and from the # cause palette (CATEGORIES, data/autoarc_ladder.json). -# family: the line whose versions supersede one another, for the panel's -# within-family rule (project lead, 2026-09-08: skip Fable 5 when Fable 5.1 -# is seated). One seat per family, the highest-ranked member takes it. The -# assignment is by hand and is a judgement: Claude's tiers (Fable, Opus, -# Sonnet, Haiku) are separate lines; OpenAI's plain GPT-n are one line, the -# Pro tier another, and the code-named GPT-5.6 Sol / GPT-6 Astra each their -# own -- so the 2026-09-08 panel seats Astra beside GPT-5.5 Pro. Fold two -# families into one here if that ever reads wrong; nothing else changes. -_ROWS = [ - # key, family, label, litellm_id, fb_name, epoch_name, color, architecture, roles - ("gpt-3.5-turbo", "gpt", "GPT-3.5", "openai/gpt-3.5-turbo-0125", None, "GPT-3.5 Turbo (Jan 2024)", "#6b7280", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("llama-3.3-70b", "llama", "Llama-3.3-70B", "deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo", None, "Llama 3.3 70B", "#5b8c2a", "dense", ["g4_ladder", "pandemic_smoke"]), - ("gpt-4o", "gpt", "GPT-4o", "openai/gpt-4o", None, "GPT-4o (Nov 2024)", "#b8860b", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("deepseek-v3", "deepseek", "DeepSeek-V3", "deepinfra/deepseek-ai/DeepSeek-V3", None, "DeepSeek-V3", "#1b5e9e", "moe", ["g4_ladder", "pandemic_smoke"]), - ("claude-sonnet-4-5", "claude-sonnet", "Sonnet 4.5", "anthropic/claude-sonnet-4-5-20250929", None, "Claude Sonnet 4.5", "#c2185b", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("gpt-5", "gpt", "GPT-5", "openai/gpt-5-2025-08-07", None, "GPT-5", "#2e7d32", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("claude-opus-4-6", "claude-opus", "Opus 4.6", "anthropic/claude-opus-4-6", None, "Claude Opus 4.6", "#795548", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("claude-opus-4-8", "claude-opus", "Opus 4.8", "anthropic/claude-opus-4-8", "Claude-Opus-4-8", "Claude Opus 4.8", "#d97e2e", "undisclosed", ["g4_ladder", "frontier5", "graph3_dash"]), - ("claude-fable-5", "claude-fable", "Fable 5", "anthropic/claude-fable-5", None, "Claude Fable 5", "#b52a55", "undisclosed", ["g4_ladder", "frontier5"]), - ("gpt-4.1", "gpt", "GPT-4.1", "openai/gpt-4.1-2025-04-14", None, "GPT-4.1", "#00838f", "undisclosed", ["g4_gapfiller", "pandemic_smoke"]), - ("claude-haiku-4-5", "claude-haiku", "Haiku 4.5", "anthropic/claude-haiku-4-5-20251001", None, "Claude Haiku 4.5", "#e0a020", "undisclosed", ["g4_gapfiller", "pandemic_smoke"]), - ("gpt-5.5-pro", "gpt-pro", "GPT-5.5 Pro", "openai/gpt-5.5-pro", None, "GPT-5.5 Pro", "#c25a2e", "undisclosed", []), # ALT_HIGH_ANCHOR; in the ECI panel since 2026-08-28 - ("gpt-5.5", "gpt", "GPT-5.5", "openai/gpt-5.5", "GPT-5.5-2026-04-23", "GPT-5.5", "#4a76c9", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), - ("gemini-3.1-pro", "gemini-pro", "Gemini 3.1 Pro", "gemini/gemini-3.1-pro-preview", "Gemini-3.1-Pro-Preview", "Gemini 3.1 Pro", "#1f8a70", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), - ("grok-4.20", "grok", "Grok 4.20", "xai/grok-4.20", "Grok-4.20-0309-Reasoning", "Grok 4.20", "#8a63c9", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), +# The rows themselves -- which models there are, their families, their ids -- +# are fri_forecaster.panel.MODELS (packages/fri-forecaster), shared by every +# dashboard since 2026-11-11. What is AIRO's is below, by the row's key: +# fb_name, color, architecture, roles. A shared row with no entry here fails +# the import by name: any runnable row can be ranked into the panel and must +# be drawable. +_AIRO = { + # key: (fb_name, color, architecture, roles) + "gpt-3.5-turbo": (None, "#6b7280", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "llama-3.3-70b": (None, "#5b8c2a", "dense", ["g4_ladder", "pandemic_smoke"]), + "gpt-4o": (None, "#b8860b", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "deepseek-v3": (None, "#1b5e9e", "moe", ["g4_ladder", "pandemic_smoke"]), + "claude-sonnet-4-5": (None, "#c2185b", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "gpt-5": (None, "#2e7d32", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "claude-opus-4-6": (None, "#795548", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "claude-opus-4-8": ("Claude-Opus-4-8", "#d97e2e", "undisclosed", ["g4_ladder", "frontier5", "graph3_dash"]), + "claude-fable-5": (None, "#b52a55", "undisclosed", ["g4_ladder", "frontier5"]), + "gpt-4.1": (None, "#00838f", "undisclosed", ["g4_gapfiller", "pandemic_smoke"]), + "claude-haiku-4-5": (None, "#e0a020", "undisclosed", ["g4_gapfiller", "pandemic_smoke"]), + "gpt-5.5-pro": (None, "#c25a2e", "undisclosed", []), # ALT_HIGH_ANCHOR; in the ECI panel since 2026-08-28 + "gpt-5.5": ("GPT-5.5-2026-04-23", "#4a76c9", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), + "gemini-3.1-pro": ("Gemini-3.1-Pro-Preview", "#1f8a70", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), + "grok-4.20": ("Grok-4.20-0309-Reasoning", "#8a63c9", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), # Added 2026-08-28 with the ECI top-4 arm (docs/eci-top4-arm.md), now # panel members by rank. On Graph 4's ladder since 2026-08-28 evening # (project lead: Graph 4 should get Opus 5 and GPT-5.6 Sol) -- run through both # simulators exactly as the rest. Their ECI is the newest snapshot's, not # the pinned vintage (neither existed on 2026-07-07); the gap is about a # point (docs/model-set.md). GPT-5.5 Pro stays off: ~$100-250 per sim. - ("claude-opus-5", "claude-opus", "Opus 5", "anthropic/claude-opus-5", None, "Claude Opus 5", "#7a4fa3", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("gpt-5.6-sol", "gpt-sol", "GPT-5.6 Sol", "openai/gpt-5.6-sol", None, "GPT-5.6 Sol", "#2f6f9f", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "claude-opus-5": (None, "#7a4fa3", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "gpt-5.6-sol": (None, "#2f6f9f", "undisclosed", ["g4_ladder", "pandemic_smoke"]), # Added 2026-09-08 with the ECI snapshot of that day: the index's new top # two. Panel members by rank from the first run after; not on Graph 4's - # ladder (never run through the simulators). GPT-6 Astra goes through - # litellm's Responses bridge (openai/responses/...): on /v1/chat/completions - # OpenAI refuses function tools for it under any reasoning_effort, and the - # grounding tools are the point (probed on the box 2026-09-08). - ("claude-fable-5-1", "claude-fable", "Fable 5.1", "anthropic/claude-fable-5-1", None, "Claude Fable 5.1", "#8e2a63", "undisclosed", []), - ("gpt-6-astra", "gpt-astra", "GPT-6 Astra", "openai/responses/gpt-6-astra", None, "GPT-6 Astra", "#0e7c7b", "undisclosed", []), + # ladder (never run through the simulators). + "claude-fable-5-1": (None, "#8e2a63", "undisclosed", []), + "gpt-6-astra": (None, "#0e7c7b", "undisclosed", []), # Added 2026-10-01: Epoch's index of that day ranks them first and third # (167.3 and 165.2), so they take the claude-opus seat from Opus 5 and a # claude-sonnet seat of their own; GPT-5.5 Pro leaves the panel. Probed on - # the box the same day: max (adaptive) reasoning accepted, priced by - # litellm 1.96.2, and -- like Fable 5.1 -- a forced tool_choice refused - # (the litellm loop's NO_FORCED_TOOL_CHOICE; pi never forces one). Not on + # the box the same day: max (adaptive) reasoning accepted, and -- like + # Fable 5.1 -- a forced tool_choice refused (pi never forces one). Not on # Graph 4's ladder. - ("claude-opus-5-5", "claude-opus", "Opus 5.5", "anthropic/claude-opus-5-5", None, "Claude Opus 5.5", "#4f5bd5", "undisclosed", []), - ("claude-sonnet-5-5", "claude-sonnet", "Sonnet 5.5", "anthropic/claude-sonnet-5-5", None, "Claude Sonnet 5.5", "#d4892a", "undisclosed", []), -] + "claude-opus-5-5": (None, "#4f5bd5", "undisclosed", []), + "claude-sonnet-5-5": (None, "#d4892a", "undisclosed", []), +} + + +def _rows(): + missing = [m["key"] for m in _panel.MODELS if m["key"] not in _AIRO] + if missing: + raise KeyError(f"fri_forecaster.panel.MODELS rows with no entry in redlines/registry.py _AIRO " + f"(fb_name, color, architecture, roles): {', '.join(missing)}") + return [(m["key"], m["family"], m["label"], m["model"], _AIRO[m["key"]][0], m["epoch_name"], + *_AIRO[m["key"]][1:]) for m in _panel.MODELS] + + +_ROWS = _rows() # Graph-4 clean-subset rule (TODO.md Phase 2). A model is excluded from the # "clean subset" trend iff its architecture is a known-confounded one -- see @@ -263,9 +276,9 @@ def frontier_models() -> list[tuple[str, str]]: # a coin-flip says so in the log. ONE SEAT PER FAMILY (2026-09-08): a model # whose family already holds a seat is passed over, named in a WARN, and the # next runnable model takes the seat -- two versions of one model are one -# opinion twice, not two members (see `family` on _ROWS). -PANEL_K = 4 -PANEL_SET = "eci_topk" +# opinion twice, not two members (see `family` in fri_forecaster.panel). +PANEL_K = _panel.PANEL_K +PANEL_SET = _panel.PANEL_SET def _warn(msg): @@ -275,48 +288,11 @@ def _warn(msg): @functools.lru_cache(maxsize=None) def _select(k, snapshot): """-> (members, warnings): members are (registry row, index record) pairs, - best first. Cached per (k, snapshot) so a build that asks five views' - worth of times reads the CSV once and warns once.""" + best first. The rule is fri_forecaster.panel.select; this binds it to + AIRO's rows and snapshots. Cached per (k, snapshot) so a build that asks + five views' worth of times reads the CSV once and warns once.""" day, path = snapshot - by_epoch = {m["epoch_name"]: m for m in MODELS} - ranked = _eci.ranked(path) - chosen, unrunnable, passed, warnings = [], [], [], [] - msg = _eci.stale_message() - if msg: - warnings.append(msg) - seated = {} - for r in ranked: - m = by_epoch.get(r["model"]) - if m is None: - unrunnable.append(r) - continue - if m["family"] in seated: - passed.append((m, r, seated[m["family"]])) - continue - chosen.append((m, r)) - seated[m["family"]] = m - if len(chosen) == k: - break - for m, r, holder in passed: - warnings.append(f"passed over {r['model']} ({r['eci']:g}): its family " - f"({m['family']}) already holds a seat through {holder['epoch_name']}; " - f"the next runnable model took the seat") - if len(chosen) < k: - warnings.append(f"only {len(chosen)} of the top {k} in {path.name} are models the " - f"registry can run; the panel is short") - if chosen: - cut = chosen[-1][1] - above = [r["model"] for r in unrunnable if (r["eci"], -r["rank"]) > (cut["eci"], -cut["rank"])] - if above: - warnings.append(f"the index ranks {', '.join(above)} above the panel's cut but the " - f"registry cannot run them (no row in redlines/registry.py, and redlines.autoreg " - f"could not add one -- its log line says why); " - f"skipped, the next runnable model took the seat") - tied = [r["model"] for r in ranked if r["eci"] == cut["eci"] - and r["model"] not in {m["epoch_name"] for m, _ in chosen}] - if tied: - warnings.append(f"tie at the cut: {cut['model']} ({cut['eci']:g}) is in the panel " - f"and {', '.join(tied)} ({cut['eci']:g}) out, on Epoch's rank order") + chosen, warnings = _panel.select(_eci.ranked(path), MODELS, k, path.name, _eci.stale_message()) for w in warnings: _warn(w) return tuple(chosen), tuple(warnings) diff --git a/tests/test_shared_panel.py b/tests/test_shared_panel.py new file mode 100644 index 0000000..f6ee21a --- /dev/null +++ b/tests/test_shared_panel.py @@ -0,0 +1,32 @@ +"""AIRO reads its panel and its Baseline condition from the harness (fri_forecaster.panel, +fri_forecaster.baseline), where other dashboards read them too: these pin that the two agree.""" +import sys +import unittest +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +for p in (REPO, REPO / "packages" / "fri-forecaster"): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +from fri_forecaster import baseline, panel # noqa: E402 + + +class TestSharedWithAiro(unittest.TestCase): + def test_airo_rows_are_the_shared_rows(self): + from redlines import registry + hand = [m for m in registry.MODELS if not m.get("auto")] + self.assertEqual([(m["key"], m["family"], m["label"], m["litellm_id"], m["epoch_name"]) for m in hand], + [tuple(m[f] for f in panel._FIELDS) for m in panel.MODELS]) + self.assertEqual(registry.PANEL_K, panel.PANEL_K) + + def test_airo_minimal_instrument_is_the_baseline_condition(self): + sys.path.insert(0, str(REPO / "code")) + import run_unified as ru + self.assertEqual((ru.MINIMAL_SYSTEM, ru.MINIMAL_MAX_TURNS), (baseline.SYSTEM, baseline.MAX_TURNS)) + airo = ru.minimal_tools(ru.research_tools([{"key": "a", "label": "A"}])[0]) + self.assertEqual([{k: t[k] for k in ("name", "description", "parameters")} for t in airo], baseline.research_tools()) + + +if __name__ == "__main__": + unittest.main()