diff --git a/README.md b/README.md index 050fc6a..f4ac625 100644 --- a/README.md +++ b/README.md @@ -283,7 +283,8 @@ packages/fri-forecaster/ the FRI Forecaster harness -- pi, the research tools redlines/ the pipeline package (stdlib only) __main__.py CLI: build / assemble / export-fb / paper config.py paths (env-overridable), round pins, thresholds - registry.py the single model table (ids, labels, ECI, colors) + panel() + registry.py AIRO's model table (colors, chart roles, pinned ECI) over the shared rows and + panel rule in packages/fri-forecaster (fri_forecaster.panel) eci.py the Epoch Capabilities Index: fri_forecaster.eci (one implementation, shared with the ForecastBench rounds) bound to data/ and the pinned vintages questions.py schema-checked loaders for the Auto-ARC question set diff --git a/code/run_unified.py b/code/run_unified.py index 7b32e7e..992a653 100644 --- a/code/run_unified.py +++ b/code/run_unified.py @@ -103,6 +103,7 @@ from redlines.config import UNBATCHED # noqa: E402 from redlines.questions import (horizon_label, horizon_sort_key, # noqa: E402 resolves_on, severity_label) +from fri_forecaster import baseline as _baseline # noqa: E402 (on the path through redlines.eci) LADDER = os.path.join(ROOT, "data", "autoarc_ladder.json") CROSS = os.path.join(ROOT, "data", "autoarc_crosscutting.json") @@ -388,8 +389,12 @@ def apply_variant(prompt, variant): # far above any call seen (64 turns); it also keeps agent-box's nudge on, so a # reply cut at the output limit with no submission (Opus 5.5, 2026-10-19) # does not end the call. -MINIMAL_SYSTEM = " " -MINIMAL_MAX_TURNS = 100 +# +# Since 2026-11-11 this is the Baseline condition every dashboard shares +# (fri_forecaster.baseline, packages/fri-forecaster): the system prompt, the +# turn limit and the web_search description are read from there. +MINIMAL_SYSTEM = _baseline.SYSTEM +MINIMAL_MAX_TURNS = _baseline.MAX_TURNS MINIMAL_JOINT_HEAD = ("forecasting questions, grouped by cause of catastrophe. Forecast\n" "all of them together, in one pass -- and forecast every") MINIMAL_JOINT_RESEARCH = re.compile(r"Research before answering, in rounds:.*?applied to the same evidence\.\n", re.S) @@ -397,10 +402,7 @@ def apply_variant(prompt, variant): "you relied on (only pages you read), and any cells not yet delivered.") DEBRIEF_SENTENCE = re.compile(r" ?After it is accepted you will be asked one more thing.*?" r"research with those questions in mind\.", re.S) -MINIMAL_TOOLS = { - "web_search": "Search the web. Returns titled snippets with URLs and publication dates. " - "recent_days limits results to the last N days.", -} +MINIMAL_TOOLS = {"web_search": _baseline.WEB_SEARCH} def minimal_joint_prompt(prompt, variant=DEFAULT_VARIANT): diff --git a/packages/fri-forecaster/README.md b/packages/fri-forecaster/README.md index abaf0fd..b70be65 100644 --- a/packages/fri-forecaster/README.md +++ b/packages/fri-forecaster/README.md @@ -157,6 +157,20 @@ The result `harness.run` returns, folded from the outbox: Only the answers asked for (`allowed_keys`) are kept; anything else the model sends is dropped. +## The panel and the Baseline condition + +What FRI's dashboards share when they ask a standing panel of models (AIRO, EEAI): + +- `fri_forecaster.eci` fetches and ranks Epoch's Capabilities Index. +- `fri_forecaster.panel` picks the panel from it: the top 4 models we can run, one + seat per model family. The table of runnable models is in that file; a box may add + rows on its own in `auto_models.json` beside the snapshots (AIRO's `redlines.autoreg`). + `python -m fri_forecaster.panel --dir DIR [K]` prints it. +- `fri_forecaster.baseline` is how the panel is asked: no system prompt, web search + offered and never required, reasoning at the top level, no debrief. + `baseline.run(model, prompt, ...)` makes one such call with the caller's own answer + schema. AIRO's published `none` condition reads its constants from here. + ## Requirements - Python ≥ 3.11, stdlib only. diff --git a/packages/fri-forecaster/examples/round.env.example b/packages/fri-forecaster/examples/round.env.example index 9769123..9e2999d 100644 --- a/packages/fri-forecaster/examples/round.env.example +++ b/packages/fri-forecaster/examples/round.env.example @@ -15,5 +15,7 @@ ROUNDS_DIR="$HOME/forecastbench/rounds" # default: /rounds # place when its members are not the panel's top N. On AIRO's box the panel is AIRO's, # refreshed from Epoch's index first (code/refresh_panel.sh, which pages a change): # panel_members() { (cd "$HOME/Projects/airo" && { code/refresh_panel.sh >&2; "$PY" -m redlines.registry 6; }); } +# Or, with no AIRO checkout, the shared panel over a directory of snapshots: +# panel_members() { "$PY" -m fri_forecaster.panel --dir "$HOME/.cache/fri-forecaster/eci" 6; } # Without this, the round ranks Epoch's published index by name # (`python -m fri_forecaster.eci --fetch --names`) and can only warn when the top N moved. diff --git a/packages/fri-forecaster/fri_forecaster/baseline.py b/packages/fri-forecaster/fri_forecaster/baseline.py new file mode 100644 index 0000000..6212c8b --- /dev/null +++ b/packages/fri-forecaster/fri_forecaster/baseline.py @@ -0,0 +1,73 @@ +"""The Baseline condition: how a dashboard asks the panel. + +One call per panel member (fri_forecaster.panel), with nothing in it but the +dashboard's own prompt: + + no system prompt one space, because pi puts its own coding-assistant + prompt in place of an empty one. + research is offered, web_search and read_page (Tavily, tools.py), described + never required by what they do; no research floor, no coverage rule. + no debrief the call ends with the answer. + reasoning at the top panel.reasoning_for. + a turn limit of 100 a runaway guard far above any call seen (64 turns); it + also keeps the harness's nudge on, so a reply cut at + the output limit with no submission does not end the + call. + +This is AIRO's published condition `none` (its minimal instrument, +combined-v10, 2026-10-19: the prompt-variant experiments of 2026-10-14..19 +found the instrument's guidance moved no headline forecast), labelled +"Baseline" on its dashboard. AIRO's runner reads the constants below, so the +two cannot drift; a dashboard that asks something other than probabilities +calls run() with its own answer schema. +""" +import copy +import os +import tempfile + +from . import panel +from .tools import READ_PAGE_TOOL, WEB_SEARCH_TOOL + +SYSTEM = " " +MAX_TURNS = 100 +# A whole call, research included. +TIMEOUT = 3 * 3600 +WEB_SEARCH = ("Search the web. Returns titled snippets with URLs and publication dates. " + "recent_days limits results to the last N days.") +LOOP = {"min_research": 0, "floor_counts": "returned", "withhold_final": True, + "warn_missing_keys": False, "max_turns": MAX_TURNS} + + +def research_tools(): + """web_search and read_page as the condition offers them: {name, + description, parameters}. web_search loses `cause`, which serves only a + coverage rule.""" + search = {k: copy.deepcopy(WEB_SEARCH_TOOL[k]) for k in ("name", "description", "parameters")} + search["description"] = WEB_SEARCH + search["parameters"]["properties"].pop("cause", None) + read = {k: copy.deepcopy(READ_PAGE_TOOL[k]) for k in ("name", "description", "parameters")} + return [search, read] + + +def run(model, prompt, *, workdir, cell_schema, key_fields, allowed_keys, submit_cells, submit_forecast, + env=None, timeout=TIMEOUT, cwd=None, retries=0, harness_run=None): + """One Baseline call. `model` is a panel row's `model`; `submit_cells` and + `submit_forecast` are the dashboard's two answer tools as {description, + parameters} (the pieces along the way, and the final answer); the cells + they carry are `cell_schema`, keyed by `key_fields`, and only + `allowed_keys` are kept. -> harness.run's result.""" + if harness_run is None: + from . import harness + harness_run = harness.run + research = research_tools() + specs = {**{t["name"]: t for t in research}, "submit_cells": submit_cells, "submit_forecast": submit_forecast} + # pi writes its working directory into its system prompt: run it from an + # empty directory whose name says nothing about the project. + cwd = cwd or os.path.join(tempfile.gettempdir(), "w") + os.makedirs(cwd, exist_ok=True) + return harness_run(model=panel.pi_model(model), system=SYSTEM, prompt=prompt, workdir=workdir, cwd=cwd, + cell_schema=cell_schema, key_fields=list(key_fields), allowed_keys=list(allowed_keys), + tool_descriptions={n: s["description"] for n, s in specs.items()}, + tool_parameters={n: s["parameters"] for n, s in specs.items()}, + research_tools=tuple(t["name"] for t in research), cells_tool=True, + thinking=panel.reasoning_for(model), timeout=timeout, env=env, retries=retries, **LOOP) diff --git a/packages/fri-forecaster/fri_forecaster/panel.py b/packages/fri-forecaster/fri_forecaster/panel.py new file mode 100644 index 0000000..fffb18a --- /dev/null +++ b/packages/fri-forecaster/fri_forecaster/panel.py @@ -0,0 +1,231 @@ +"""The model panel: the top k of Epoch's index that we can run, one seat per family. + +Every FRI dashboard that asks a standing panel of frontier models asks THIS +panel (AIRO since 2026-08-28; EEAI; the ForecastBench rounds through `names +sync`). It is not a hand-typed list: it is the PANEL_K highest-ECI models in +the newest snapshot of Epoch's index (fri_forecaster.eci, beside this module) +that MODELS below can run, re-ranked every time it is asked for. Until +2026-11-11 the rule and the table lived in AIRO's redlines/registry.py while +the index reader lived here; they are one thing and now sit together. + + k = 4 averaging the top 3-4 by ECI beat the single best-ranked + model and closed the gap to the hindsight-best one; at 5 the + marginal model is already below the frontier (AIRO's + docs/eci-top4-arm.md). + one seat per a model whose family already holds a seat is passed over + family and the next runnable model takes it: two versions of one + model are one opinion twice (2026-09-08). + runnable a model is runnable when it has a row: a hand row in MODELS, + or one a dashboard's box added on its own after probing the + model (AIRO's redlines.autoreg writes auto_models.json + beside the snapshots). A hand row wins. + +Everything the rule does out of the ordinary is a warning, returned and +printed once: a stale snapshot, a model passed over for its family, a model +above the cut with no row, a tie at the cut, a short panel. + + python -m fri_forecaster.panel [--dir DIR] [K] + # the panel's top K (default PANEL_K), best first, one per line as + # ` TAB TAB `: what `names sync` reads +""" +import json +import os +import sys +from pathlib import Path + +from . import eci + +PANEL_K = 4 +PANEL_SET = "eci_topk" + +# ── The runnable models ────────────────────────────────────────────────────── +# key a stable slug, what a dashboard keys its own columns on (AIRO's +# colors and chart roles are in redlines/registry.py, by this key). +# family the line whose versions supersede one another, for the one-seat +# rule. By hand, and a judgement: Claude's tiers (Fable, Opus, +# Sonnet, Haiku) are separate lines; OpenAI's plain GPT-n are one +# line, the Pro tier another, and the code-named GPT-5.6 Sol / +# GPT-6 Astra each their own -- so the 2026-09-08 panel seats Astra +# beside GPT-5.5 Pro. Fold two families into one here if that ever +# reads wrong; nothing else changes. +# label the display name. +# model `provider/id`. OpenAI ids that name the Responses API by a path +# segment (`openai/responses/...`) are what AIRO's rows have always +# recorded; pi_model() gives pi's spelling. +# epoch_name Epoch's name for the model, exactly: the join to the index. +# +# Rows are never removed: a past run's panel is re-derived from its snapshot. +_FIELDS = ("key", "family", "label", "model", "epoch_name") +_ROWS = [ + ("gpt-3.5-turbo", "gpt", "GPT-3.5", "openai/gpt-3.5-turbo-0125", "GPT-3.5 Turbo (Jan 2024)"), + ("llama-3.3-70b", "llama", "Llama-3.3-70B", "deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo", "Llama 3.3 70B"), + ("gpt-4o", "gpt", "GPT-4o", "openai/gpt-4o", "GPT-4o (Nov 2024)"), + ("deepseek-v3", "deepseek", "DeepSeek-V3", "deepinfra/deepseek-ai/DeepSeek-V3", "DeepSeek-V3"), + ("claude-sonnet-4-5", "claude-sonnet", "Sonnet 4.5", "anthropic/claude-sonnet-4-5-20250929", "Claude Sonnet 4.5"), + ("gpt-5", "gpt", "GPT-5", "openai/gpt-5-2025-08-07", "GPT-5"), + ("claude-opus-4-6", "claude-opus", "Opus 4.6", "anthropic/claude-opus-4-6", "Claude Opus 4.6"), + ("claude-opus-4-8", "claude-opus", "Opus 4.8", "anthropic/claude-opus-4-8", "Claude Opus 4.8"), + ("claude-fable-5", "claude-fable", "Fable 5", "anthropic/claude-fable-5", "Claude Fable 5"), + ("gpt-4.1", "gpt", "GPT-4.1", "openai/gpt-4.1-2025-04-14", "GPT-4.1"), + ("claude-haiku-4-5", "claude-haiku", "Haiku 4.5", "anthropic/claude-haiku-4-5-20251001", "Claude Haiku 4.5"), + ("gpt-5.5-pro", "gpt-pro", "GPT-5.5 Pro", "openai/gpt-5.5-pro", "GPT-5.5 Pro"), + ("gpt-5.5", "gpt", "GPT-5.5", "openai/gpt-5.5", "GPT-5.5"), + ("gemini-3.1-pro", "gemini-pro", "Gemini 3.1 Pro", "gemini/gemini-3.1-pro-preview", "Gemini 3.1 Pro"), + ("grok-4.20", "grok", "Grok 4.20", "xai/grok-4.20", "Grok 4.20"), + # Added 2026-08-28 with the ECI top-4 arm. + ("claude-opus-5", "claude-opus", "Opus 5", "anthropic/claude-opus-5", "Claude Opus 5"), + ("gpt-5.6-sol", "gpt-sol", "GPT-5.6 Sol", "openai/gpt-5.6-sol", "GPT-5.6 Sol"), + # Added 2026-09-08: the index's new top two. GPT-6 Astra takes the + # Responses API: on /v1/chat/completions OpenAI refuses function tools for + # it under any reasoning effort, and the research tools are the point. + ("claude-fable-5-1", "claude-fable", "Fable 5.1", "anthropic/claude-fable-5-1", "Claude Fable 5.1"), + ("gpt-6-astra", "gpt-astra", "GPT-6 Astra", "openai/responses/gpt-6-astra", "GPT-6 Astra"), + # Added 2026-10-01: the index ranks them first and third, so they take the + # claude-opus seat from Opus 5 and a claude-sonnet seat of their own. + ("claude-opus-5-5", "claude-opus", "Opus 5.5", "anthropic/claude-opus-5-5", "Claude Opus 5.5"), + ("claude-sonnet-5-5", "claude-sonnet", "Sonnet 5.5", "anthropic/claude-sonnet-5-5", "Claude Sonnet 5.5"), +] +MODELS = [dict(zip(_FIELDS, row)) for row in _ROWS] + +AUTO_MODELS = "auto_models.json" + +# Reasoning level per provider, pi's `--thinking`: the top rung, which is +# spelled differently per provider (AIRO's project lead, 2026-09-02: reasoning +# at its maximum; Anthropic's effort also governs how many tool calls the model +# makes, which is the point). +REASONING = {"anthropic": "max", "openai": "xhigh"} +REASONING_DEFAULT = "high" + + +def reasoning_for(model): + """pi's `--thinking` for `model` (`provider/id`).""" + return REASONING.get(model.split("/", 1)[0], REASONING_DEFAULT) + + +def pi_model(model): + """A row's `model` -> pi's id for it. pi picks the API itself, so the + Responses-API path segment is dropped.""" + return model.replace("openai/responses/", "openai/", 1) + + +def auto_models_path(snapshot_dir=None): + """Where a box's automatic rows are: $FORECASTER_AUTO_MODELS, else + auto_models.json beside the snapshots.""" + explicit = os.environ.get("FORECASTER_AUTO_MODELS") + return Path(explicit) if explicit else eci._dir(snapshot_dir) / AUTO_MODELS + + +def load_auto(path): + """The automatic rows at `path` ([] if there is no file), in this module's + fields; an entry written by AIRO's autoreg spells `model` as `litellm_id`.""" + path = Path(path) + if not path.exists(): + return [] + out = [] + for e in json.loads(path.read_text(encoding="utf-8")): + row = {f: e.get(f) for f in _FIELDS} + row["model"] = row["model"] or e.get("litellm_id") + out.append(row) + return out + + +def models(auto=None, snapshot_dir=None): + """Every runnable model: the hand rows, then the automatic ones (from + `auto`, a path, or auto_models_path) that no hand row already covers.""" + out = [dict(m) for m in MODELS] + hand = {m["epoch_name"] for m in out} | {m["key"] for m in out} + for row in load_auto(auto or auto_models_path(snapshot_dir)): + if row["epoch_name"] not in hand and row["key"] not in hand: + out.append({**row, "auto": True}) + return out + + +def select(ranked, table, k=PANEL_K, snapshot_name="the snapshot", stale=None): + """THE RULE. `ranked` is the index best first (eci.ranked); `table` the + runnable rows, each with at least `epoch_name` and `family` (a dashboard + may pass its own rows, with its own extra columns, and gets them back). + -> (members, warnings): members are (row, index record) pairs, best + first; `stale` is a staleness warning to lead with, if any.""" + by_epoch = {m["epoch_name"]: m for m in table} + chosen, unrunnable, passed, warnings = [], [], [], [] + if stale: + warnings.append(stale) + seated = {} + for r in ranked: + m = by_epoch.get(r["model"]) + if m is None: + unrunnable.append(r) + continue + if m["family"] in seated: + passed.append((m, r, seated[m["family"]])) + continue + chosen.append((m, r)) + seated[m["family"]] = m + if len(chosen) == k: + break + for m, r, holder in passed: + warnings.append(f"passed over {r['model']} ({r['eci']:g}): its family " + f"({m['family']}) already holds a seat through {holder['epoch_name']}; " + f"the next runnable model took the seat") + if len(chosen) < k: + warnings.append(f"only {len(chosen)} of the top {k} in {snapshot_name} are models the " + f"registry can run; the panel is short") + if chosen: + cut = chosen[-1][1] + above = [r["model"] for r in unrunnable if (r["eci"], -r["rank"]) > (cut["eci"], -cut["rank"])] + if above: + warnings.append(f"the index ranks {', '.join(above)} above the panel's cut but the " + f"registry cannot run them (no row in fri_forecaster/panel.py, and no " + f"automatic one -- the autoreg log line says why); " + f"skipped, the next runnable model took the seat") + tied = [r["model"] for r in ranked if r["eci"] == cut["eci"] + and r["model"] not in {m["epoch_name"] for m, _ in chosen}] + if tied: + warnings.append(f"tie at the cut: {cut['model']} ({cut['eci']:g}) is in the panel " + f"and {', '.join(tied)} ({cut['eci']:g}) out, on Epoch's rank order") + return chosen, warnings + + +def panel(k=PANEL_K, snapshot=None, snapshot_dir=None, auto=None, warn=True): + """The panel for a dashboard with no table of its own: the k highest-ECI + runnable models of `snapshot` ((date, path); the newest in `snapshot_dir` + by default), best first. -> (members, provenance). A member is its row + plus `pi_model`, `reasoning`, and the index's `eci`, `rank` and + `organization`; `provenance` is what a run stamps on its rows (which + snapshot chose the panel, how old it was, the warnings).""" + day, path = snapshot or eci.latest(snapshot_dir) + stale = None if snapshot else eci.stale_message(snapshot_dir=snapshot_dir) + chosen, warnings = select(eci.ranked(path), models(auto, snapshot_dir), k, Path(path).name, stale) + if warn: + for w in warnings: + print(f"WARN: {w}", file=sys.stderr) + members = [{**m, "pi_model": pi_model(m["model"]), "reasoning": reasoning_for(m["model"]), + "eci": r["eci"], "rank": r["rank"], "organization": r.get("organization")} + for m, r in chosen] + provenance = {"set": PANEL_SET, "k": k, "snapshot": day.isoformat(), + "age_days": eci.age_days(snapshot_dir=snapshot_dir) if not snapshot else None, + "members": [{"label": m["label"], "eci": m["eci"], "rank": m["rank"]} for m in members], + "warnings": warnings} + return members, provenance + + +def main(argv=None): + args = list(sys.argv[1:] if argv is None else argv) + snapshot_dir = None + if "--dir" in args: + i = args.index("--dir") + snapshot_dir = args[i + 1] + del args[i:i + 2] + k = int(args[0]) if args else PANEL_K + try: + members, _ = panel(k, snapshot_dir=snapshot_dir) + except FileNotFoundError as e: + print(e, file=sys.stderr) + return 1 + for m in members: + print(m["epoch_name"], m["pi_model"], m["organization"] or "", sep="\t") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/packages/fri-forecaster/tests/test_panel.py b/packages/fri-forecaster/tests/test_panel.py new file mode 100644 index 0000000..8eff207 --- /dev/null +++ b/packages/fri-forecaster/tests/test_panel.py @@ -0,0 +1,118 @@ +"""The shared panel (top k by ECI, one seat per family) and the Baseline condition.""" +import csv +import json +import tempfile +import unittest +from datetime import date +from pathlib import Path + +from fri_forecaster import baseline, eci, panel + + +def _snapshot(tmp, day, rows): + """A rank-format snapshot: rows are (model, eci).""" + p = Path(tmp) / f"{eci.SNAPSHOT_PREFIX}{day}.csv" + with open(p, "w", newline="") as f: + w = csv.writer(f) + w.writerow(["rank", "model", "eci", "ci_low", "ci_high", "retrieved"]) + for i, (m, e) in enumerate(rows, 1): + w.writerow([i, m, e, e - 3, e + 3, day]) + return date.fromisoformat(day), p + + +class TestRule(unittest.TestCase): + TABLE = [{"epoch_name": "A 2", "family": "a"}, {"epoch_name": "A 1", "family": "a"}, + {"epoch_name": "B 1", "family": "b"}, {"epoch_name": "C 1", "family": "c"}] + + def _ranked(self, rows): + return [{"model": m, "eci": e, "rank": i} for i, (m, e) in enumerate(rows, 1)] + + def test_top_k_one_seat_per_family_and_what_it_says(self): + ranked = self._ranked([("A 2", 170), ("X 9", 169), ("A 1", 168), ("B 1", 160), ("C 1", 150)]) + members, warnings = panel.select(ranked, self.TABLE, 2, "snap.csv") + self.assertEqual([m["epoch_name"] for m, _ in members], ["A 2", "B 1"]) + self.assertTrue(any("passed over A 1 (168)" in w and "A 2" in w for w in warnings), warnings) + self.assertTrue(any("X 9" in w and "cannot run" in w for w in warnings), warnings) + + def test_a_tie_at_the_cut_and_a_short_panel(self): + ranked = self._ranked([("B 1", 160), ("C 1", 160)]) + members, warnings = panel.select(ranked, self.TABLE, 1, "snap.csv") + self.assertEqual(members[0][0]["epoch_name"], "B 1") + self.assertTrue(any(w.startswith("tie at the cut") and "C 1" in w for w in warnings), warnings) + _, warnings = panel.select(ranked, self.TABLE, 4, "snap.csv", stale="old") + self.assertEqual(warnings[0], "old") + self.assertTrue(any("only 2 of the top 4 in snap.csv" in w and "panel is short" in w for w in warnings)) + + def test_rows_come_back_as_given(self): + table = [dict(r, color="#123456") for r in self.TABLE] + members, _ = panel.select(self._ranked([("C 1", 150)]), table, 1) + self.assertEqual(members[0][0]["color"], "#123456") + + +class TestPanel(unittest.TestCase): + def test_table_is_well_formed(self): + keys = [m["key"] for m in panel.MODELS] + self.assertEqual(len(keys), len(set(keys))) + self.assertEqual(len({m["epoch_name"] for m in panel.MODELS}), len(panel.MODELS)) + for m in panel.MODELS: + self.assertTrue(all(m[f] for f in panel._FIELDS), m) + self.assertIn("/", m["model"]) + + def test_panel_from_a_snapshot_directory_with_an_automatic_row(self): + with tempfile.TemporaryDirectory() as tmp: + snap = _snapshot(tmp, "2026-11-01", [("Nova 9", 180), ("Claude Opus 5.5", 167), ("Claude Opus 5", 163), + ("GPT-6 Astra", 166), ("Claude Fable 5.1", 164)]) + members, prov = panel.panel(3, snapshot=snap, snapshot_dir=tmp, warn=False) + self.assertEqual([m["key"] for m in members], ["claude-opus-5-5", "gpt-6-astra", "claude-fable-5-1"]) + self.assertTrue(any("Nova 9" in w for w in prov["warnings"])) + # A box's automatic row (AIRO's autoreg spells `model` litellm_id) makes Nova runnable. + (Path(tmp) / panel.AUTO_MODELS).write_text(json.dumps([ + {"key": "nova-9", "family": "nova", "label": "Nova 9", "litellm_id": "openai/responses/nova-9", + "epoch_name": "Nova 9", "color": "#000000"}, + {"key": "claude-opus-5-5", "family": "x", "label": "x", "litellm_id": "x/x", "epoch_name": "Claude Opus 5.5"}])) + members, prov = panel.panel(3, snapshot=snap, snapshot_dir=tmp, warn=False) + self.assertEqual([m["key"] for m in members], ["nova-9", "claude-opus-5-5", "gpt-6-astra"]) + self.assertEqual(members[0]["pi_model"], "openai/nova-9") + self.assertEqual(members[0]["reasoning"], "xhigh") + self.assertEqual(members[1]["family"], "claude-opus") # the hand row wins + self.assertEqual(prov["snapshot"], "2026-11-01") + self.assertEqual([m["label"] for m in prov["members"]], ["Nova 9", "Opus 5.5", "GPT-6 Astra"]) + + def test_ids_and_reasoning(self): + self.assertEqual(panel.pi_model("openai/responses/gpt-6-astra"), "openai/gpt-6-astra") + self.assertEqual(panel.pi_model("anthropic/claude-opus-5-5"), "anthropic/claude-opus-5-5") + self.assertEqual(panel.reasoning_for("anthropic/claude-opus-5-5"), "max") + self.assertEqual(panel.reasoning_for("gemini/gemini-3.1-pro-preview"), "high") + + +class TestBaseline(unittest.TestCase): + def test_research_is_offered_plainly(self): + search, read = baseline.research_tools() + self.assertEqual((search["name"], read["name"]), ("web_search", "read_page")) + self.assertEqual(search["description"], baseline.WEB_SEARCH) + self.assertNotIn("cause", search["parameters"]["properties"]) + self.assertEqual(search["parameters"]["required"], ["query"]) + + def test_run_is_the_minimal_loop(self): + seen = {} + + def fake(**kw): + seen.update(kw) + return {"cells": {}} + tool = {"description": "d", "parameters": {"type": "object"}} + with tempfile.TemporaryDirectory() as tmp: + baseline.run("openai/responses/gpt-6-astra", "P", workdir=tmp, cell_schema={"type": "object"}, + key_fields=["question_id"], allowed_keys=["a"], submit_cells=tool, submit_forecast=tool, + cwd=tmp, harness_run=fake) + self.assertEqual(seen["model"], "openai/gpt-6-astra") + self.assertEqual(seen["system"], " ") + self.assertEqual(seen["thinking"], "xhigh") + self.assertEqual(seen["research_tools"], ("web_search", "read_page")) + self.assertEqual({k: seen[k] for k in baseline.LOOP}, + {"min_research": 0, "floor_counts": "returned", "withhold_final": True, + "warn_missing_keys": False, "max_turns": 100}) + self.assertEqual(set(seen["tool_parameters"]), {"web_search", "read_page", "submit_cells", "submit_forecast"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/redlines/llm.py b/redlines/llm.py index 0b5d7d7..68398d3 100644 --- a/redlines/llm.py +++ b/redlines/llm.py @@ -91,8 +91,9 @@ def load_keys(): # effort also governs how many tool calls the model makes, which is the # point. REDLINES_REASONING overrides for an experiment ("none" turns # thinking off). -REASONING = {"anthropic": "max", "openai": "xhigh"} -REASONING_DEFAULT = "high" +# The table is the shared panel's (fri_forecaster.panel, since 2026-11-11). +from . import eci as _eci # noqa: E402,F401 (puts packages/fri-forecaster on the path in a bare checkout) +from fri_forecaster.panel import REASONING, REASONING_DEFAULT # noqa: E402 def reasoning_for(model): diff --git a/redlines/pi_loop.py b/redlines/pi_loop.py index dd17967..df099f4 100644 --- a/redlines/pi_loop.py +++ b/redlines/pi_loop.py @@ -28,12 +28,10 @@ DEFAULT_TIMEOUT = 3 * 3600 -def pi_model(model): - """The registry's id -> pi's. The registry's ids name OpenAI's Responses - API by a path segment (`openai/responses/...`, the route the litellm loop - took, and what rows and data/auto_models.json record); pi picks the API - itself.""" - return model.replace("openai/responses/", "openai/", 1) +# The registry's id -> pi's: the registry's ids name OpenAI's Responses API by a +# path segment (`openai/responses/...`, what rows and data/auto_models.json +# record); pi picks the API itself. Shared with every dashboard's panel. +from fri_forecaster.panel import pi_model # noqa: E402,F401 def call_tools(model, prompt, tools, final_tool, *, max_iters=6, system=None, diff --git a/redlines/registry.py b/redlines/registry.py index 818f017..24884c9 100644 --- a/redlines/registry.py +++ b/redlines/registry.py @@ -23,6 +23,12 @@ 3-4 by ECI; the specific list goes stale fast, so never inherit one. The hand-typed frontier five survive as the named set `frontier5`, the panel the series ran on until then. + +Since 2026-11-11 the rule (top k, one seat per family) and the rows every +dashboard shares (key, family, label, id, Epoch name) are +fri_forecaster.panel, beside the index reader; this module adds AIRO's +columns (colors, chart roles, the pinned ECI vintage) and binds the rule to +AIRO's snapshots under data/. """ from datetime import date import functools @@ -31,7 +37,8 @@ from pathlib import Path from . import autoreg as _autoreg -from . import eci as _eci +from . import eci as _eci # also puts packages/fri-forecaster on the path +from fri_forecaster import panel as _panel REPO_ROOT = Path(__file__).resolve().parent.parent # Graph 4's vintage. Models already in this file keep its values, so the @@ -98,57 +105,63 @@ def load_epoch_index(csv_path: Path = EPOCH_CSV, supplement: Path | None = None) # into the panel, the views can draw it -- none of those has been drawn yet, # and they are chosen only to sit apart from the eight above and from the # cause palette (CATEGORIES, data/autoarc_ladder.json). -# family: the line whose versions supersede one another, for the panel's -# within-family rule (project lead, 2026-09-08: skip Fable 5 when Fable 5.1 -# is seated). One seat per family, the highest-ranked member takes it. The -# assignment is by hand and is a judgement: Claude's tiers (Fable, Opus, -# Sonnet, Haiku) are separate lines; OpenAI's plain GPT-n are one line, the -# Pro tier another, and the code-named GPT-5.6 Sol / GPT-6 Astra each their -# own -- so the 2026-09-08 panel seats Astra beside GPT-5.5 Pro. Fold two -# families into one here if that ever reads wrong; nothing else changes. -_ROWS = [ - # key, family, label, litellm_id, fb_name, epoch_name, color, architecture, roles - ("gpt-3.5-turbo", "gpt", "GPT-3.5", "openai/gpt-3.5-turbo-0125", None, "GPT-3.5 Turbo (Jan 2024)", "#6b7280", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("llama-3.3-70b", "llama", "Llama-3.3-70B", "deepinfra/meta-llama/Llama-3.3-70B-Instruct-Turbo", None, "Llama 3.3 70B", "#5b8c2a", "dense", ["g4_ladder", "pandemic_smoke"]), - ("gpt-4o", "gpt", "GPT-4o", "openai/gpt-4o", None, "GPT-4o (Nov 2024)", "#b8860b", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("deepseek-v3", "deepseek", "DeepSeek-V3", "deepinfra/deepseek-ai/DeepSeek-V3", None, "DeepSeek-V3", "#1b5e9e", "moe", ["g4_ladder", "pandemic_smoke"]), - ("claude-sonnet-4-5", "claude-sonnet", "Sonnet 4.5", "anthropic/claude-sonnet-4-5-20250929", None, "Claude Sonnet 4.5", "#c2185b", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("gpt-5", "gpt", "GPT-5", "openai/gpt-5-2025-08-07", None, "GPT-5", "#2e7d32", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("claude-opus-4-6", "claude-opus", "Opus 4.6", "anthropic/claude-opus-4-6", None, "Claude Opus 4.6", "#795548", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("claude-opus-4-8", "claude-opus", "Opus 4.8", "anthropic/claude-opus-4-8", "Claude-Opus-4-8", "Claude Opus 4.8", "#d97e2e", "undisclosed", ["g4_ladder", "frontier5", "graph3_dash"]), - ("claude-fable-5", "claude-fable", "Fable 5", "anthropic/claude-fable-5", None, "Claude Fable 5", "#b52a55", "undisclosed", ["g4_ladder", "frontier5"]), - ("gpt-4.1", "gpt", "GPT-4.1", "openai/gpt-4.1-2025-04-14", None, "GPT-4.1", "#00838f", "undisclosed", ["g4_gapfiller", "pandemic_smoke"]), - ("claude-haiku-4-5", "claude-haiku", "Haiku 4.5", "anthropic/claude-haiku-4-5-20251001", None, "Claude Haiku 4.5", "#e0a020", "undisclosed", ["g4_gapfiller", "pandemic_smoke"]), - ("gpt-5.5-pro", "gpt-pro", "GPT-5.5 Pro", "openai/gpt-5.5-pro", None, "GPT-5.5 Pro", "#c25a2e", "undisclosed", []), # ALT_HIGH_ANCHOR; in the ECI panel since 2026-08-28 - ("gpt-5.5", "gpt", "GPT-5.5", "openai/gpt-5.5", "GPT-5.5-2026-04-23", "GPT-5.5", "#4a76c9", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), - ("gemini-3.1-pro", "gemini-pro", "Gemini 3.1 Pro", "gemini/gemini-3.1-pro-preview", "Gemini-3.1-Pro-Preview", "Gemini 3.1 Pro", "#1f8a70", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), - ("grok-4.20", "grok", "Grok 4.20", "xai/grok-4.20", "Grok-4.20-0309-Reasoning", "Grok 4.20", "#8a63c9", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), +# The rows themselves -- which models there are, their families, their ids -- +# are fri_forecaster.panel.MODELS (packages/fri-forecaster), shared by every +# dashboard since 2026-11-11. What is AIRO's is below, by the row's key: +# fb_name, color, architecture, roles. A shared row with no entry here fails +# the import by name: any runnable row can be ranked into the panel and must +# be drawable. +_AIRO = { + # key: (fb_name, color, architecture, roles) + "gpt-3.5-turbo": (None, "#6b7280", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "llama-3.3-70b": (None, "#5b8c2a", "dense", ["g4_ladder", "pandemic_smoke"]), + "gpt-4o": (None, "#b8860b", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "deepseek-v3": (None, "#1b5e9e", "moe", ["g4_ladder", "pandemic_smoke"]), + "claude-sonnet-4-5": (None, "#c2185b", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "gpt-5": (None, "#2e7d32", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "claude-opus-4-6": (None, "#795548", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "claude-opus-4-8": ("Claude-Opus-4-8", "#d97e2e", "undisclosed", ["g4_ladder", "frontier5", "graph3_dash"]), + "claude-fable-5": (None, "#b52a55", "undisclosed", ["g4_ladder", "frontier5"]), + "gpt-4.1": (None, "#00838f", "undisclosed", ["g4_gapfiller", "pandemic_smoke"]), + "claude-haiku-4-5": (None, "#e0a020", "undisclosed", ["g4_gapfiller", "pandemic_smoke"]), + "gpt-5.5-pro": (None, "#c25a2e", "undisclosed", []), # ALT_HIGH_ANCHOR; in the ECI panel since 2026-08-28 + "gpt-5.5": ("GPT-5.5-2026-04-23", "#4a76c9", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), + "gemini-3.1-pro": ("Gemini-3.1-Pro-Preview", "#1f8a70", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), + "grok-4.20": ("Grok-4.20-0309-Reasoning", "#8a63c9", "undisclosed", ["g4_ladder", "pandemic_smoke", "frontier5", "graph3_dash"]), # Added 2026-08-28 with the ECI top-4 arm (docs/eci-top4-arm.md), now # panel members by rank. On Graph 4's ladder since 2026-08-28 evening # (project lead: Graph 4 should get Opus 5 and GPT-5.6 Sol) -- run through both # simulators exactly as the rest. Their ECI is the newest snapshot's, not # the pinned vintage (neither existed on 2026-07-07); the gap is about a # point (docs/model-set.md). GPT-5.5 Pro stays off: ~$100-250 per sim. - ("claude-opus-5", "claude-opus", "Opus 5", "anthropic/claude-opus-5", None, "Claude Opus 5", "#7a4fa3", "undisclosed", ["g4_ladder", "pandemic_smoke"]), - ("gpt-5.6-sol", "gpt-sol", "GPT-5.6 Sol", "openai/gpt-5.6-sol", None, "GPT-5.6 Sol", "#2f6f9f", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "claude-opus-5": (None, "#7a4fa3", "undisclosed", ["g4_ladder", "pandemic_smoke"]), + "gpt-5.6-sol": (None, "#2f6f9f", "undisclosed", ["g4_ladder", "pandemic_smoke"]), # Added 2026-09-08 with the ECI snapshot of that day: the index's new top # two. Panel members by rank from the first run after; not on Graph 4's - # ladder (never run through the simulators). GPT-6 Astra goes through - # litellm's Responses bridge (openai/responses/...): on /v1/chat/completions - # OpenAI refuses function tools for it under any reasoning_effort, and the - # grounding tools are the point (probed on the box 2026-09-08). - ("claude-fable-5-1", "claude-fable", "Fable 5.1", "anthropic/claude-fable-5-1", None, "Claude Fable 5.1", "#8e2a63", "undisclosed", []), - ("gpt-6-astra", "gpt-astra", "GPT-6 Astra", "openai/responses/gpt-6-astra", None, "GPT-6 Astra", "#0e7c7b", "undisclosed", []), + # ladder (never run through the simulators). + "claude-fable-5-1": (None, "#8e2a63", "undisclosed", []), + "gpt-6-astra": (None, "#0e7c7b", "undisclosed", []), # Added 2026-10-01: Epoch's index of that day ranks them first and third # (167.3 and 165.2), so they take the claude-opus seat from Opus 5 and a # claude-sonnet seat of their own; GPT-5.5 Pro leaves the panel. Probed on - # the box the same day: max (adaptive) reasoning accepted, priced by - # litellm 1.96.2, and -- like Fable 5.1 -- a forced tool_choice refused - # (the litellm loop's NO_FORCED_TOOL_CHOICE; pi never forces one). Not on + # the box the same day: max (adaptive) reasoning accepted, and -- like + # Fable 5.1 -- a forced tool_choice refused (pi never forces one). Not on # Graph 4's ladder. - ("claude-opus-5-5", "claude-opus", "Opus 5.5", "anthropic/claude-opus-5-5", None, "Claude Opus 5.5", "#4f5bd5", "undisclosed", []), - ("claude-sonnet-5-5", "claude-sonnet", "Sonnet 5.5", "anthropic/claude-sonnet-5-5", None, "Claude Sonnet 5.5", "#d4892a", "undisclosed", []), -] + "claude-opus-5-5": (None, "#4f5bd5", "undisclosed", []), + "claude-sonnet-5-5": (None, "#d4892a", "undisclosed", []), +} + + +def _rows(): + missing = [m["key"] for m in _panel.MODELS if m["key"] not in _AIRO] + if missing: + raise KeyError(f"fri_forecaster.panel.MODELS rows with no entry in redlines/registry.py _AIRO " + f"(fb_name, color, architecture, roles): {', '.join(missing)}") + return [(m["key"], m["family"], m["label"], m["model"], _AIRO[m["key"]][0], m["epoch_name"], + *_AIRO[m["key"]][1:]) for m in _panel.MODELS] + + +_ROWS = _rows() # Graph-4 clean-subset rule (TODO.md Phase 2). A model is excluded from the # "clean subset" trend iff its architecture is a known-confounded one -- see @@ -263,9 +276,9 @@ def frontier_models() -> list[tuple[str, str]]: # a coin-flip says so in the log. ONE SEAT PER FAMILY (2026-09-08): a model # whose family already holds a seat is passed over, named in a WARN, and the # next runnable model takes the seat -- two versions of one model are one -# opinion twice, not two members (see `family` on _ROWS). -PANEL_K = 4 -PANEL_SET = "eci_topk" +# opinion twice, not two members (see `family` in fri_forecaster.panel). +PANEL_K = _panel.PANEL_K +PANEL_SET = _panel.PANEL_SET def _warn(msg): @@ -275,48 +288,11 @@ def _warn(msg): @functools.lru_cache(maxsize=None) def _select(k, snapshot): """-> (members, warnings): members are (registry row, index record) pairs, - best first. Cached per (k, snapshot) so a build that asks five views' - worth of times reads the CSV once and warns once.""" + best first. The rule is fri_forecaster.panel.select; this binds it to + AIRO's rows and snapshots. Cached per (k, snapshot) so a build that asks + five views' worth of times reads the CSV once and warns once.""" day, path = snapshot - by_epoch = {m["epoch_name"]: m for m in MODELS} - ranked = _eci.ranked(path) - chosen, unrunnable, passed, warnings = [], [], [], [] - msg = _eci.stale_message() - if msg: - warnings.append(msg) - seated = {} - for r in ranked: - m = by_epoch.get(r["model"]) - if m is None: - unrunnable.append(r) - continue - if m["family"] in seated: - passed.append((m, r, seated[m["family"]])) - continue - chosen.append((m, r)) - seated[m["family"]] = m - if len(chosen) == k: - break - for m, r, holder in passed: - warnings.append(f"passed over {r['model']} ({r['eci']:g}): its family " - f"({m['family']}) already holds a seat through {holder['epoch_name']}; " - f"the next runnable model took the seat") - if len(chosen) < k: - warnings.append(f"only {len(chosen)} of the top {k} in {path.name} are models the " - f"registry can run; the panel is short") - if chosen: - cut = chosen[-1][1] - above = [r["model"] for r in unrunnable if (r["eci"], -r["rank"]) > (cut["eci"], -cut["rank"])] - if above: - warnings.append(f"the index ranks {', '.join(above)} above the panel's cut but the " - f"registry cannot run them (no row in redlines/registry.py, and redlines.autoreg " - f"could not add one -- its log line says why); " - f"skipped, the next runnable model took the seat") - tied = [r["model"] for r in ranked if r["eci"] == cut["eci"] - and r["model"] not in {m["epoch_name"] for m, _ in chosen}] - if tied: - warnings.append(f"tie at the cut: {cut['model']} ({cut['eci']:g}) is in the panel " - f"and {', '.join(tied)} ({cut['eci']:g}) out, on Epoch's rank order") + chosen, warnings = _panel.select(_eci.ranked(path), MODELS, k, path.name, _eci.stale_message()) for w in warnings: _warn(w) return tuple(chosen), tuple(warnings) diff --git a/tests/test_shared_panel.py b/tests/test_shared_panel.py new file mode 100644 index 0000000..f6ee21a --- /dev/null +++ b/tests/test_shared_panel.py @@ -0,0 +1,32 @@ +"""AIRO reads its panel and its Baseline condition from the harness (fri_forecaster.panel, +fri_forecaster.baseline), where other dashboards read them too: these pin that the two agree.""" +import sys +import unittest +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +for p in (REPO, REPO / "packages" / "fri-forecaster"): + if str(p) not in sys.path: + sys.path.insert(0, str(p)) + +from fri_forecaster import baseline, panel # noqa: E402 + + +class TestSharedWithAiro(unittest.TestCase): + def test_airo_rows_are_the_shared_rows(self): + from redlines import registry + hand = [m for m in registry.MODELS if not m.get("auto")] + self.assertEqual([(m["key"], m["family"], m["label"], m["litellm_id"], m["epoch_name"]) for m in hand], + [tuple(m[f] for f in panel._FIELDS) for m in panel.MODELS]) + self.assertEqual(registry.PANEL_K, panel.PANEL_K) + + def test_airo_minimal_instrument_is_the_baseline_condition(self): + sys.path.insert(0, str(REPO / "code")) + import run_unified as ru + self.assertEqual((ru.MINIMAL_SYSTEM, ru.MINIMAL_MAX_TURNS), (baseline.SYSTEM, baseline.MAX_TURNS)) + airo = ru.minimal_tools(ru.research_tools([{"key": "a", "label": "A"}])[0]) + self.assertEqual([{k: t[k] for k in ("name", "description", "parameters")} for t in airo], baseline.research_tools()) + + +if __name__ == "__main__": + unittest.main()