Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,8 @@ packages/fri-forecaster/ the FRI Forecaster harness -- pi, the research tools
redlines/ the pipeline package (stdlib only)
__main__.py CLI: build / assemble / export-fb / paper
config.py paths (env-overridable), round pins, thresholds
registry.py the single model table (ids, labels, ECI, colors) + panel()
registry.py AIRO's model table (colors, chart roles, pinned ECI) over the shared rows and
panel rule in packages/fri-forecaster (fri_forecaster.panel)
eci.py the Epoch Capabilities Index: fri_forecaster.eci (one implementation,
shared with the ForecastBench rounds) bound to data/ and the pinned vintages
questions.py schema-checked loaders for the Auto-ARC question set
Expand Down
14 changes: 8 additions & 6 deletions code/run_unified.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@
from redlines.config import UNBATCHED # noqa: E402
from redlines.questions import (horizon_label, horizon_sort_key, # noqa: E402
resolves_on, severity_label)
from fri_forecaster import baseline as _baseline # noqa: E402 (on the path through redlines.eci)

LADDER = os.path.join(ROOT, "data", "autoarc_ladder.json")
CROSS = os.path.join(ROOT, "data", "autoarc_crosscutting.json")
Expand Down Expand Up @@ -388,19 +389,20 @@ def apply_variant(prompt, variant):
# far above any call seen (64 turns); it also keeps agent-box's nudge on, so a
# reply cut at the output limit with no submission (Opus 5.5, 2026-10-19)
# does not end the call.
MINIMAL_SYSTEM = " "
MINIMAL_MAX_TURNS = 100
#
# Since 2026-11-11 this is the Baseline condition every dashboard shares
# (fri_forecaster.baseline, packages/fri-forecaster): the system prompt, the
# turn limit and the web_search description are read from there.
MINIMAL_SYSTEM = _baseline.SYSTEM
MINIMAL_MAX_TURNS = _baseline.MAX_TURNS
MINIMAL_JOINT_HEAD = ("forecasting questions, grouped by cause of catastrophe. Forecast\n"
"all of them together, in one pass -- and forecast every")
MINIMAL_JOINT_RESEARCH = re.compile(r"Research before answering, in rounds:.*?applied to the same evidence\.\n", re.S)
MINIMAL_JOINT_RATIONALE = ("a 3-6 sentence rationale covering the whole set, the key sources\n"
"you relied on (only pages you read), and any cells not yet delivered.")
DEBRIEF_SENTENCE = re.compile(r" ?After it is accepted you will be asked one more thing.*?"
r"research with those questions in mind\.", re.S)
MINIMAL_TOOLS = {
"web_search": "Search the web. Returns titled snippets with URLs and publication dates. "
"recent_days limits results to the last N days.",
}
MINIMAL_TOOLS = {"web_search": _baseline.WEB_SEARCH}


def minimal_joint_prompt(prompt, variant=DEFAULT_VARIANT):
Expand Down
14 changes: 14 additions & 0 deletions packages/fri-forecaster/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,20 @@ The result `harness.run` returns, folded from the outbox:
Only the answers asked for (`allowed_keys`) are kept; anything else the model sends is
dropped.

## The panel and the Baseline condition

What FRI's dashboards share when they ask a standing panel of models (AIRO, EEAI):

- `fri_forecaster.eci` fetches and ranks Epoch's Capabilities Index.
- `fri_forecaster.panel` picks the panel from it: the top 4 models we can run, one
seat per model family. The table of runnable models is in that file; a box may add
rows on its own in `auto_models.json` beside the snapshots (AIRO's `redlines.autoreg`).
`python -m fri_forecaster.panel --dir DIR [K]` prints it.
- `fri_forecaster.baseline` is how the panel is asked: no system prompt, web search
offered and never required, reasoning at the top level, no debrief.
`baseline.run(model, prompt, ...)` makes one such call with the caller's own answer
schema. AIRO's published `none` condition reads its constants from here.

## Requirements

- Python ≥ 3.11, stdlib only.
Expand Down
2 changes: 2 additions & 0 deletions packages/fri-forecaster/examples/round.env.example
Original file line number Diff line number Diff line change
Expand Up @@ -15,5 +15,7 @@ ROUNDS_DIR="$HOME/forecastbench/rounds" # default: <checkout>/rounds
# place when its members are not the panel's top N. On AIRO's box the panel is AIRO's,
# refreshed from Epoch's index first (code/refresh_panel.sh, which pages a change):
# panel_members() { (cd "$HOME/Projects/airo" && { code/refresh_panel.sh >&2; "$PY" -m redlines.registry 6; }); }
# Or, with no AIRO checkout, the shared panel over a directory of snapshots:
# panel_members() { "$PY" -m fri_forecaster.panel --dir "$HOME/.cache/fri-forecaster/eci" 6; }
# Without this, the round ranks Epoch's published index by name
# (`python -m fri_forecaster.eci --fetch --names`) and can only warn when the top N moved.
73 changes: 73 additions & 0 deletions packages/fri-forecaster/fri_forecaster/baseline.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
"""The Baseline condition: how a dashboard asks the panel.

One call per panel member (fri_forecaster.panel), with nothing in it but the
dashboard's own prompt:

no system prompt one space, because pi puts its own coding-assistant
prompt in place of an empty one.
research is offered, web_search and read_page (Tavily, tools.py), described
never required by what they do; no research floor, no coverage rule.
no debrief the call ends with the answer.
reasoning at the top panel.reasoning_for.
a turn limit of 100 a runaway guard far above any call seen (64 turns); it
also keeps the harness's nudge on, so a reply cut at
the output limit with no submission does not end the
call.

This is AIRO's published condition `none` (its minimal instrument,
combined-v10, 2026-10-19: the prompt-variant experiments of 2026-10-14..19
found the instrument's guidance moved no headline forecast), labelled
"Baseline" on its dashboard. AIRO's runner reads the constants below, so the
two cannot drift; a dashboard that asks something other than probabilities
calls run() with its own answer schema.
"""
import copy
import os
import tempfile

from . import panel
from .tools import READ_PAGE_TOOL, WEB_SEARCH_TOOL

SYSTEM = " "
MAX_TURNS = 100
# A whole call, research included.
TIMEOUT = 3 * 3600
WEB_SEARCH = ("Search the web. Returns titled snippets with URLs and publication dates. "
"recent_days limits results to the last N days.")
LOOP = {"min_research": 0, "floor_counts": "returned", "withhold_final": True,
"warn_missing_keys": False, "max_turns": MAX_TURNS}


def research_tools():
"""web_search and read_page as the condition offers them: {name,
description, parameters}. web_search loses `cause`, which serves only a
coverage rule."""
search = {k: copy.deepcopy(WEB_SEARCH_TOOL[k]) for k in ("name", "description", "parameters")}
search["description"] = WEB_SEARCH
search["parameters"]["properties"].pop("cause", None)
read = {k: copy.deepcopy(READ_PAGE_TOOL[k]) for k in ("name", "description", "parameters")}
return [search, read]


def run(model, prompt, *, workdir, cell_schema, key_fields, allowed_keys, submit_cells, submit_forecast,
env=None, timeout=TIMEOUT, cwd=None, retries=0, harness_run=None):
"""One Baseline call. `model` is a panel row's `model`; `submit_cells` and
`submit_forecast` are the dashboard's two answer tools as {description,
parameters} (the pieces along the way, and the final answer); the cells
they carry are `cell_schema`, keyed by `key_fields`, and only
`allowed_keys` are kept. -> harness.run's result."""
if harness_run is None:
from . import harness
harness_run = harness.run
research = research_tools()
specs = {**{t["name"]: t for t in research}, "submit_cells": submit_cells, "submit_forecast": submit_forecast}
# pi writes its working directory into its system prompt: run it from an
# empty directory whose name says nothing about the project.
cwd = cwd or os.path.join(tempfile.gettempdir(), "w")
os.makedirs(cwd, exist_ok=True)
return harness_run(model=panel.pi_model(model), system=SYSTEM, prompt=prompt, workdir=workdir, cwd=cwd,
cell_schema=cell_schema, key_fields=list(key_fields), allowed_keys=list(allowed_keys),
tool_descriptions={n: s["description"] for n, s in specs.items()},
tool_parameters={n: s["parameters"] for n, s in specs.items()},
research_tools=tuple(t["name"] for t in research), cells_tool=True,
thinking=panel.reasoning_for(model), timeout=timeout, env=env, retries=retries, **LOOP)
Loading
Loading