loopmath 0.2.2__tar.gz → 0.2.3__tar.gz

{loopmath-0.2.2 → loopmath-0.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: loopmath
-Version: 0.2.2
+Version: 0.2.3
 Summary: Plan agent workflows, record how they went, and learn from your own Claude Code and Codex logs
 Author: QKV Labs
 License-Expression: MIT
@@ -103,6 +103,8 @@ Reading the history uses up to 8 worker processes (one per CPU, at most 8). Set
 
 loopmath ships with a prior built from our own sweeps and experiments, so `recommend` gives an answer before you have any history. Your own runs then move it.
 
+The prior ships inside the package: our runs, in the shared form (no titles, paths, commands or free text), and the published benchmark results. They are the same files we fit with, and each release updates them. `onboard` and the results page say in one line which prior they start from, and `loopmath prior show` lists it source by source.
+
 ## The loop
 
 The plan and record skills take an agent through the loop in three commands:

{loopmath-0.2.2 → loopmath-0.2.3}/README.md

@@ -65,6 +65,8 @@ Reading the history uses up to 8 worker processes (one per CPU, at most 8). Set
 
 loopmath ships with a prior built from our own sweeps and experiments, so `recommend` gives an answer before you have any history. Your own runs then move it.
 
+The prior ships inside the package: our runs, in the shared form (no titles, paths, commands or free text), and the published benchmark results. They are the same files we fit with, and each release updates them. `onboard` and the results page say in one line which prior they start from, and `loopmath prior show` lists it source by source.
+
 ## The loop
 
 The plan and record skills take an agent through the loop in three commands:

{loopmath-0.2.2 → loopmath-0.2.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "loopmath"
-version = "0.2.2"
+version = "0.2.3"
 description = "Plan agent workflows, record how they went, and learn from your own Claude Code and Codex logs"
 readme = "README.md"
 license = "MIT"

{loopmath-0.2.2 → loopmath-0.2.3}/src/loopmath/__init__.py

@@ -5,4 +5,4 @@ The command line (`loopmath`, or its alias `loop`) is the interface; see the
 README and `loopmath --help`.
 """
 
-__version__ = "0.2.2"
+__version__ = "0.2.3"

{loopmath-0.2.2 → loopmath-0.2.3}/src/loopmath/belief/design.py

@@ -464,10 +464,12 @@ def source_label(doc: dict) -> str:
     ext = run.get("ext") or {}
     share = ext.get("dev.loopmath.share")
     if isinstance(share, dict):
-        # the importer writes `source` and `task.org` as `shared:<org_hash>` already: never prefix twice
+        # the importer writes `source` and `task.org` as `shared:<org_hash>` already: never prefix twice. With no
+        # org at all the source is `shared`, the name `fit --without shared` takes (0.2.3, 22X review N1)
         org = (share.get("source") or share.get("org") or share.get("org_hash") or (run.get("task") or {}).get("org")
-               or "shared")
-        return "shared:" + str(org).removeprefix("shared:")
+               or "")
+        name = str(org).removeprefix("shared:")
+        return f"shared:{name}" if name and name != "shared" else "shared"
     src = (run.get("task") or {}).get("source")
     kind = src.get("kind") if isinstance(src, dict) else src
     kind = str(kind or "").strip().lower()

{loopmath-0.2.2 → loopmath-0.2.3}/src/loopmath/belief/fit.py

@@ -526,7 +526,8 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
     with fit_lock(home, wait_s):
         remove_partials(home)
         config = _read_config(home)
-        weight = float(config.get("benchmark_prior_weight", prior_data.BENCHMARK_PRIOR_WEIGHT))
+        raw_weight = config.get("benchmark_prior_weight")  # unset: each benchmark's own weight (spec 04 section 5)
+        weight = None if raw_weight is None else float(raw_weight)
         try:
             features, features_error = FeatureSet.from_config(config.get("features")), None
         except FeatureConfigError as exc:  # a hand-edited config.toml: fit with the built-ins, say why
@@ -550,9 +551,10 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
             raise UnknownSource(unknown_source_message(unknown, known))
         if not any(hr.rows for hr in heads_rows.values()):
             raise NothingToFit(_nothing_to_fit(no_prior, without, dropped))
-        specs = []
+        specs, bench_weights = [], {}
         if not no_prior and "benchmark" not in without:
             specs = prior_data.benchmark_factors(benchmarks, weight=weight)
+            bench_weights = prior_data.benchmark_weights(benchmarks, weight=weight)
         seed_key = input_key(heads_rows, specs, {"no_prior": no_prior, "without": sorted(without), "eb": eb,
                                                   "benchmark_prior_weight": weight})
         forest = Forest()
@@ -604,7 +606,7 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
             "seed_key": seed_key, "timebox_effort": design_rows.TIMEBOX_EFFORT_COST,
             "timebox_terms": list(design_rows.TIMEBOX_LEVELS),
             "options": {"no_prior": no_prior, "without": list(without), "full": full, "eb": eb,
-                        "benchmark_prior_weight": weight},
+                        "benchmark_prior_weight": weight, "benchmark_weights": bench_weights},
             "runs_by_source": runs_by_source,
             "n_runs": {"prior": sum(v for k, v in runs_by_source.items() if k != "user"),
                        "user": runs_by_source.get("user", 0)},

{loopmath-0.2.2 → loopmath-0.2.3}/src/loopmath/belief/priors.py

@@ -10,8 +10,11 @@
   version nodes: success head, the logit gap to the benchmark's reference model with variance
   `1 / (w p (1 - p))`; cost head, the log cost ratio with variance `1 / w`; tokens head
   (`kind = "tokens"`, published total tokens for the evaluation), the log token ratio
-  with variance `1 / w`. `w` is config `benchmark_prior_weight` (default 5): one benchmark
-  result counts like about 5 runs.
+  with variance `1 / w`. `w` is one model's share of the benchmark's weight `w_b` (0.2.3,
+  lane 23K): the `weight` of its `[[benchmark]]` table (default 5, about 5 runs), shared by
+  the model's k results on that benchmark, so each gets `w_b / k` and a model with five
+  efforts pulls as hard in total as a model with one. Config `benchmark_prior_weight`, when
+  set, is one `w_b` for every benchmark; 0 turns the factors off.
 """
 
 from __future__ import annotations
@@ -91,8 +94,30 @@ def _model_terms(model: str, effort: str | None, sign: float) -> list[tuple[str,
     return terms
 
 
-def benchmark_factors(path: Path | None = None, *, weight: float = BENCHMARK_PRIOR_WEIGHT) -> list[FactorSpec]:
-    """Prior factors from `benchmarks.toml`. A missing file gives no factors."""
+def benchmark_weights(path: Path | None = None, *, weight: float | None = None) -> dict[str, float]:
+    """Each benchmark's weight `w_b`: `weight` for every benchmark when it is given (config
+    `benchmark_prior_weight`), else the benchmark's own `weight` field, else 5. Empty for a missing file."""
+    path = path or benchmark_path()
+    if not path.is_file():
+        return {}
+    data = tomllib.loads(path.read_text(encoding="utf-8"))
+    return {str(b["id"]): _weight_of(b, weight) for b in data.get("benchmark") or [] if b.get("id")}
+
+
+def _weight_of(bench: dict, weight: float | None) -> float:
+    if weight is not None:
+        return float(weight)
+    own = bench.get("weight")
+    return float(own) if isinstance(own, (int, float)) and not isinstance(own, bool) else BENCHMARK_PRIOR_WEIGHT
+
+
+def benchmark_factors(path: Path | None = None, *, weight: float | None = None) -> list[FactorSpec]:
+    """Prior factors from `benchmarks.toml`. A missing file gives no factors.
+
+    `weight` (config `benchmark_prior_weight`) overrides every benchmark's own weight; None reads
+    each benchmark's `weight`. A model's k results on one benchmark share its weight: each factor
+    has weight `w_b / k` (B1). A benchmark with weight 0 or less gives no factors.
+    """
     path = path or benchmark_path()
     if not path.is_file():
         return []
@@ -103,11 +128,15 @@ def benchmark_factors(path: Path | None = None, *, weight: float = BENCHMARK_PRI
     for bid, bench in benches.items():
         kind = str(bench.get("kind") or "success")
         ref = canonical_model_id(str(bench.get("reference") or ""))
+        w_b = _weight_of(bench, weight)
+        if w_b <= 0:
+            continue
         rows = [r for r in results if r["benchmark"] == bid]
         ref_rows = [r for r in rows if canonical_model_id(str(r["model"])) == ref]
         if not ref_rows:
             continue
         ref_value = float(ref_rows[0]["value"])
+        by_model: dict[str, list[tuple[str, list, float, dict]]] = {}  # model -> (head, terms, mean, row) per result
         for r in rows:
             model = canonical_model_id(str(r["model"]))
             if model == ref:
@@ -117,14 +146,24 @@ def benchmark_factors(path: Path | None = None, *, weight: float = BENCHMARK_PRI
             terms = _cancel(terms)
             if not terms:
                 continue
-            note = f"{bid}: {model} {value} against {ref} {ref_value} ({r.get('url', '')}, {r.get('date', '')})"
             if kind == "success":
                 p = min(max(value, 0.01), 0.99)
                 p_ref = min(max(ref_value, 0.01), 0.99)
                 gap = math.log(p / (1 - p)) - math.log(p_ref / (1 - p_ref))
-                out.append(FactorSpec("success", terms, gap, 1.0 / (weight * p * (1 - p)), note))
+                by_model.setdefault(model, []).append(("success", terms, gap, r))
             elif kind in ("cost", "tokens") and value > 0 and ref_value > 0:  # Tokens feed the tokens head only
-                out.append(FactorSpec(kind, terms, math.log(value / ref_value), 1.0 / weight, note))
+                by_model.setdefault(model, []).append((kind, terms, math.log(value / ref_value), r))
+        for model, items in by_model.items():
+            w = w_b / len(items)  # one budget per model and benchmark (B1)
+            share = f"weight {w_b:g} over {len(items)}" if len(items) > 1 else f"weight {w_b:g}"
+            for head, terms, mean, r in items:
+                note = (f"{bid}: {model} {float(r['value'])} against {ref} {ref_value} ({share}; "
+                        f"{r.get('url', '')}, {r.get('date', '')})")
+                if head == "success":
+                    p = min(max(float(r["value"]), 0.01), 0.99)
+                    out.append(FactorSpec("success", terms, mean, 1.0 / (w * p * (1 - p)), note))
+                else:
+                    out.append(FactorSpec(head, terms, mean, 1.0 / w, note))
     return out
 
 

{loopmath-0.2.2 → loopmath-0.2.3}/src/loopmath/belief/state.py

@@ -792,10 +792,13 @@ class FitState:
                                    gate_pass=res[hg] if hg is not None else None, rounds=res[hr],
                                    cost_per_round=Money(usd=res[hpr[0]], tokens=res[hpr[1]]) if hpr else None)
             for piece, (hu, ht, hg, hr, hpr) in handles.items()}
+        # The typical run (0.2.3): the median of the same simulated runs, one per draw, the 80% range is read from
+        run_usd = dataclasses.replace(res[h_run[1]], median=float(np.median(rd.sim_usd)))
+        run_tokens = dataclasses.replace(res[h_run[2]], median=float(np.median(rd.sim_tokens)))
         pred = Prediction(
             config=cfg.id,
             p_success=res[h_run[0]],
-            cost=Money(usd=res[h_run[1]], tokens=res[h_run[2]]),
+            cost=Money(usd=run_usd, tokens=run_tokens),
             ell=Money(usd=res[h_run[3]], tokens=res[h_run[4]]),
             rounds=res[h_run[5]],
             per_piece=per_piece,

{loopmath-0.2.2 → loopmath-0.2.3}/src/loopmath/builder/context.py

@@ -137,7 +137,7 @@ def build_session(args: argparse.Namespace) -> tuple[Session | None, int]:
         rec = engine.recommend(belief, asked, rule, usual=usual, usual_from=usual_from, configs=configs,
                                settings=settings, diff=C.diff_fn(), keep=[*(u.id for u in user),
                                                                         *(r.id for r in recorded)],
-                               **offered_kw(offered))
+                               own_runs=storeread.own_runs(home), **offered_kw(offered))
         rec.task = task
         start = None
         if getattr(args, "start", None):
@@ -305,6 +305,17 @@ def candidate_entry(rec: Recommendation, c: Any) -> dict[str, Any]:
             "origin": c.origin, **rec.search_fields(c.config.id)}
 
 
+def rescue_entry(session: Session, cfg: Configuration) -> dict[str, Any]:
+    """The rescue workflow as a candidate when the search did not offer it (22W N1, 0.2.3): predicted as a
+    candidate is, under the recommendation's task, rule and rescue, with origin `rescue`."""
+    rec = session.rec
+    bands: dict[str, Any] = {}
+    preds, medians = engine.predict_with_medians(session.belief, session.task, [cfg], session.rule, rec.rescue, bands)
+    view = with_numbers(rec, medians=medians, bands=bands)
+    return {"config": cfg.to_dict(), "label": rec.label(cfg), "numbers": numbers_of(view, preds[0]),
+            "origin": "rescue", **rec.search_fields(cfg.id)}
+
+
 def context_payload(session: Session) -> dict[str, Any]:
     """`GET /api/context` (spec 02, `builder`), built once per session."""
     if session.context is not None:
@@ -329,6 +340,10 @@ def context_payload(session: Session) -> dict[str, Any]:
         wanted.add(rec.rescue_config.id)
     top += [c for c in offer[TOP_CANDIDATES:]
             if c.config.id not in ids and (c.origin in KEEP_ORIGINS or c.config.id in wanted)]
+    candidates = [candidate_entry(rec, c) for c in top]
+    fix = rec.rescue_config
+    if fix is not None and fix.id not in retired and rec.by_id(fix.id) is None:
+        candidates.append(rescue_entry(session, fix))
     session.context = {
         "schema": SCHEMA,
         "task": C.task_block(rec.task, session.belief),
@@ -339,9 +354,10 @@ def context_payload(session: Session) -> dict[str, Any]:
         "rescue": core["rescue"],
         "reference": reference,
         "choices": choices,
-        "candidates": [candidate_entry(rec, c) for c in top],
+        "candidates": candidates,
         "catalog": catalog(session),
         "start": session.start.to_dict() if session.start is not None else None,
+        "own_runs": rec.own_runs,  # 0.2.3: the user's recorded runs; 0 leads each run cost with the typical run
     }
     return session.context
 

{loopmath-0.2.2 → loopmath-0.2.3}/src/loopmath/cli_registry.py

@@ -274,7 +274,11 @@ def _add_recording(sub) -> None:
     q.add_argument("key", nargs="?", default=None, metavar="KEY")
     _common(q)
     q.set_defaults(func=_lazy("loopmath.store.commands:config_get"))
-    q = cs.add_parser("set", help="set one key")
+    q = cs.add_parser("set", help="set one key",
+                      description="Sets one key in config.toml. benchmark_prior_weight is unset by default: each "
+                                  "benchmark in the shipped benchmarks.toml has its own weight, in runs, shared by "
+                                  "a model's results on it; a number here is one weight for every benchmark, and 0 "
+                                  "turns benchmark priors off.")
     q.add_argument("key", metavar="KEY")
     q.add_argument("value", metavar="VALUE")
     _common(q)
@@ -335,7 +339,7 @@ def _add_learning(sub) -> None:
     ps = p.add_subparsers(dest="prior_command", required=True)
     q = ps.add_parser("build", help="rebuild the packaged prior bundle from the sweep, E0 and RQ1 inputs")
     q.add_argument("--out", default=None, metavar="DIR", help="where to write the bundle (default: the packaged bundle folder, which it replaces)")
-    q.add_argument("--sweep-dir", default=None, metavar="PATH", help="sweep results (default: LOOPMATH_SWEEP_DIR)")
+    q.add_argument("--sweep-dir", action="append", default=None, metavar="PATH", help="sweep results, one folder per batch; repeatable (default: LOOPMATH_SWEEP_DIR, one folder)")
     q.add_argument("--e0-corpus", default=None, metavar="PATH", help="E0 corpus (default: LOOPMATH_E0_CORPUS)")
     q.add_argument("--rq1-dir", default=None, metavar="PATH", help="RQ1 OCP documents (default: LOOPMATH_PRIOR_RQ1)")
     q.add_argument("--lanes-dir", default=None, metavar="PATH", help="build lane rows (default: LOOPMATH_PRIOR_LANES)")

{loopmath-0.2.2 → loopmath-0.2.3}/src/loopmath/onboard/commands.py

@@ -31,6 +31,7 @@ from typing import Any, Callable
 from .. import output
 from ..output import EXIT_NOT_FOUND, EXIT_OK, EXIT_USER, emit_json, fail
 from .. import taskmodel
+from ..priors.show import starting_prior
 from ..store.ids import since_days as since_window
 from ..taskmodel import LABEL_VERSION
 from . import history as H
@@ -124,6 +125,9 @@ def onboard(args: argparse.Namespace) -> int:
     # The one line printed even when stderr is captured: the wait that follows can be long.
     print(f"onboard: reading Claude Code and Codex history, {_window_words(since)}; "
           "a large history can take several minutes", file=sys.stderr, flush=True)
+    # Then, also when captured, what the answers start from, once (the skill shows this line to the user).
+    prior = starting_prior()
+    print(prior["line"], file=sys.stderr, flush=True)
     hist = deps.load_history(since_days, logs=deps.logs, progress=_progress, stage=_stage)
     now = deps.now()
     groups, before_window = H.in_window(H.group_sessions(hist.graph, hist.by_id), since_days, now=now)
@@ -160,6 +164,7 @@ def onboard(args: argparse.Namespace) -> int:
 
     payload: dict[str, Any] = {
         "dry_run": dry_run,
+        "prior": prior,
         "since": since,
         "since_days": round(since_days, 3),
         "window": _window(groups),

{loopmath-0.2.2 → loopmath-0.2.3}/src/loopmath/priors/__init__.py

@@ -8,7 +8,9 @@ The read API for the belief model (lane 5):
 - `bundle_entries()`: the same documents as `(source, document)` pairs, so the fit
   knows where each run came from whatever its label says (spec 04 section 1).
 - `shipped_overlap(run_ids)` and `overlap_note(counts)`: how many of the user's runs
-  are also in a shipped source; `run import`, `fit` and `status` say it once.
+  are also in a shipped source; `run import`, `fit` and `status` say it once. A
+  stored run and its shipped copy are matched on `run_id_of`, the run id less a
+  start stamp (`stable_run_id`), since the shipped rq1 ids lose theirs (23B).
 - `manifest()`, `sources()`, `bundle_dir()`: what the bundle holds and where.
 
 Reading never touches the network or the store; the bundle is package data.
@@ -18,6 +20,7 @@ from __future__ import annotations
 
 import gzip
 import json
+import re
 from collections import Counter
 from pathlib import Path
 from typing import Iterable, Iterator
@@ -71,15 +74,33 @@ def bundle_docs(without: Iterable[str] = (), sources: Iterable[str] | None = Non
         yield doc
 
 
+_STAMP = re.compile(r"-\d{8}-\d{6}(?=-|$)")  # lane 10's run start, YYYYMMDD-HHMMSS local time
+
+
+def stable_run_id(run_id: str) -> str:
+    """A run id less its `-YYYYMMDD-HHMMSS` start stamp; an id without one is unchanged.
+
+    The shipped rq1 runs carry lane 10's ids without the stamp (a real date and time
+    does not ship, 23B) while an RQ1 store keeps it, so a stored run and its shipped
+    copy compare on this key. It is the id itself, not a hash: the stamp space is
+    small enough that a hash of the stamped id would give the time away.
+    """
+    return _STAMP.sub("", run_id)
+
+
 def run_id_of(doc: dict) -> str:
-    """The run id the fit keys on (`belief.design.parse_run`), read without parsing the run."""
+    """The key a stored run and its shipped copy are matched on, read without parsing the run.
+
+    The run id the fit keys on (`belief.design.parse_run`), less a start stamp
+    (`stable_run_id`); the fit and `shipped_overlap` both compare on it.
+    """
     run = doc.get("run") or {}
     task = run.get("task") or {}
-    return str(run.get("id") or task.get("id") or (run.get("labels") or {}).get("task") or "unknown")
+    return stable_run_id(str(run.get("id") or task.get("id") or (run.get("labels") or {}).get("task") or "unknown"))
 
 
 def shipped_ids(directory: Path | None = None) -> dict[str, str]:
-    """Run id to its shipped source, for every bundled run."""
+    """Run id (`run_id_of`) to its shipped source, for every bundled run."""
     out: dict[str, str] = {}
     for name, doc in bundle_entries(directory=directory):
         out.setdefault(run_id_of(doc), name)
@@ -89,10 +110,12 @@ def shipped_ids(directory: Path | None = None) -> dict[str, str]:
 def shipped_overlap(run_ids: Iterable[str], directory: Path | None = None) -> dict[str, int]:
     """How many of `run_ids` (the user's runs) are also in each shipped source.
 
-    A fit uses the store's copy of such a run and leaves the shipped copy out.
+    A fit uses the store's copy of such a run and leaves the shipped copy out. The
+    ids compare as `run_id_of` does, without a start stamp.
     """
     ids = shipped_ids(directory)
-    return dict(Counter(ids[r] for r in set(run_ids) if r in ids))
+    keys = [stable_run_id(r) for r in set(run_ids)]
+    return dict(Counter(ids[k] for k in keys if k in ids))
 
 
 def overlap_note(counts: dict[str, int] | None) -> str | None:
@@ -106,8 +129,8 @@ def overlap_note(counts: dict[str, int] | None) -> str | None:
     else:
         where = "the shipped prior (" + ", ".join(f"{k} {v}" for k, v in sorted(overlap.items(), key=lambda kv: -kv[1])) + ")"
     if total == 1:
-        return f"1 of your runs is also in {where} (same run id): fits use your copy"
-    return f"{total} of your runs are also in {where} (same run ids): fits use your copies"
+        return f"1 of your runs is also in {where} (same run): fits use your copy"
+    return f"{total} of your runs are also in {where} (same runs): fits use your copies"
 
 
 iter_runs = bundle_docs

{loopmath-0.2.2 → loopmath-0.2.3}/src/loopmath/priors/benchmarks.py

@@ -1,11 +1,13 @@
 """Published benchmark results (lane 11): `priors/benchmarks.toml`.
 
 The file has `[[benchmark]]` tables (`id`, `title`, `metric`, `kind` success,
-cost or tokens, `reference` model, `reference_effort`, `harness`, `url`, `note`) and
+cost or tokens, `reference` model, `reference_effort`, `weight`, `harness`, `url`, `note`) and
 `[[result]]` tables (`benchmark`, `model`, `effort`, `value`, `harness`, `date`,
 `url`, `note`). Lane 5's `belief.priors.benchmark_factors` turns them into prior
 factors on version nodes (spec 04 section 5); it reads the first result of the
 reference model as the reference, so that row is the one at `reference_effort`.
+`weight` is the benchmark's weight in runs (5 when absent), shared by each model's
+results on it (0.2.3, lane 23K; spec 04 section 5).
 
 This module loads the file and checks it: every value is a published number
 with its own source line, success values are fractions, the reference
@@ -49,6 +51,9 @@ def check_benchmarks(data: dict) -> list[str]:
         problems += [f"{where}: no {k}" for k in _BENCH_KEYS if not b.get(k)]
         if b.get("kind") and b["kind"] not in KINDS:
             problems.append(f"{where}: kind {b['kind']!r} is not one of {', '.join(KINDS)}")
+        if "weight" in b and (isinstance(b["weight"], bool) or not isinstance(b["weight"], (int, float))
+                              or b["weight"] < 0):
+            problems.append(f"{where}: weight is not a number of 0 or more")
         if b.get("id") in benches:
             problems.append(f"{where}: duplicate id")
         benches[str(b.get("id"))] = b

{loopmath-0.2.2 → loopmath-0.2.3}/src/loopmath/priors/benchmarks.toml

@@ -3,8 +3,9 @@
 # design/0.1/data/benchmarks/artificialanalysis-2026-09-23.json (Terminal-Bench 4.0) and
 # artificialanalysis-2026-09-25.json (Terminal-Bench 2.1). Values are copied as published;
 # a missing cell stays missing. Lane 5's belief.priors.benchmark_factors reads this file: success
-# results become logit gaps to the reference model with variance 1 / (w p (1 - p)), w =
-# benchmark_prior_weight (default 5). The first result for the reference model is the reference.
+# results become logit gaps to the reference model with variance 1 / (w p (1 - p)), w = the
+# benchmark's weight over the model's k results on it (config benchmark_prior_weight, when set,
+# replaces every weight). The first result for the reference model is the reference.
 
 [[benchmark]]
 id = "aa-terminal-bench-4.0"
@@ -13,6 +14,7 @@ metric = "pass@1 averaged over three repeats of 66 tasks, as a fraction"
 kind = "success"
 reference = "claude-opus-5"
 reference_effort = "max"
+weight = 5.0
 harness = "mini-swe-agent (Artificial Analysis)"
 url = "https://artificialanalysis.ai/evaluations/terminalbench-4-0"
 note = "One harness for every model, so the gaps compare models rather than scaffolds. Variants without reasoning are left out (loopmath does not run them). Haiku 4.5 is the reasoning variant, which names no effort. Values are k of 198 task runs."
@@ -24,6 +26,7 @@ metric = "total tokens for the whole evaluation (66 tasks x 3 repeats): input (w
 kind = "tokens"
 reference = "claude-opus-5"
 reference_effort = "max"
+weight = 5.0
 harness = "mini-swe-agent (Artificial Analysis)"
 url = "https://artificialanalysis.ai/evaluations/terminalbench-4-0"
 note = "D54: a log-ratio factor on the tokens head only, never converted to dollars. The same three streams for every model, so the ratio is like for like."
@@ -35,6 +38,7 @@ metric = "pass@1 averaged over three repeats of 89 tasks, as a fraction"
 kind = "success"
 reference = "claude-opus-5"
 reference_effort = "max"
+weight = 5.0
 harness = "Terminus 2 (Artificial Analysis)"
 url = "https://artificialanalysis.ai/evaluations/terminalbench-2-1"
 note = "One harness for every model (Terminus 2, as the page says), so the gaps compare models rather than scaffolds. Variants without reasoning are left out. Haiku 4.5 is the reasoning variant, which names no effort. Values are k of 267 task runs. Artificial Analysis publishes no Terminal-Bench 2.1 result for claude-opus-5-5, gpt-6-sol or gpt-6-luna, and only the max effort for claude-sonnet-5 and claude-fable-5; those cells stay missing. The official Terminal-Bench 2.1 board mixes six harnesses and is left out (lane 22K)."
@@ -46,6 +50,7 @@ metric = "total tokens for the whole evaluation (89 tasks x 3 repeats): input (w
 kind = "tokens"
 reference = "claude-opus-5"
 reference_effort = "max"
+weight = 5.0
 harness = "Terminus 2 (Artificial Analysis)"
 url = "https://artificialanalysis.ai/evaluations/terminalbench-2-1"
 note = "D54: a log-ratio factor on the tokens head only, never converted to dollars. The same three streams for every model, so the ratio is like for like. Only the variants the evaluation pages show carry token counts."