loopmath 0.2.5__py3-none-any.whl → 0.2.7__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loopmath/__init__.py +1 -1
- loopmath/adapters/atrium.py +3 -3
- loopmath/adapters/bb.py +1 -1
- loopmath/adapters/omp_types.py +1 -1
- loopmath/adapters/orca.py +1 -1
- loopmath/adapters/otel_genai_types.py +1 -1
- loopmath/adapters/paseo.py +2 -2
- loopmath/belief/commands.py +26 -2
- loopmath/belief/design.py +18 -9
- loopmath/belief/fit.py +67 -11
- loopmath/belief/forest.py +2 -2
- loopmath/belief/outcome.py +21 -8
- loopmath/belief/pricing.py +1 -1
- loopmath/belief/priors.py +18 -18
- loopmath/belief/rules.py +139 -0
- loopmath/builder/__init__.py +1 -1
- loopmath/builder/context.py +4 -4
- loopmath/builder/predict.py +5 -5
- loopmath/cli_registry.py +24 -6
- loopmath/cli_support.py +28 -37
- loopmath/e0/arms.py +19 -22
- loopmath/e0/figures.py +1 -1
- loopmath/fit.py +36 -44
- loopmath/fit_assembly.py +7 -7
- loopmath/fit_predict.py +1 -1
- loopmath/fit_pricing.py +9 -9
- loopmath/fit_transfer.py +4 -4
- loopmath/grade.py +5 -5
- loopmath/grade_support.py +1 -1
- loopmath/graph/dataset_build.py +1 -1
- loopmath/graph/dataset_core.py +1 -1
- loopmath/graph/extract.py +14 -14
- loopmath/graph/html_common.py +33 -10
- loopmath/graph/html_costcurve.py +7 -3
- loopmath/graph/html_force.py +7 -3
- loopmath/graph/html_swimlanes.py +2 -2
- loopmath/graph/labels.py +1 -1
- loopmath/graph/launch.py +1 -1
- loopmath/graph/launch_origin.py +2 -2
- loopmath/graph/ocp_support.py +4 -4
- loopmath/graph/scan.py +5 -5
- loopmath/graph/schema.py +1 -1
- loopmath/ingest/__init__.py +13 -15
- loopmath/ingest/base.py +1 -1
- loopmath/ingest/claude_code.py +4 -4
- loopmath/ingest/codex.py +13 -13
- loopmath/ingest/codex_support.py +2 -2
- loopmath/logmatch/artifacts.py +10 -5
- loopmath/logmatch/clip.py +35 -21
- loopmath/logmatch/codex_window.py +164 -0
- loopmath/logmatch/match.py +3 -1
- loopmath/logmatch/settle.py +18 -11
- loopmath/logmatch/tariff.py +1 -1
- loopmath/ocp/migrate.py +2 -2
- loopmath/onboard/commands.py +4 -4
- loopmath/onboard/history.py +3 -3
- loopmath/onboard/label.py +1 -1
- loopmath/onboard/usual.py +1 -1
- loopmath/output.py +1 -1
- loopmath/price.py +1 -1
- loopmath/prices.toml +1 -1
- loopmath/priors/__init__.py +5 -5
- loopmath/priors/benchmarks.py +4 -4
- loopmath/priors/benchmarks.toml +114 -115
- loopmath/priors/build.py +1 -1
- loopmath/priors/commands.py +2 -2
- loopmath/priors/e0.py +4 -4
- loopmath/priors/history.py +1 -1
- loopmath/priors/lanes.py +3 -3
- loopmath/priors/ocpdoc.py +10 -10
- loopmath/priors/registry.py +3 -3
- loopmath/priors/rq1.py +22 -22
- loopmath/priors/show.py +3 -3
- loopmath/priors/sweep.py +1 -1
- loopmath/priors/validate.py +3 -3
- loopmath/receipts.py +1 -1
- loopmath/recommend/commands.py +21 -20
- loopmath/recommend/curve.py +1 -1
- loopmath/recommend/engine.py +12 -12
- loopmath/recommend/message.py +7 -7
- loopmath/recommend/receipts.py +1 -1
- loopmath/recommend/storeread.py +6 -6
- loopmath/report/terminal.py +10 -12
- loopmath/report/terminal_read.py +5 -6
- loopmath/share/commands.py +15 -3
- loopmath/share/export.py +10 -10
- loopmath/share/import_.py +17 -1
- loopmath/skill/doctor.py +7 -2
- loopmath/skill/install.py +9 -2
- loopmath/skill/skills/loopmath-record-run/SKILL.md +1 -1
- loopmath/skill/skills/reference.md +1 -1
- loopmath/store/commands.py +75 -5
- loopmath/store/config.py +129 -6
- loopmath/store/finish.py +243 -46
- loopmath/store/fitjob.py +6 -5
- loopmath/store/home.py +17 -5
- loopmath/store/ids.py +1 -1
- loopmath/store/record.py +55 -14
- loopmath/store/report.py +12 -3
- loopmath/store/rule_change.py +137 -0
- loopmath/store/runs.py +2 -2
- loopmath/store/status.py +1 -1
- loopmath/surface.py +6 -7
- loopmath/surface_bootstrap.py +4 -4
- loopmath/surface_estimation.py +15 -16
- loopmath/surface_frame.py +1 -1
- loopmath/surface_presentation.py +7 -8
- loopmath/surface_support.py +4 -5
- loopmath/taskmodel.py +1 -1
- loopmath/types.py +2 -2
- loopmath/views/__init__.py +1 -1
- loopmath/views/assets/README.md +1 -1
- loopmath/views/assets/base.css +2 -2
- loopmath/views/assets/builder.css +1 -3
- loopmath/views/assets/builder.js +55 -18
- loopmath/views/assets/common.js +40 -8
- loopmath/views/assets/estimates.js +7 -7
- loopmath/views/assets/graph.css +1 -1
- loopmath/views/assets/graph.js +2 -2
- loopmath/views/assets/plans.css +2 -2
- loopmath/views/assets/plans.js +23 -23
- loopmath/views/assets/results.css +1 -1
- loopmath/views/assets/results.js +12 -12
- loopmath/views/assets/runs.js +25 -8
- loopmath/views/assets/viz.css +2 -2
- loopmath/views/assets/viz.js +7 -7
- loopmath/views/common.py +9 -16
- loopmath/views/plans.py +4 -4
- loopmath/views/posterior.py +49 -26
- loopmath/views/runs.py +74 -22
- loopmath/workflows/__init__.py +1 -1
- loopmath/workflows/candidates.py +3 -3
- loopmath/workflows/commands.py +2 -2
- loopmath/workflows/graphview.py +2 -2
- loopmath/workflows/ids.py +3 -3
- loopmath/workflows/models.py +3 -3
- loopmath/workflows/ocp.py +4 -4
- {loopmath-0.2.5.dist-info → loopmath-0.2.7.dist-info}/METADATA +3 -3
- {loopmath-0.2.5.dist-info → loopmath-0.2.7.dist-info}/RECORD +146 -143
- {loopmath-0.2.5.data → loopmath-0.2.7.data}/data/spec/ocp-v0.2.schema.json +0 -0
- {loopmath-0.2.5.data → loopmath-0.2.7.data}/data/spec/ocp-v0.schema.json +0 -0
- {loopmath-0.2.5.data → loopmath-0.2.7.data}/data/spec/ocp_conformance.py +0 -0
- {loopmath-0.2.5.dist-info → loopmath-0.2.7.dist-info}/WHEEL +0 -0
- {loopmath-0.2.5.dist-info → loopmath-0.2.7.dist-info}/entry_points.txt +0 -0
- {loopmath-0.2.5.dist-info → loopmath-0.2.7.dist-info}/licenses/LICENSE +0 -0
- {loopmath-0.2.5.dist-info → loopmath-0.2.7.dist-info}/top_level.txt +0 -0
loopmath/__init__.py
CHANGED
loopmath/adapters/atrium.py
CHANGED
|
@@ -5,12 +5,12 @@ typed subagent-stop events whose subagent id may be empty. The stored facts are
|
|
|
5
5
|
``verified``, but runtime pane ids are not stable OCP task-node ids and the
|
|
6
6
|
closed OCP edge vocabulary has no neutral message or runtime-correlation edge.
|
|
7
7
|
This adapter therefore preserves them as typed metadata-only events under the
|
|
8
|
-
documented ``dev.
|
|
8
|
+
documented ``dev.loopmath.adapter.atrium`` extension and emits no core graph edge. It
|
|
9
9
|
does not infer a spawn relation from a subagent-stop record or infer acceptance
|
|
10
10
|
from segment closure, session end, or subagent stop.
|
|
11
11
|
|
|
12
12
|
The source has no token counts, cost, model, effort, stable task node, or
|
|
13
|
-
acceptance result.
|
|
13
|
+
acceptance result. Locally inspected data can be weeks stale, and
|
|
14
14
|
its pane-to-session coverage is sparse. SQL projection excludes prompt,
|
|
15
15
|
message, title, body, transcript, actor, and task text before rows reach
|
|
16
16
|
Python; none is retained or emitted.
|
|
@@ -35,7 +35,7 @@ from .registry import register
|
|
|
35
35
|
|
|
36
36
|
_DEFAULT_ROOT = Path.home() / ".atrium"
|
|
37
37
|
_STORE_NAMES = ("tasks.db", "timeline.db")
|
|
38
|
-
_NAMESPACE = "dev.
|
|
38
|
+
_NAMESPACE = "dev.loopmath.adapter.atrium"
|
|
39
39
|
_TIMELINE_KINDS = ("agent-message", "session-end", "subagent-stop")
|
|
40
40
|
_DIAGNOSTIC_REASONS = (
|
|
41
41
|
"timeline_kind_unsupported",
|
loopmath/adapters/bb.py
CHANGED
loopmath/adapters/omp_types.py
CHANGED
|
@@ -10,7 +10,7 @@ from typing import Any, Mapping
|
|
|
10
10
|
from .base import Usage
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
_EXT_NAMESPACE = "dev.
|
|
13
|
+
_EXT_NAMESPACE = "dev.loopmath.adapter.omp"
|
|
14
14
|
_KNOWN_MESSAGE_ROLES = {"assistant", "system", "toolResult", "user"}
|
|
15
15
|
_TOKEN_FIELDS = {
|
|
16
16
|
"input": "input_tokens",
|
loopmath/adapters/orca.py
CHANGED
|
@@ -39,7 +39,7 @@ from .registry import register
|
|
|
39
39
|
|
|
40
40
|
|
|
41
41
|
_DEFAULT_STORE = Path.home() / "Library" / "Application Support" / "orca" / "orchestration.db"
|
|
42
|
-
_NAMESPACE = "dev.
|
|
42
|
+
_NAMESPACE = "dev.loopmath.adapter.orca"
|
|
43
43
|
_MESSAGE_TYPES = ("decision_gate", "escalation", "merge_ready", "worker_done")
|
|
44
44
|
_DIAGNOSTIC_REASONS = (
|
|
45
45
|
"start_options_missing",
|
|
@@ -13,7 +13,7 @@ from typing import Any, Iterable, Mapping
|
|
|
13
13
|
from urllib.parse import quote
|
|
14
14
|
|
|
15
15
|
|
|
16
|
-
EXT = "dev.
|
|
16
|
+
EXT = "dev.loopmath.adapter.otel-genai"
|
|
17
17
|
_TRACE_ID = re.compile(r"[0-9a-fA-F]{32}")
|
|
18
18
|
_SPAN_ID = re.compile(r"[0-9a-fA-F]{16}")
|
|
19
19
|
_RFC3339 = re.compile(
|
loopmath/adapters/paseo.py
CHANGED
|
@@ -9,7 +9,7 @@ evidence that the work was accepted.
|
|
|
9
9
|
The vendor-session join is mechanically supported when
|
|
10
10
|
``persistence.sessionId`` equals both available native corroborators,
|
|
11
11
|
``persistence.nativeHandle`` and ``runtimeInfo.sessionId``. The matching value
|
|
12
|
-
is emitted as ``attempt.session``.
|
|
12
|
+
is emitted as ``attempt.session``. A read-only inspection also
|
|
13
13
|
matched every local value to a Claude or Codex session filename without reading
|
|
14
14
|
transcripts. This is a verified identity correlation, not a graph relation.
|
|
15
15
|
Paseo records no distinct launcher or parent endpoint, so this adapter emits no
|
|
@@ -38,7 +38,7 @@ from .registry import register
|
|
|
38
38
|
|
|
39
39
|
|
|
40
40
|
_DEFAULT_STORE = Path.home() / ".paseo" / "agents"
|
|
41
|
-
_NAMESPACE = "dev.
|
|
41
|
+
_NAMESPACE = "dev.loopmath.adapter.paseo"
|
|
42
42
|
_TERMINAL = {"failed", "canceled", "settled_unverified"}
|
|
43
43
|
_STATUS = {
|
|
44
44
|
"working": "working",
|
loopmath/belief/commands.py
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
"""Handler for `loopmath fit` (spec 02
|
|
1
|
+
"""Handler for `loopmath fit` (spec 02).
|
|
2
2
|
|
|
3
3
|
`loopmath fit [--background] [--full] [--no-prior] [--without SOURCE ...] [--home PATH] [--json]`
|
|
4
4
|
|
|
5
5
|
- Foreground: builds the belief (belief.fit.fit), waiting up to 30 s for a running fit to
|
|
6
6
|
finish (exit 4 after that), and prints a summary or `loopmath.fit/1` JSON.
|
|
7
7
|
- `--background`: detaches and returns at once through the store's fit trigger
|
|
8
|
-
(`store.fitjob.spawn_fit
|
|
8
|
+
(`store.fitjob.spawn_fit`), so a running fit queues one more fit instead of
|
|
9
9
|
racing it.
|
|
10
10
|
- A `--without` name that matches no source exits 2 and lists the known names; a fit
|
|
11
11
|
with nothing to fit writes nothing, leaves `fits/latest` where it was and exits 1.
|
|
@@ -74,6 +74,8 @@ def _summary(path: Path) -> dict:
|
|
|
74
74
|
"full": meta.get("full"),
|
|
75
75
|
"features": {k: v for k, v in (meta.get("features") or {}).items() if k != "declared"},
|
|
76
76
|
"horizons": meta.get("horizons") or {},
|
|
77
|
+
**({"receipts_rescored": meta["receipts_rescored"]} if meta.get("receipts_rescored") else {}),
|
|
78
|
+
**({"notes": meta["notes"]} if meta.get("notes") else {}),
|
|
77
79
|
}
|
|
78
80
|
|
|
79
81
|
|
|
@@ -85,6 +87,10 @@ def _print_summary(s: dict) -> None:
|
|
|
85
87
|
f = s["fit"]
|
|
86
88
|
n = f.get("n_runs") or {}
|
|
87
89
|
print(f"fit {f['id']} written to {s['path']} in {s.get('seconds')} s; recommend and posterior now read it")
|
|
90
|
+
rescored = s.get("receipts_rescored")
|
|
91
|
+
if rescored:
|
|
92
|
+
print(f"{rescored} receipt{'' if rescored == 1 else 's'} scored before 0.2.7 re-scored first: the run's rule is "
|
|
93
|
+
"read as the fit reads it (a note on each run)")
|
|
88
94
|
by_src = ", ".join(f"{k} {v}" for k, v in sorted(s["runs_by_source"].items(), key=lambda kv: -kv[1]))
|
|
89
95
|
labels = ", ".join(f"{k} {v}" for k, v in sorted((s.get("user_labels") or {}).items(), key=lambda kv: -kv[1]))
|
|
90
96
|
detail = "; ".join(x for x in (by_src, f"yours labelled {labels}" if labels else "") if x)
|
|
@@ -175,6 +181,9 @@ def fit(args: argparse.Namespace) -> int:
|
|
|
175
181
|
|
|
176
182
|
from .fit import FitBusy, NothingToFit, UnknownSource, fit as run_fit
|
|
177
183
|
|
|
184
|
+
notes = _config_notes(home)
|
|
185
|
+
for note in notes:
|
|
186
|
+
print(f"note: {note}", file=sys.stderr)
|
|
178
187
|
# --full: a failed compile makes pytensor print where it left the C code on stdout, which is for the result
|
|
179
188
|
quiet = contextlib.redirect_stdout(sys.stderr) if full else contextlib.nullcontext()
|
|
180
189
|
try:
|
|
@@ -192,8 +201,23 @@ def fit(args: argparse.Namespace) -> int:
|
|
|
192
201
|
raise
|
|
193
202
|
return _pymc_check_failed(home, exc)
|
|
194
203
|
summary = _summary(path)
|
|
204
|
+
if notes:
|
|
205
|
+
summary["notes"] = notes + summary.get("notes", []) # the fit printed its own on stderr
|
|
195
206
|
if as_json:
|
|
196
207
|
output.emit_json(SCHEMA, summary)
|
|
197
208
|
else:
|
|
198
209
|
_print_summary(summary)
|
|
199
210
|
return output.EXIT_OK
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _config_notes(home: Path) -> list[str]:
|
|
214
|
+
"""A config.toml the fit cannot read leaves it on the built-in values; say so, as `runs` does (0.2.7)."""
|
|
215
|
+
from ..store.config import READ_ERRORS, Config, read_error
|
|
216
|
+
|
|
217
|
+
path = Path(home) / "config.toml"
|
|
218
|
+
try:
|
|
219
|
+
Config.load(path)
|
|
220
|
+
except READ_ERRORS as exc:
|
|
221
|
+
return [f"{read_error(path, exc)}; outcomes use the built-in q, and the fit the built-in features and "
|
|
222
|
+
"benchmark weights"]
|
|
223
|
+
return []
|
loopmath/belief/design.py
CHANGED
|
@@ -36,7 +36,7 @@ HEURISTIC_COST_WEIGHT = 0.7 # spec 04 section 2 and D27: allocated (heuristic)
|
|
|
36
36
|
USER_SOURCES = ("live", "backlog", "history", "user", "orchestrator", "onboard", "habit", "")
|
|
37
37
|
HORIZON_NODES = ("horizon:timebox", "horizon:log2h") # fixed terms of a timeboxed run (spec 04 section 1)
|
|
38
38
|
RQ1_EXT = "dev.loopmath.rq1" # loopmath-exp's ext key; its horizon_s is the fallback for older RQ1 runs
|
|
39
|
-
# 0.2.1 (
|
|
39
|
+
# 0.2.1 (spec 04 section 1): in the cost and tokens rows of a timeboxed task the terms a timebox caps (effort
|
|
40
40
|
# and shape) enter at this value, not 1. Set from the RQ1 2 h runs: open-ended effort terms gave max over low x2.64
|
|
41
41
|
# (solo luna) and x2.23 (solo sol) against x1.34 and x1.27 observed; ln 1.34 / ln 2.64 = ln 1.27 / ln 2.23 = 0.30.
|
|
42
42
|
TIMEBOX_EFFORT_COST = 0.3
|
|
@@ -465,7 +465,7 @@ def source_label(doc: dict) -> str:
|
|
|
465
465
|
share = ext.get("dev.loopmath.share")
|
|
466
466
|
if isinstance(share, dict):
|
|
467
467
|
# the importer writes `source` and `task.org` as `shared:<org_hash>` already: never prefix twice. With no
|
|
468
|
-
# org at all the source is `shared`, the name `fit --without shared` takes (0.2.3
|
|
468
|
+
# org at all the source is `shared`, the name `fit --without shared` takes (0.2.3)
|
|
469
469
|
org = (share.get("source") or share.get("org") or share.get("org_hash") or (run.get("task") or {}).get("org")
|
|
470
470
|
or "")
|
|
471
471
|
name = str(org).removeprefix("shared:")
|
|
@@ -640,7 +640,7 @@ def _ocp_fields(cost: dict) -> dict[str, float]:
|
|
|
640
640
|
|
|
641
641
|
def _price_split(split: dict, table) -> float | None:
|
|
642
642
|
"""Dollars for `{model: OCP token fields}`, each model at its own current rate, summed; None when any
|
|
643
|
-
part is unpriced or unlabelled (
|
|
643
|
+
part is unpriced or unlabelled (`price.price_model_tokens`)."""
|
|
644
644
|
from ..price import price_model_tokens
|
|
645
645
|
|
|
646
646
|
usd = price_model_tokens(split, table).get("usd")
|
|
@@ -720,14 +720,21 @@ def _check_name(a: dict, node: dict) -> str | None:
|
|
|
720
720
|
return str(gate.get("rule") or node.get("id") or a.get("node") or a.get("harness") or "check")
|
|
721
721
|
|
|
722
722
|
|
|
723
|
+
def run_evidence(doc: dict, rule: AcceptanceRule, *, now: datetime,
|
|
724
|
+
q_by_tier: dict[str, float] | None = None) -> Evidence:
|
|
725
|
+
"""The run's outcome as the fit reads it: `rules.evidence` under `rule` (None `q_by_tier`: the built-in q)."""
|
|
726
|
+
from . import rules
|
|
727
|
+
|
|
728
|
+
return rules.evidence(doc, rule, now=now, q_by_tier=q_by_tier)
|
|
729
|
+
|
|
730
|
+
|
|
723
731
|
def parse_run(doc: dict, *, now: datetime | None = None, rule: AcceptanceRule | None = None,
|
|
724
|
-
origin: str | None = None) -> ParsedRun:
|
|
732
|
+
origin: str | None = None, q_by_tier: dict[str, float] | None = None) -> ParsedRun:
|
|
725
733
|
"""Everything the fit needs from one finished OCP v0.3 run document. Raises Unusable.
|
|
726
734
|
|
|
727
|
-
`origin` is where the fit found the document (`data_source`); None reads its label.
|
|
735
|
+
`origin` is where the fit found the document (`data_source`); None reads its label. `q_by_tier` is
|
|
736
|
+
`outcome.q.<tier>` from config.toml, passed to `outcome_evidence` (None: the built-in q).
|
|
728
737
|
"""
|
|
729
|
-
from .outcome import outcome_evidence
|
|
730
|
-
|
|
731
738
|
run = doc.get("run") or {}
|
|
732
739
|
cfg = run.get("configuration")
|
|
733
740
|
if not isinstance(cfg, dict):
|
|
@@ -785,8 +792,10 @@ def parse_run(doc: dict, *, now: datetime | None = None, rule: AcceptanceRule |
|
|
|
785
792
|
streams))
|
|
786
793
|
|
|
787
794
|
gates = _gate_observations(doc, st, by_piece_round)
|
|
788
|
-
|
|
789
|
-
|
|
795
|
+
from .rules import resolve_rule
|
|
796
|
+
|
|
797
|
+
use_rule = rule or resolve_rule(doc) # one resolver for every reader (0.2.7); raises on a malformed score
|
|
798
|
+
evidence = run_evidence(doc, use_rule, now=now or datetime.now().astimezone(), q_by_tier=q_by_tier)
|
|
790
799
|
score_meta: dict[str, dict] = {}
|
|
791
800
|
for s in run.get("signals") or []:
|
|
792
801
|
if s.get("kind") == "score" and isinstance(s.get("value"), (int, float)) and not isinstance(s.get("value"), bool):
|
loopmath/belief/fit.py
CHANGED
|
@@ -4,7 +4,7 @@ Spec 04 section 4. `fit()` reads finished OCP v0.3 runs (the store, the shipped
|
|
|
4
4
|
shared imports), builds the rows of every head, sets the level scales by empirical Bayes,
|
|
5
5
|
and writes `fits/<fit>.partial/{state.npz, meta.json, design.json}`, renamed to
|
|
6
6
|
`fits/<fit>/` when complete; `fits/latest` then points at it. A non-blocking `flock` on
|
|
7
|
-
`fits/.fit.lock` stops two fits running together (
|
|
7
|
+
`fits/.fit.lock` stops two fits running together (the store's fit trigger has its own job lock on
|
|
8
8
|
`fits/.lock`, so the two never wait on each other).
|
|
9
9
|
"""
|
|
10
10
|
|
|
@@ -136,7 +136,7 @@ def _nothing_to_fit(no_prior: bool, without: tuple[str, ...], dropped: dict[str,
|
|
|
136
136
|
# ---------------------------------------------------------------- reading documents
|
|
137
137
|
|
|
138
138
|
def store_docs(home: Path) -> Iterator[dict]:
|
|
139
|
-
"""Finished run documents in the store, through
|
|
139
|
+
"""Finished run documents in the store, through the store's reader (it decides what is finished)."""
|
|
140
140
|
from ..store.home import Store
|
|
141
141
|
|
|
142
142
|
yield from Store(home).finished_docs()
|
|
@@ -146,7 +146,7 @@ def _read_config(home: Path) -> dict:
|
|
|
146
146
|
path = Path(home) / "config.toml"
|
|
147
147
|
try:
|
|
148
148
|
return tomllib.loads(path.read_text(encoding="utf-8"))
|
|
149
|
-
except (OSError,
|
|
149
|
+
except (OSError, ValueError): # unreadable, not TOML, or not UTF-8: the built-in values (`fit` says so)
|
|
150
150
|
return {}
|
|
151
151
|
|
|
152
152
|
|
|
@@ -165,6 +165,8 @@ class HeadRows:
|
|
|
165
165
|
types: list[str] = field(default_factory=list)
|
|
166
166
|
not_repriced: int = 0 # cost rows on recorded dollars, not the current tariff
|
|
167
167
|
prices: dict = field(default_factory=dict) # cost head: the fit's price offsets (pricing.fit_offsets)
|
|
168
|
+
# success head, with a configured q only: each row's w under the built-in q, which the seed hashes (0.2.7)
|
|
169
|
+
seed_w: list[float] = field(default_factory=list)
|
|
168
170
|
|
|
169
171
|
def add(self, terms, y, w, run, source, ttype):
|
|
170
172
|
self.rows.append(terms)
|
|
@@ -183,7 +185,8 @@ def _excluded(source: str, without: tuple[str, ...]) -> bool:
|
|
|
183
185
|
|
|
184
186
|
|
|
185
187
|
def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tuple[str, ...] = (),
|
|
186
|
-
now: datetime | None = None, features: FeatureSet | None = None
|
|
188
|
+
now: datetime | None = None, features: FeatureSet | None = None,
|
|
189
|
+
q_by_tier: dict[str, float] | None = None):
|
|
187
190
|
"""Rows of every head from run documents, plus counts of what was used and dropped.
|
|
188
191
|
|
|
189
192
|
`docs` yields documents, or `(origin, document)` pairs from `fit()`: `user` for the store
|
|
@@ -199,11 +202,13 @@ def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tup
|
|
|
199
202
|
their document names, when that is not `user`. Features are read with `features` (the
|
|
200
203
|
built-ins by default); only admitted values keep their `feature:` terms, and `tally`
|
|
201
204
|
(features.FeatureTally) holds the support, the admitted values and the recorded horizons.
|
|
205
|
+
`q_by_tier` (`outcome.q.<tier>` from config.toml) reaches each run's outcome; None is the built-in q.
|
|
202
206
|
"""
|
|
203
207
|
now = now or datetime.now().astimezone()
|
|
204
208
|
features = features or FeatureSet()
|
|
205
209
|
tally = FeatureTally(features)
|
|
206
210
|
heads = {name: HeadRows(name, *HEAD_KIND[name]) for name in HEAD_KIND}
|
|
211
|
+
configured_q = custom_q(q_by_tier) is not None
|
|
207
212
|
score_raw: dict[str, list] = defaultdict(list)
|
|
208
213
|
score_meta: dict[str, Counter] = defaultdict(Counter)
|
|
209
214
|
score_info: dict[str, dict] = {}
|
|
@@ -232,7 +237,7 @@ def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tup
|
|
|
232
237
|
dropped[SHIPPED_COPY] += 1
|
|
233
238
|
continue
|
|
234
239
|
try:
|
|
235
|
-
pr = parse_run(doc, now=now, origin=origin)
|
|
240
|
+
pr = parse_run(doc, now=now, origin=origin, q_by_tier=q_by_tier)
|
|
236
241
|
except Unusable as exc:
|
|
237
242
|
dropped[str(exc)] += 1
|
|
238
243
|
continue
|
|
@@ -275,6 +280,8 @@ def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tup
|
|
|
275
280
|
ev = pr.evidence
|
|
276
281
|
if ev is not None and ev.z is not None:
|
|
277
282
|
heads["success"].add(row, float(ev.z), float(ev.q), pr.run_id, pr.source, ttype)
|
|
283
|
+
if configured_q: # z and the tier do not depend on q, so this read differs from `ev` in q only
|
|
284
|
+
heads["success"].seed_w.append(float(design_rows.run_evidence(doc, pr.rule, now=now).q))
|
|
278
285
|
for name, value in (ev.scores if ev else {}).items():
|
|
279
286
|
meta = pr.score_meta.get(name) or {}
|
|
280
287
|
score_meta[name][(meta.get("scale") or "linear", meta.get("better") or "higher", meta.get("unit"))] += 1
|
|
@@ -394,6 +401,16 @@ def _support(hr: HeadRows, X: sparse.csr_matrix, node_ids: list[str]) -> dict[st
|
|
|
394
401
|
return out
|
|
395
402
|
|
|
396
403
|
|
|
404
|
+
def custom_q(q_by_tier: dict[str, float] | None) -> dict[str, float] | None:
|
|
405
|
+
"""`outcome.q.<tier>` for every tier when it differs from the built-in q anywhere, else None."""
|
|
406
|
+
from .outcome import TIER_Q
|
|
407
|
+
|
|
408
|
+
if not q_by_tier:
|
|
409
|
+
return None
|
|
410
|
+
merged = {tier: float(q_by_tier.get(tier, q)) for tier, q in TIER_Q.items()}
|
|
411
|
+
return merged if merged != TIER_Q else None
|
|
412
|
+
|
|
413
|
+
|
|
397
414
|
def seed_for(fit_id: str, head: str) -> int:
|
|
398
415
|
"""A head's seed from the fit's `seed_key` (its fit id before 0.2.1)."""
|
|
399
416
|
return int.from_bytes(hashlib.sha256(f"{fit_id}/{head}".encode()).digest()[:4], "little")
|
|
@@ -402,7 +419,10 @@ def seed_for(fit_id: str, head: str) -> int:
|
|
|
402
419
|
def input_key(heads_rows: dict[str, HeadRows], specs: list, settings: dict) -> str:
|
|
403
420
|
"""The fit's `seed_key` (spec 04 section 3): the sha256 of the design version, the settings that change the
|
|
404
421
|
posterior, the prior factors and every head's rows (terms, response, weight, run, source), each head's rows
|
|
405
|
-
sorted, so the same data and settings seed the same draws whatever the store's reading order.
|
|
422
|
+
sorted, so the same data and settings seed the same draws whatever the store's reading order.
|
|
423
|
+
|
|
424
|
+
A success row's weight is its q. With `outcome.q` configured, the key takes each row's q under the built-in
|
|
425
|
+
map (`HeadRows.seed_w`), so setting q redraws nothing: only the success head's posterior moves (0.2.7)."""
|
|
406
426
|
h = hashlib.sha256()
|
|
407
427
|
h.update(repr((DESIGN_VERSION, design_rows.TIMEBOX_EFFORT_COST, design_rows.TIMEBOX_LEVELS,
|
|
408
428
|
sorted(settings.items()))).encode())
|
|
@@ -411,7 +431,8 @@ def input_key(heads_rows: dict[str, HeadRows], specs: list, settings: dict) -> s
|
|
|
411
431
|
for name in sorted(heads_rows):
|
|
412
432
|
hr = heads_rows[name]
|
|
413
433
|
h.update(f"\n{name}:{len(hr.rows)}".encode())
|
|
414
|
-
|
|
434
|
+
weights = hr.seed_w or hr.w
|
|
435
|
+
for line in sorted(repr((t, y, w, r, src)) for t, y, w, r, src in zip(hr.rows, hr.y, weights, hr.runs, hr.sources)):
|
|
415
436
|
h.update(line.encode())
|
|
416
437
|
return h.hexdigest()
|
|
417
438
|
|
|
@@ -511,12 +532,38 @@ def point_latest(fits: Path, fit_id: str) -> None:
|
|
|
511
532
|
os.replace(tmp, fits / "latest")
|
|
512
533
|
|
|
513
534
|
|
|
535
|
+
def _settle_receipts(home: Path) -> tuple[list[str], str | None]:
|
|
536
|
+
"""Receipts `run finish` scored before 0.2.7 under a rule it read otherwise than the fit are re-scored before
|
|
537
|
+
every fit of the store, foreground or background (`store.finish.settle_rule_receipts`). A receipt that could
|
|
538
|
+
not be read, re-scored or written, or a run file that could not be written, is named in a note on stderr, and the
|
|
539
|
+
next fit does what is left. Returns the receipts re-scored and the note, if any."""
|
|
540
|
+
import sys
|
|
541
|
+
|
|
542
|
+
from ..store.finish import settle_rule_receipts
|
|
543
|
+
from ..store.home import Store
|
|
544
|
+
|
|
545
|
+
try:
|
|
546
|
+
out = settle_rule_receipts(Store(home))
|
|
547
|
+
except Exception as exc: # the fit goes on; the receipts wait for the next one
|
|
548
|
+
note = f"receipts scored before 0.2.7 were not re-scored: {type(exc).__name__}: {exc}"[:300]
|
|
549
|
+
print(f"note: {note}", file=sys.stderr)
|
|
550
|
+
return [], note
|
|
551
|
+
failed = out["failed"]
|
|
552
|
+
if not failed:
|
|
553
|
+
return out["rescored"], None
|
|
554
|
+
shown = "; ".join(failed[:3]) + (f"; and {len(failed) - 3} more" if len(failed) > 3 else "")
|
|
555
|
+
note = f"receipts scored before 0.2.7 are not all settled yet; the next fit does what is left: {shown}"
|
|
556
|
+
print(f"note: {note}", file=sys.stderr)
|
|
557
|
+
return out["rescored"], note
|
|
558
|
+
|
|
559
|
+
|
|
514
560
|
def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), full: bool = False,
|
|
515
561
|
now: datetime | None = None, docs: Iterable[dict] | None = None, bundle_dir: Path | None = None,
|
|
516
562
|
benchmarks: Path | None = None, eb: bool = True, wait_s: float = 0.0) -> Path:
|
|
517
|
-
"""Build the belief and write `fits/<fit>/`; returns its path.
|
|
563
|
+
"""Build the belief and write `fits/<fit>/`; returns its path. A fit of the store's own runs (`docs` None)
|
|
564
|
+
first settles the receipts scored before 0.2.7 (`_settle_receipts`).
|
|
518
565
|
|
|
519
|
-
Raises FitBusy, or UnknownSource / NothingToFit before
|
|
566
|
+
Raises FitBusy, or UnknownSource / NothingToFit before any fit file is written.
|
|
520
567
|
"""
|
|
521
568
|
home = Path(home)
|
|
522
569
|
fits = home / "fits"
|
|
@@ -525,6 +572,7 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
525
572
|
without = tuple(without)
|
|
526
573
|
with fit_lock(home, wait_s):
|
|
527
574
|
remove_partials(home)
|
|
575
|
+
settled, settle_note = _settle_receipts(home) if docs is None else ([], None)
|
|
528
576
|
config = _read_config(home)
|
|
529
577
|
raw_weight = config.get("benchmark_prior_weight") # unset: each benchmark's own weight (spec 04 section 5)
|
|
530
578
|
weight = None if raw_weight is None else float(raw_weight)
|
|
@@ -532,6 +580,11 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
532
580
|
features, features_error = FeatureSet.from_config(config.get("features")), None
|
|
533
581
|
except FeatureConfigError as exc: # a hand-edited config.toml: fit with the built-ins, say why
|
|
534
582
|
features, features_error = FeatureSet(), str(exc)
|
|
583
|
+
from ..store.config import Config
|
|
584
|
+
from .outcome import TIER_Q
|
|
585
|
+
|
|
586
|
+
# `outcome.q.<tier>`, read as `run finish` reads it; unset, the built-in q (0.2.6, fix (1))
|
|
587
|
+
q_by_tier = {tier: Config(home / "config.toml", config).q_for_tier(tier) for tier in TIER_Q}
|
|
535
588
|
|
|
536
589
|
def all_docs() -> Iterator[dict | tuple[str | None, dict]]:
|
|
537
590
|
if docs is not None:
|
|
@@ -544,7 +597,7 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
544
597
|
yield from prior_data.shared_docs(home)
|
|
545
598
|
|
|
546
599
|
(heads_rows, score_info, dropped, runs_by_source, config_support, checks, overlap,
|
|
547
|
-
labels, tally) = collect_rows(all_docs(), without=without, now=now, features=features)
|
|
600
|
+
labels, tally) = collect_rows(all_docs(), without=without, now=now, features=features, q_by_tier=q_by_tier)
|
|
548
601
|
removed = {k[len("without "):] for k in dropped if k.startswith("without ")}
|
|
549
602
|
unknown, known = unknown_sources(without, set(runs_by_source) | removed, bundle_dir)
|
|
550
603
|
if unknown:
|
|
@@ -603,7 +656,10 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
603
656
|
_atomic_json(partial / "design.json", design)
|
|
604
657
|
meta = {
|
|
605
658
|
"fit": fit_id, "created_at": now.isoformat(timespec="seconds"), "code_version": _code_version(),
|
|
606
|
-
"seed_key": seed_key, "
|
|
659
|
+
"seed_key": seed_key, **({"outcome_q": custom_q(q_by_tier)} if custom_q(q_by_tier) else {}),
|
|
660
|
+
**({"receipts_rescored": len(settled)} if settled else {}),
|
|
661
|
+
**({"notes": [settle_note]} if settle_note else {}),
|
|
662
|
+
"timebox_effort": design_rows.TIMEBOX_EFFORT_COST,
|
|
607
663
|
"timebox_terms": list(design_rows.TIMEBOX_LEVELS),
|
|
608
664
|
"options": {"no_prior": no_prior, "without": list(without), "full": full, "eb": eb,
|
|
609
665
|
"benchmark_prior_weight": weight, "benchmark_weights": bench_weights},
|
loopmath/belief/forest.py
CHANGED
|
@@ -85,7 +85,7 @@ ROLE_WORDS = ("planner", "implementer", "reviewer", "tester", "referee", "worker
|
|
|
85
85
|
|
|
86
86
|
|
|
87
87
|
def canonical_role(role: str | None) -> str:
|
|
88
|
-
"""Fold a role word into the types.py vocabulary through `workflows.normalize_role()
|
|
88
|
+
"""Fold a role word into the types.py vocabulary through `workflows.normalize_role()`;
|
|
89
89
|
no role is `worker`."""
|
|
90
90
|
return _canonical_role(str(role or "").strip().lower())
|
|
91
91
|
|
|
@@ -107,7 +107,7 @@ _GPT_RE = re.compile(r"^gpt-(\d+(?:\.\d+)?)(?:-(.+))?$")
|
|
|
107
107
|
def canonical_model_id(model: str | None) -> str:
|
|
108
108
|
"""One spelling per model: `claude-opus-5-5`, `claude-haiku-4-5`, `gpt-6-astra`.
|
|
109
109
|
|
|
110
|
-
Passes through `ingest.base.canonical_model` first (
|
|
110
|
+
Passes through `ingest.base.canonical_model` first (it owns the aliases), then
|
|
111
111
|
rewrites Claude ids to the `claude-<family>-<major>[-<minor>]` form of spec 04.
|
|
112
112
|
"""
|
|
113
113
|
from ..ingest.base import canonical_model
|
loopmath/belief/outcome.py
CHANGED
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
with reason `late:<name>`.
|
|
10
10
|
- `q` comes from the weakest tier used: verified 0.98, reported 0.95, heuristic 0.8, asserted 0.7.
|
|
11
11
|
- `scores`: every measured score, raw, whatever the rule is.
|
|
12
|
-
- D22: a shared run (`run.ext["dev.loopmath.share"]`) returns the sender's carried
|
|
12
|
+
- D22: a shared run (`run.ext["dev.loopmath.share"]`) returns the sender's carried outcome; a share that
|
|
13
|
+
carries no q takes the tier's q from `q_by_tier`.
|
|
13
14
|
"""
|
|
14
15
|
|
|
15
16
|
from __future__ import annotations
|
|
@@ -60,17 +61,29 @@ def _last(signals: list[dict]) -> dict | None:
|
|
|
60
61
|
return indexed[-1][1]
|
|
61
62
|
|
|
62
63
|
|
|
63
|
-
def
|
|
64
|
-
|
|
64
|
+
def _shared_scores(raw: Any) -> dict[str, float]:
|
|
65
|
+
"""A share's scores: the list `share` writes (`{name, value, ...}`), or an object of name to value."""
|
|
66
|
+
if isinstance(raw, list):
|
|
67
|
+
pairs = [(s.get("name"), s.get("value")) for s in raw if isinstance(s, dict)]
|
|
68
|
+
elif isinstance(raw, dict):
|
|
69
|
+
pairs = list(raw.items())
|
|
70
|
+
else:
|
|
71
|
+
return {}
|
|
72
|
+
return {str(k): float(v) for k, v in pairs if k is not None and _is_number(v)}
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _shared_evidence(share: dict, qmap: dict[str, float]) -> Evidence | None:
|
|
76
|
+
"""The outcome a shared run carries: `outcome` as `share` writes it, else an `evidence` object or z and q
|
|
77
|
+
at the top of the ext (older shapes). With no carried q, `qmap` gives the tier's q, as for a local run."""
|
|
78
|
+
ev = next((share[k] for k in ("outcome", "evidence") if isinstance(share.get(k), dict)), share)
|
|
65
79
|
if "z" not in ev and "q" not in ev:
|
|
66
80
|
return None
|
|
67
81
|
z = ev.get("z")
|
|
68
82
|
tier = _tier(ev.get("tier"))
|
|
69
83
|
q = ev.get("q")
|
|
70
|
-
scores = share.get("scores") if
|
|
71
|
-
return Evidence(z=None if z is None else float(z), q=float(q) if q is not None else
|
|
72
|
-
scores=
|
|
73
|
-
reasons=("shared",))
|
|
84
|
+
scores = share.get("scores") if share.get("scores") is not None else ev.get("scores")
|
|
85
|
+
return Evidence(z=None if z is None else float(z), q=float(q) if q is not None else float(qmap[tier]), tier=tier,
|
|
86
|
+
scores=_shared_scores(scores), reasons=("shared",))
|
|
74
87
|
|
|
75
88
|
|
|
76
89
|
def outcome_evidence(run_doc: dict, rule: AcceptanceRule, *, now: datetime,
|
|
@@ -79,7 +92,7 @@ def outcome_evidence(run_doc: dict, rule: AcceptanceRule, *, now: datetime,
|
|
|
79
92
|
run = run_doc.get("run") or {}
|
|
80
93
|
share = (run.get("ext") or {}).get("dev.loopmath.share")
|
|
81
94
|
if isinstance(share, dict):
|
|
82
|
-
carried = _shared_evidence(share)
|
|
95
|
+
carried = _shared_evidence(share, qmap)
|
|
83
96
|
if carried is not None:
|
|
84
97
|
return carried
|
|
85
98
|
|
loopmath/belief/pricing.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""The price offset on the cost head (spec 04 section 2, 0.2.2
|
|
1
|
+
"""The price offset on the cost head (spec 04 section 2, 0.2.2).
|
|
2
2
|
|
|
3
3
|
A model's run cost moves with its list price. Every cost row of a model version `m` (not the tokens, gate,
|
|
4
4
|
success or score rows) carries the fixed term `price:offset` with the value
|
loopmath/belief/priors.py
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
"""Prior bundle and benchmark prior factors (spec 04 section 5).
|
|
2
2
|
|
|
3
|
-
- The shipped bundle
|
|
4
|
-
(`sweep`, `e0`, `rq1`, `repo_history`). `bundle_docs()` reads it through
|
|
3
|
+
- The shipped bundle enters the fit as data, each run under its `source` node
|
|
4
|
+
(`sweep`, `e0`, `rq1`, `repo_history`). `bundle_docs()` reads it through
|
|
5
5
|
`loopmath.priors.bundle_docs()`; `bundle_entries()` pairs each run with its manifest
|
|
6
6
|
source, so a store run with a shipped label stays the user's.
|
|
7
|
-
- Shared imports
|
|
7
|
+
- Shared imports enter under their `shared:<org>` source through
|
|
8
8
|
`share.import_.shared_runs(home)`.
|
|
9
9
|
- Benchmark priors (`priors/benchmarks.toml`, layout D36) become Gaussian prior factors on
|
|
10
10
|
version nodes: success head, the logit gap to the benchmark's reference model with variance
|
|
11
11
|
`1 / (w p (1 - p))`; cost head, the log cost ratio with variance `1 / w`; tokens head
|
|
12
12
|
(`kind = "tokens"`, published total tokens for the evaluation), the log token ratio
|
|
13
|
-
with variance `1 / w`. `w` is one model's share of the benchmark's weight `w_b` (0.2.3
|
|
14
|
-
|
|
13
|
+
with variance `1 / w`. `w` is one model's share of the benchmark's weight `w_b` (0.2.3):
|
|
14
|
+
the `weight` of its `[[benchmark]]` table (default 5, about 5 runs), shared by
|
|
15
15
|
the model's k results on that benchmark, so each gets `w_b / k` and a model with five
|
|
16
16
|
efforts pulls as hard in total as a model with one. Config `benchmark_prior_weight`, when
|
|
17
17
|
set, is one `w_b` for every benchmark; 0 turns the factors off.
|
|
@@ -33,38 +33,38 @@ PACKAGE_PRIORS = Path(__file__).resolve().parent.parent / "priors"
|
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
def bundle_docs(bundle_dir: Path | None = None, without: tuple[str, ...] = ()) -> Iterator[dict]:
|
|
36
|
-
"""OCP v0.3 run documents of the shipped prior bundle, or of `bundle_dir
|
|
37
|
-
from .. import priors as
|
|
36
|
+
"""OCP v0.3 run documents of the shipped prior bundle, or of `bundle_dir`, read by `loopmath.priors`."""
|
|
37
|
+
from .. import priors as bundle
|
|
38
38
|
|
|
39
|
-
yield from
|
|
39
|
+
yield from bundle.bundle_docs(without=tuple(without), directory=bundle_dir)
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
def bundle_entries(bundle_dir: Path | None = None) -> Iterator[tuple[str, dict]]:
|
|
43
43
|
"""(manifest source, document) for every shipped run: the fit's source for it (spec 04 section 1)."""
|
|
44
|
-
from .. import priors as
|
|
44
|
+
from .. import priors as bundle
|
|
45
45
|
|
|
46
|
-
yield from
|
|
46
|
+
yield from bundle.bundle_entries(directory=bundle_dir)
|
|
47
47
|
|
|
48
48
|
|
|
49
49
|
def run_id_of(doc: dict) -> str:
|
|
50
|
-
from .. import priors as
|
|
50
|
+
from .. import priors as bundle
|
|
51
51
|
|
|
52
|
-
return
|
|
52
|
+
return bundle.run_id_of(doc)
|
|
53
53
|
|
|
54
54
|
|
|
55
55
|
def bundle_sources(bundle_dir: Path | None = None) -> list[str]:
|
|
56
|
-
"""Source names in the shipped prior bundle, or in `bundle_dir` (
|
|
56
|
+
"""Source names in the shipped prior bundle, or in `bundle_dir` (from its manifest).
|
|
57
57
|
|
|
58
58
|
Read from `manifest()`: the package attribute `sources` turns into the `priors.sources`
|
|
59
59
|
submodule once anything imports it, so calling it fails in a full run.
|
|
60
60
|
"""
|
|
61
|
-
from .. import priors as
|
|
61
|
+
from .. import priors as bundle
|
|
62
62
|
|
|
63
|
-
return list(
|
|
63
|
+
return list(bundle.manifest(bundle_dir).get("sources") or {})
|
|
64
64
|
|
|
65
65
|
|
|
66
66
|
def shared_docs(home: Path) -> Iterator[dict]:
|
|
67
|
-
"""Runs imported from other organizations (
|
|
67
|
+
"""Runs imported from other organizations (`share.import_.shared_runs`)."""
|
|
68
68
|
from ..share.import_ import shared_runs
|
|
69
69
|
|
|
70
70
|
yield from shared_runs(home)
|
|
@@ -116,7 +116,7 @@ def benchmark_factors(path: Path | None = None, *, weight: float | None = None)
|
|
|
116
116
|
|
|
117
117
|
`weight` (config `benchmark_prior_weight`) overrides every benchmark's own weight; None reads
|
|
118
118
|
each benchmark's `weight`. A model's k results on one benchmark share its weight: each factor
|
|
119
|
-
has weight `w_b / k
|
|
119
|
+
has weight `w_b / k`. A benchmark with weight 0 or less gives no factors.
|
|
120
120
|
"""
|
|
121
121
|
path = path or benchmark_path()
|
|
122
122
|
if not path.is_file():
|
|
@@ -154,7 +154,7 @@ def benchmark_factors(path: Path | None = None, *, weight: float | None = None)
|
|
|
154
154
|
elif kind in ("cost", "tokens") and value > 0 and ref_value > 0: # Tokens feed the tokens head only
|
|
155
155
|
by_model.setdefault(model, []).append((kind, terms, math.log(value / ref_value), r))
|
|
156
156
|
for model, items in by_model.items():
|
|
157
|
-
w = w_b / len(items) # one budget per model and benchmark
|
|
157
|
+
w = w_b / len(items) # one budget per model and benchmark
|
|
158
158
|
share = f"weight {w_b:g} over {len(items)}" if len(items) > 1 else f"weight {w_b:g}"
|
|
159
159
|
for head, terms, mean, r in items:
|
|
160
160
|
note = (f"{bid}: {model} {float(r['value'])} against {ref} {ref_value} ({share}; "
|