loopmath 0.2.4__py3-none-any.whl → 0.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loopmath/__init__.py +1 -1
- loopmath/adapters/atrium.py +1 -1
- loopmath/adapters/opencode.py +1 -1
- loopmath/adapters/paseo.py +1 -1
- loopmath/adapters/pi.py +2 -2
- loopmath/belief/commands.py +2 -2
- loopmath/belief/design.py +16 -6
- loopmath/belief/fit.py +12 -5
- loopmath/belief/forest.py +2 -2
- loopmath/belief/pricing.py +1 -1
- loopmath/belief/priors.py +18 -18
- loopmath/builder/__init__.py +1 -1
- loopmath/builder/context.py +4 -4
- loopmath/builder/predict.py +5 -5
- loopmath/cli.py +1 -1
- loopmath/cli_registry.py +7 -6
- loopmath/cli_support.py +31 -50
- loopmath/e0/arms.py +19 -22
- loopmath/e0/figures.py +1 -1
- loopmath/fit.py +36 -44
- loopmath/fit_assembly.py +7 -7
- loopmath/fit_predict.py +1 -1
- loopmath/fit_pricing.py +9 -9
- loopmath/fit_transfer.py +4 -4
- loopmath/grade.py +5 -5
- loopmath/grade_support.py +1 -1
- loopmath/graph/dataset_build.py +1 -1
- loopmath/graph/dataset_core.py +1 -1
- loopmath/graph/extract.py +14 -14
- loopmath/graph/html_common.py +32 -9
- loopmath/graph/html_costcurve.py +7 -3
- loopmath/graph/html_force.py +7 -3
- loopmath/graph/html_swimlanes.py +1 -1
- loopmath/graph/labels.py +1 -1
- loopmath/graph/launch.py +1 -1
- loopmath/graph/launch_origin.py +2 -2
- loopmath/graph/ocp_support.py +4 -4
- loopmath/graph/scan.py +5 -5
- loopmath/graph/schema.py +1 -1
- loopmath/ingest/__init__.py +13 -15
- loopmath/ingest/base.py +1 -1
- loopmath/ingest/claude_code.py +4 -4
- loopmath/ingest/codex.py +13 -13
- loopmath/ingest/codex_support.py +2 -2
- loopmath/logmatch/artifacts.py +10 -5
- loopmath/logmatch/clip.py +35 -21
- loopmath/logmatch/codex_window.py +164 -0
- loopmath/logmatch/match.py +3 -1
- loopmath/logmatch/settle.py +18 -11
- loopmath/logmatch/tariff.py +1 -1
- loopmath/ocp/contractv3.py +13 -4
- loopmath/ocp/graph_contract.py +86 -0
- loopmath/ocp/membership.py +69 -0
- loopmath/ocp/migrate.py +20 -13
- loopmath/onboard/commands.py +4 -4
- loopmath/onboard/history.py +3 -3
- loopmath/onboard/label.py +1 -1
- loopmath/onboard/usual.py +1 -1
- loopmath/output.py +1 -1
- loopmath/price.py +1 -1
- loopmath/prices.toml +1 -1
- loopmath/priors/__init__.py +5 -5
- loopmath/priors/benchmarks.py +4 -4
- loopmath/priors/benchmarks.toml +114 -115
- loopmath/priors/build.py +1 -1
- loopmath/priors/commands.py +2 -2
- loopmath/priors/e0.py +4 -4
- loopmath/priors/history.py +1 -1
- loopmath/priors/lanes.py +3 -3
- loopmath/priors/ocpdoc.py +10 -10
- loopmath/priors/registry.py +3 -3
- loopmath/priors/rq1.py +22 -22
- loopmath/priors/show.py +3 -3
- loopmath/priors/sweep.py +1 -1
- loopmath/priors/validate.py +3 -3
- loopmath/receipts.py +1 -1
- loopmath/recommend/commands.py +21 -20
- loopmath/recommend/curve.py +1 -1
- loopmath/recommend/engine.py +12 -12
- loopmath/recommend/message.py +7 -7
- loopmath/recommend/receipts.py +1 -1
- loopmath/recommend/storeread.py +6 -6
- loopmath/report/eligibility.py +45 -0
- loopmath/report/html.py +5 -2
- loopmath/report/terminal.py +13 -30
- loopmath/report/terminal_read.py +5 -6
- loopmath/share/commands.py +14 -2
- loopmath/share/export.py +4 -4
- loopmath/share/import_.py +1 -1
- loopmath/skill/skills/loopmath-record-run/SKILL.md +1 -1
- loopmath/skill/skills/reference.md +1 -1
- loopmath/store/commands.py +4 -4
- loopmath/store/config.py +13 -3
- loopmath/store/finish.py +24 -23
- loopmath/store/fitjob.py +9 -5
- loopmath/store/home.py +2 -2
- loopmath/store/ids.py +1 -1
- loopmath/store/record.py +38 -10
- loopmath/store/runs.py +2 -2
- loopmath/store/status.py +1 -1
- loopmath/surface.py +6 -7
- loopmath/surface_bootstrap.py +4 -4
- loopmath/surface_estimation.py +15 -16
- loopmath/surface_frame.py +1 -1
- loopmath/surface_presentation.py +7 -8
- loopmath/surface_support.py +4 -5
- loopmath/taskmodel.py +1 -1
- loopmath/types.py +2 -2
- loopmath/views/__init__.py +1 -1
- loopmath/views/assets/README.md +1 -1
- loopmath/views/assets/base.css +2 -2
- loopmath/views/assets/builder.css +1 -1
- loopmath/views/assets/builder.js +5 -5
- loopmath/views/assets/common.js +40 -8
- loopmath/views/assets/estimates.js +6 -6
- loopmath/views/assets/graph.css +1 -1
- loopmath/views/assets/graph.js +2 -2
- loopmath/views/assets/plans.css +2 -2
- loopmath/views/assets/plans.js +23 -23
- loopmath/views/assets/results.css +1 -1
- loopmath/views/assets/results.js +11 -11
- loopmath/views/assets/runs.js +2 -2
- loopmath/views/assets/viz.css +2 -2
- loopmath/views/assets/viz.js +5 -5
- loopmath/views/common.py +9 -16
- loopmath/views/plans.py +4 -4
- loopmath/views/posterior.py +25 -25
- loopmath/views/runs.py +31 -15
- loopmath/workflows/__init__.py +1 -1
- loopmath/workflows/candidates.py +3 -3
- loopmath/workflows/commands.py +2 -2
- loopmath/workflows/graphview.py +2 -2
- loopmath/workflows/ids.py +3 -3
- loopmath/workflows/models.py +3 -3
- loopmath/workflows/ocp.py +4 -4
- {loopmath-0.2.4.dist-info → loopmath-0.2.6.dist-info}/METADATA +4 -4
- {loopmath-0.2.4.dist-info → loopmath-0.2.6.dist-info}/RECORD +144 -140
- {loopmath-0.2.4.data → loopmath-0.2.6.data}/data/spec/ocp-v0.2.schema.json +0 -0
- {loopmath-0.2.4.data → loopmath-0.2.6.data}/data/spec/ocp-v0.schema.json +0 -0
- {loopmath-0.2.4.data → loopmath-0.2.6.data}/data/spec/ocp_conformance.py +0 -0
- {loopmath-0.2.4.dist-info → loopmath-0.2.6.dist-info}/WHEEL +0 -0
- {loopmath-0.2.4.dist-info → loopmath-0.2.6.dist-info}/entry_points.txt +0 -0
- {loopmath-0.2.4.dist-info → loopmath-0.2.6.dist-info}/licenses/LICENSE +0 -0
- {loopmath-0.2.4.dist-info → loopmath-0.2.6.dist-info}/top_level.txt +0 -0
loopmath/__init__.py
CHANGED
loopmath/adapters/atrium.py
CHANGED
|
@@ -10,7 +10,7 @@ does not infer a spawn relation from a subagent-stop record or infer acceptance
|
|
|
10
10
|
from segment closure, session end, or subagent stop.
|
|
11
11
|
|
|
12
12
|
The source has no token counts, cost, model, effort, stable task node, or
|
|
13
|
-
acceptance result.
|
|
13
|
+
acceptance result. Locally inspected data can be weeks stale, and
|
|
14
14
|
its pane-to-session coverage is sparse. SQL projection excludes prompt,
|
|
15
15
|
message, title, body, transcript, actor, and task text before rows reach
|
|
16
16
|
Python; none is retained or emitted.
|
loopmath/adapters/opencode.py
CHANGED
|
@@ -103,7 +103,7 @@ _sys.modules[__name__].__class__ = _OpenCodeModule
|
|
|
103
103
|
del _ModuleType, _OpenCodeModule, _opencode_store, _sys
|
|
104
104
|
|
|
105
105
|
|
|
106
|
-
_EXT = "dev.
|
|
106
|
+
_EXT = "dev.loopmath.adapter.opencode"
|
|
107
107
|
|
|
108
108
|
def _session_created(session: _SessionSource) -> datetime | None:
|
|
109
109
|
return _timestamp(_as_mapping(session.info.get("time")).get("created"))
|
loopmath/adapters/paseo.py
CHANGED
|
@@ -9,7 +9,7 @@ evidence that the work was accepted.
|
|
|
9
9
|
The vendor-session join is mechanically supported when
|
|
10
10
|
``persistence.sessionId`` equals both available native corroborators,
|
|
11
11
|
``persistence.nativeHandle`` and ``runtimeInfo.sessionId``. The matching value
|
|
12
|
-
is emitted as ``attempt.session``.
|
|
12
|
+
is emitted as ``attempt.session``. A read-only inspection also
|
|
13
13
|
matched every local value to a Claude or Codex session filename without reading
|
|
14
14
|
transcripts. This is a verified identity correlation, not a graph relation.
|
|
15
15
|
Paseo records no distinct launcher or parent endpoint, so this adapter emits no
|
loopmath/adapters/pi.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
One Pi session is one OCP node with one attempt. Usage-bearing entries are
|
|
4
4
|
measured requests: their token and USD fields are summed into the attempt's
|
|
5
|
-
core cost record, while ``dev.
|
|
5
|
+
core cost record, while ``dev.loopmath.adapter.pi.usage_records`` retains the exact
|
|
6
6
|
per-entry breakdown that OCP v0.2 cannot express. Model labels on assistant
|
|
7
7
|
messages are verified; thinking effort is reported by Pi's explicit
|
|
8
8
|
``thinking_level_change`` entries.
|
|
@@ -57,7 +57,7 @@ from .registry import register
|
|
|
57
57
|
|
|
58
58
|
|
|
59
59
|
_DEFAULT_STORE = Path.home() / ".pi" / "agent" / "sessions"
|
|
60
|
-
_EXT = "dev.
|
|
60
|
+
_EXT = "dev.loopmath.adapter.pi"
|
|
61
61
|
|
|
62
62
|
|
|
63
63
|
class _PiModule(_ModuleType):
|
loopmath/belief/commands.py
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
"""Handler for `loopmath fit` (spec 02
|
|
1
|
+
"""Handler for `loopmath fit` (spec 02).
|
|
2
2
|
|
|
3
3
|
`loopmath fit [--background] [--full] [--no-prior] [--without SOURCE ...] [--home PATH] [--json]`
|
|
4
4
|
|
|
5
5
|
- Foreground: builds the belief (belief.fit.fit), waiting up to 30 s for a running fit to
|
|
6
6
|
finish (exit 4 after that), and prints a summary or `loopmath.fit/1` JSON.
|
|
7
7
|
- `--background`: detaches and returns at once through the store's fit trigger
|
|
8
|
-
(`store.fitjob.spawn_fit
|
|
8
|
+
(`store.fitjob.spawn_fit`), so a running fit queues one more fit instead of
|
|
9
9
|
racing it.
|
|
10
10
|
- A `--without` name that matches no source exits 2 and lists the known names; a fit
|
|
11
11
|
with nothing to fit writes nothing, leaves `fits/latest` where it was and exits 1.
|
loopmath/belief/design.py
CHANGED
|
@@ -36,7 +36,7 @@ HEURISTIC_COST_WEIGHT = 0.7 # spec 04 section 2 and D27: allocated (heuristic)
|
|
|
36
36
|
USER_SOURCES = ("live", "backlog", "history", "user", "orchestrator", "onboard", "habit", "")
|
|
37
37
|
HORIZON_NODES = ("horizon:timebox", "horizon:log2h") # fixed terms of a timeboxed run (spec 04 section 1)
|
|
38
38
|
RQ1_EXT = "dev.loopmath.rq1" # loopmath-exp's ext key; its horizon_s is the fallback for older RQ1 runs
|
|
39
|
-
# 0.2.1 (
|
|
39
|
+
# 0.2.1 (spec 04 section 1): in the cost and tokens rows of a timeboxed task the terms a timebox caps (effort
|
|
40
40
|
# and shape) enter at this value, not 1. Set from the RQ1 2 h runs: open-ended effort terms gave max over low x2.64
|
|
41
41
|
# (solo luna) and x2.23 (solo sol) against x1.34 and x1.27 observed; ln 1.34 / ln 2.64 = ln 1.27 / ln 2.23 = 0.30.
|
|
42
42
|
TIMEBOX_EFFORT_COST = 0.3
|
|
@@ -465,7 +465,7 @@ def source_label(doc: dict) -> str:
|
|
|
465
465
|
share = ext.get("dev.loopmath.share")
|
|
466
466
|
if isinstance(share, dict):
|
|
467
467
|
# the importer writes `source` and `task.org` as `shared:<org_hash>` already: never prefix twice. With no
|
|
468
|
-
# org at all the source is `shared`, the name `fit --without shared` takes (0.2.3
|
|
468
|
+
# org at all the source is `shared`, the name `fit --without shared` takes (0.2.3)
|
|
469
469
|
org = (share.get("source") or share.get("org") or share.get("org_hash") or (run.get("task") or {}).get("org")
|
|
470
470
|
or "")
|
|
471
471
|
name = str(org).removeprefix("shared:")
|
|
@@ -640,7 +640,7 @@ def _ocp_fields(cost: dict) -> dict[str, float]:
|
|
|
640
640
|
|
|
641
641
|
def _price_split(split: dict, table) -> float | None:
|
|
642
642
|
"""Dollars for `{model: OCP token fields}`, each model at its own current rate, summed; None when any
|
|
643
|
-
part is unpriced or unlabelled (
|
|
643
|
+
part is unpriced or unlabelled (`price.price_model_tokens`)."""
|
|
644
644
|
from ..price import price_model_tokens
|
|
645
645
|
|
|
646
646
|
usd = price_model_tokens(split, table).get("usd")
|
|
@@ -721,10 +721,11 @@ def _check_name(a: dict, node: dict) -> str | None:
|
|
|
721
721
|
|
|
722
722
|
|
|
723
723
|
def parse_run(doc: dict, *, now: datetime | None = None, rule: AcceptanceRule | None = None,
|
|
724
|
-
origin: str | None = None) -> ParsedRun:
|
|
724
|
+
origin: str | None = None, q_by_tier: dict[str, float] | None = None) -> ParsedRun:
|
|
725
725
|
"""Everything the fit needs from one finished OCP v0.3 run document. Raises Unusable.
|
|
726
726
|
|
|
727
|
-
`origin` is where the fit found the document (`data_source`); None reads its label.
|
|
727
|
+
`origin` is where the fit found the document (`data_source`); None reads its label. `q_by_tier` is
|
|
728
|
+
`outcome.q.<tier>` from config.toml, passed to `outcome_evidence` (None: the built-in q).
|
|
728
729
|
"""
|
|
729
730
|
from .outcome import outcome_evidence
|
|
730
731
|
|
|
@@ -741,10 +742,19 @@ def parse_run(doc: dict, *, now: datetime | None = None, rule: AcceptanceRule |
|
|
|
741
742
|
|
|
742
743
|
nodes = {n.get("id"): n for n in doc.get("nodes") or [] if isinstance(n, dict)}
|
|
743
744
|
node_vertex = {i: n.get("vertex") for i, n in nodes.items()}
|
|
745
|
+
from ..ocp.membership import imported_node_map
|
|
746
|
+
|
|
747
|
+
imported_vertices = imported_node_map(doc)
|
|
744
748
|
attempts: list[AttemptObs] = []
|
|
745
749
|
by_piece_round: dict[tuple[str, int], list[dict]] = {}
|
|
746
750
|
for a in doc.get("attempts") or []:
|
|
747
751
|
piece = a.get("vertex") or node_vertex.get(a.get("node"))
|
|
752
|
+
node = nodes.get(a.get("node"))
|
|
753
|
+
if (piece not in st.settings and node is not None and "vertex" not in a and "vertex" not in node
|
|
754
|
+
and _check_name(a, node) is None):
|
|
755
|
+
mapped = imported_vertices.get(a.get("node"))
|
|
756
|
+
if isinstance(mapped, str) and mapped in st.settings:
|
|
757
|
+
piece = mapped
|
|
748
758
|
if piece not in st.settings:
|
|
749
759
|
name = _check_name(a, nodes.get(a.get("node")) or {})
|
|
750
760
|
if name is not None:
|
|
@@ -777,7 +787,7 @@ def parse_run(doc: dict, *, now: datetime | None = None, rule: AcceptanceRule |
|
|
|
777
787
|
|
|
778
788
|
gates = _gate_observations(doc, st, by_piece_round)
|
|
779
789
|
use_rule = rule or rule_from_doc(doc)
|
|
780
|
-
evidence = outcome_evidence(doc, use_rule, now=now or datetime.now().astimezone())
|
|
790
|
+
evidence = outcome_evidence(doc, use_rule, now=now or datetime.now().astimezone(), q_by_tier=q_by_tier)
|
|
781
791
|
score_meta: dict[str, dict] = {}
|
|
782
792
|
for s in run.get("signals") or []:
|
|
783
793
|
if s.get("kind") == "score" and isinstance(s.get("value"), (int, float)) and not isinstance(s.get("value"), bool):
|
loopmath/belief/fit.py
CHANGED
|
@@ -4,7 +4,7 @@ Spec 04 section 4. `fit()` reads finished OCP v0.3 runs (the store, the shipped
|
|
|
4
4
|
shared imports), builds the rows of every head, sets the level scales by empirical Bayes,
|
|
5
5
|
and writes `fits/<fit>.partial/{state.npz, meta.json, design.json}`, renamed to
|
|
6
6
|
`fits/<fit>/` when complete; `fits/latest` then points at it. A non-blocking `flock` on
|
|
7
|
-
`fits/.fit.lock` stops two fits running together (
|
|
7
|
+
`fits/.fit.lock` stops two fits running together (the store's fit trigger has its own job lock on
|
|
8
8
|
`fits/.lock`, so the two never wait on each other).
|
|
9
9
|
"""
|
|
10
10
|
|
|
@@ -136,7 +136,7 @@ def _nothing_to_fit(no_prior: bool, without: tuple[str, ...], dropped: dict[str,
|
|
|
136
136
|
# ---------------------------------------------------------------- reading documents
|
|
137
137
|
|
|
138
138
|
def store_docs(home: Path) -> Iterator[dict]:
|
|
139
|
-
"""Finished run documents in the store, through
|
|
139
|
+
"""Finished run documents in the store, through the store's reader (it decides what is finished)."""
|
|
140
140
|
from ..store.home import Store
|
|
141
141
|
|
|
142
142
|
yield from Store(home).finished_docs()
|
|
@@ -183,7 +183,8 @@ def _excluded(source: str, without: tuple[str, ...]) -> bool:
|
|
|
183
183
|
|
|
184
184
|
|
|
185
185
|
def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tuple[str, ...] = (),
|
|
186
|
-
now: datetime | None = None, features: FeatureSet | None = None
|
|
186
|
+
now: datetime | None = None, features: FeatureSet | None = None,
|
|
187
|
+
q_by_tier: dict[str, float] | None = None):
|
|
187
188
|
"""Rows of every head from run documents, plus counts of what was used and dropped.
|
|
188
189
|
|
|
189
190
|
`docs` yields documents, or `(origin, document)` pairs from `fit()`: `user` for the store
|
|
@@ -199,6 +200,7 @@ def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tup
|
|
|
199
200
|
their document names, when that is not `user`. Features are read with `features` (the
|
|
200
201
|
built-ins by default); only admitted values keep their `feature:` terms, and `tally`
|
|
201
202
|
(features.FeatureTally) holds the support, the admitted values and the recorded horizons.
|
|
203
|
+
`q_by_tier` (`outcome.q.<tier>` from config.toml) reaches each run's outcome; None is the built-in q.
|
|
202
204
|
"""
|
|
203
205
|
now = now or datetime.now().astimezone()
|
|
204
206
|
features = features or FeatureSet()
|
|
@@ -232,7 +234,7 @@ def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tup
|
|
|
232
234
|
dropped[SHIPPED_COPY] += 1
|
|
233
235
|
continue
|
|
234
236
|
try:
|
|
235
|
-
pr = parse_run(doc, now=now, origin=origin)
|
|
237
|
+
pr = parse_run(doc, now=now, origin=origin, q_by_tier=q_by_tier)
|
|
236
238
|
except Unusable as exc:
|
|
237
239
|
dropped[str(exc)] += 1
|
|
238
240
|
continue
|
|
@@ -532,6 +534,11 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
532
534
|
features, features_error = FeatureSet.from_config(config.get("features")), None
|
|
533
535
|
except FeatureConfigError as exc: # a hand-edited config.toml: fit with the built-ins, say why
|
|
534
536
|
features, features_error = FeatureSet(), str(exc)
|
|
537
|
+
from ..store.config import Config
|
|
538
|
+
from .outcome import TIER_Q
|
|
539
|
+
|
|
540
|
+
# `outcome.q.<tier>`, read as `run finish` reads it; unset, the built-in q (0.2.6, fix (1))
|
|
541
|
+
q_by_tier = {tier: Config(home / "config.toml", config).q_for_tier(tier) for tier in TIER_Q}
|
|
535
542
|
|
|
536
543
|
def all_docs() -> Iterator[dict | tuple[str | None, dict]]:
|
|
537
544
|
if docs is not None:
|
|
@@ -544,7 +551,7 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
544
551
|
yield from prior_data.shared_docs(home)
|
|
545
552
|
|
|
546
553
|
(heads_rows, score_info, dropped, runs_by_source, config_support, checks, overlap,
|
|
547
|
-
labels, tally) = collect_rows(all_docs(), without=without, now=now, features=features)
|
|
554
|
+
labels, tally) = collect_rows(all_docs(), without=without, now=now, features=features, q_by_tier=q_by_tier)
|
|
548
555
|
removed = {k[len("without "):] for k in dropped if k.startswith("without ")}
|
|
549
556
|
unknown, known = unknown_sources(without, set(runs_by_source) | removed, bundle_dir)
|
|
550
557
|
if unknown:
|
loopmath/belief/forest.py
CHANGED
|
@@ -85,7 +85,7 @@ ROLE_WORDS = ("planner", "implementer", "reviewer", "tester", "referee", "worker
|
|
|
85
85
|
|
|
86
86
|
|
|
87
87
|
def canonical_role(role: str | None) -> str:
|
|
88
|
-
"""Fold a role word into the types.py vocabulary through `workflows.normalize_role()
|
|
88
|
+
"""Fold a role word into the types.py vocabulary through `workflows.normalize_role()`;
|
|
89
89
|
no role is `worker`."""
|
|
90
90
|
return _canonical_role(str(role or "").strip().lower())
|
|
91
91
|
|
|
@@ -107,7 +107,7 @@ _GPT_RE = re.compile(r"^gpt-(\d+(?:\.\d+)?)(?:-(.+))?$")
|
|
|
107
107
|
def canonical_model_id(model: str | None) -> str:
|
|
108
108
|
"""One spelling per model: `claude-opus-5-5`, `claude-haiku-4-5`, `gpt-6-astra`.
|
|
109
109
|
|
|
110
|
-
Passes through `ingest.base.canonical_model` first (
|
|
110
|
+
Passes through `ingest.base.canonical_model` first (it owns the aliases), then
|
|
111
111
|
rewrites Claude ids to the `claude-<family>-<major>[-<minor>]` form of spec 04.
|
|
112
112
|
"""
|
|
113
113
|
from ..ingest.base import canonical_model
|
loopmath/belief/pricing.py
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""The price offset on the cost head (spec 04 section 2, 0.2.2
|
|
1
|
+
"""The price offset on the cost head (spec 04 section 2, 0.2.2).
|
|
2
2
|
|
|
3
3
|
A model's run cost moves with its list price. Every cost row of a model version `m` (not the tokens, gate,
|
|
4
4
|
success or score rows) carries the fixed term `price:offset` with the value
|
loopmath/belief/priors.py
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
"""Prior bundle and benchmark prior factors (spec 04 section 5).
|
|
2
2
|
|
|
3
|
-
- The shipped bundle
|
|
4
|
-
(`sweep`, `e0`, `rq1`, `repo_history`). `bundle_docs()` reads it through
|
|
3
|
+
- The shipped bundle enters the fit as data, each run under its `source` node
|
|
4
|
+
(`sweep`, `e0`, `rq1`, `repo_history`). `bundle_docs()` reads it through
|
|
5
5
|
`loopmath.priors.bundle_docs()`; `bundle_entries()` pairs each run with its manifest
|
|
6
6
|
source, so a store run with a shipped label stays the user's.
|
|
7
|
-
- Shared imports
|
|
7
|
+
- Shared imports enter under their `shared:<org>` source through
|
|
8
8
|
`share.import_.shared_runs(home)`.
|
|
9
9
|
- Benchmark priors (`priors/benchmarks.toml`, layout D36) become Gaussian prior factors on
|
|
10
10
|
version nodes: success head, the logit gap to the benchmark's reference model with variance
|
|
11
11
|
`1 / (w p (1 - p))`; cost head, the log cost ratio with variance `1 / w`; tokens head
|
|
12
12
|
(`kind = "tokens"`, published total tokens for the evaluation), the log token ratio
|
|
13
|
-
with variance `1 / w`. `w` is one model's share of the benchmark's weight `w_b` (0.2.3
|
|
14
|
-
|
|
13
|
+
with variance `1 / w`. `w` is one model's share of the benchmark's weight `w_b` (0.2.3):
|
|
14
|
+
the `weight` of its `[[benchmark]]` table (default 5, about 5 runs), shared by
|
|
15
15
|
the model's k results on that benchmark, so each gets `w_b / k` and a model with five
|
|
16
16
|
efforts pulls as hard in total as a model with one. Config `benchmark_prior_weight`, when
|
|
17
17
|
set, is one `w_b` for every benchmark; 0 turns the factors off.
|
|
@@ -33,38 +33,38 @@ PACKAGE_PRIORS = Path(__file__).resolve().parent.parent / "priors"
|
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
def bundle_docs(bundle_dir: Path | None = None, without: tuple[str, ...] = ()) -> Iterator[dict]:
|
|
36
|
-
"""OCP v0.3 run documents of the shipped prior bundle, or of `bundle_dir
|
|
37
|
-
from .. import priors as
|
|
36
|
+
"""OCP v0.3 run documents of the shipped prior bundle, or of `bundle_dir`, read by `loopmath.priors`."""
|
|
37
|
+
from .. import priors as bundle
|
|
38
38
|
|
|
39
|
-
yield from
|
|
39
|
+
yield from bundle.bundle_docs(without=tuple(without), directory=bundle_dir)
|
|
40
40
|
|
|
41
41
|
|
|
42
42
|
def bundle_entries(bundle_dir: Path | None = None) -> Iterator[tuple[str, dict]]:
|
|
43
43
|
"""(manifest source, document) for every shipped run: the fit's source for it (spec 04 section 1)."""
|
|
44
|
-
from .. import priors as
|
|
44
|
+
from .. import priors as bundle
|
|
45
45
|
|
|
46
|
-
yield from
|
|
46
|
+
yield from bundle.bundle_entries(directory=bundle_dir)
|
|
47
47
|
|
|
48
48
|
|
|
49
49
|
def run_id_of(doc: dict) -> str:
|
|
50
|
-
from .. import priors as
|
|
50
|
+
from .. import priors as bundle
|
|
51
51
|
|
|
52
|
-
return
|
|
52
|
+
return bundle.run_id_of(doc)
|
|
53
53
|
|
|
54
54
|
|
|
55
55
|
def bundle_sources(bundle_dir: Path | None = None) -> list[str]:
|
|
56
|
-
"""Source names in the shipped prior bundle, or in `bundle_dir` (
|
|
56
|
+
"""Source names in the shipped prior bundle, or in `bundle_dir` (from its manifest).
|
|
57
57
|
|
|
58
58
|
Read from `manifest()`: the package attribute `sources` turns into the `priors.sources`
|
|
59
59
|
submodule once anything imports it, so calling it fails in a full run.
|
|
60
60
|
"""
|
|
61
|
-
from .. import priors as
|
|
61
|
+
from .. import priors as bundle
|
|
62
62
|
|
|
63
|
-
return list(
|
|
63
|
+
return list(bundle.manifest(bundle_dir).get("sources") or {})
|
|
64
64
|
|
|
65
65
|
|
|
66
66
|
def shared_docs(home: Path) -> Iterator[dict]:
|
|
67
|
-
"""Runs imported from other organizations (
|
|
67
|
+
"""Runs imported from other organizations (`share.import_.shared_runs`)."""
|
|
68
68
|
from ..share.import_ import shared_runs
|
|
69
69
|
|
|
70
70
|
yield from shared_runs(home)
|
|
@@ -116,7 +116,7 @@ def benchmark_factors(path: Path | None = None, *, weight: float | None = None)
|
|
|
116
116
|
|
|
117
117
|
`weight` (config `benchmark_prior_weight`) overrides every benchmark's own weight; None reads
|
|
118
118
|
each benchmark's `weight`. A model's k results on one benchmark share its weight: each factor
|
|
119
|
-
has weight `w_b / k
|
|
119
|
+
has weight `w_b / k`. A benchmark with weight 0 or less gives no factors.
|
|
120
120
|
"""
|
|
121
121
|
path = path or benchmark_path()
|
|
122
122
|
if not path.is_file():
|
|
@@ -154,7 +154,7 @@ def benchmark_factors(path: Path | None = None, *, weight: float | None = None)
|
|
|
154
154
|
elif kind in ("cost", "tokens") and value > 0 and ref_value > 0: # Tokens feed the tokens head only
|
|
155
155
|
by_model.setdefault(model, []).append((kind, terms, math.log(value / ref_value), r))
|
|
156
156
|
for model, items in by_model.items():
|
|
157
|
-
w = w_b / len(items) # one budget per model and benchmark
|
|
157
|
+
w = w_b / len(items) # one budget per model and benchmark
|
|
158
158
|
share = f"weight {w_b:g} over {len(items)}" if len(items) > 1 else f"weight {w_b:g}"
|
|
159
159
|
for head, terms, mean, r in items:
|
|
160
160
|
note = (f"{bid}: {model} {float(r['value'])} against {ref} {ref_value} ({share}; "
|
loopmath/builder/__init__.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"""`loopmath builder`: the workflow builder
|
|
1
|
+
"""`loopmath builder`: the workflow builder, a local page over the fitted model (spec 02, `builder`)."""
|
loopmath/builder/context.py
CHANGED
|
@@ -66,8 +66,8 @@ def rec_task(stored: dict[str, Any], args: argparse.Namespace) -> tuple[Task, bo
|
|
|
66
66
|
|
|
67
67
|
def asked_task(task: Task, id_given: bool, belief: Any, rule: AcceptanceRule, configs: list) -> Task:
|
|
68
68
|
"""The task as `recommend` asks the belief: the user's when they named it, else `commands.question_task`
|
|
69
|
-
keyed as `recommend` keys it, by `commands.question_key(belief)` once `recommend` has it (0.2.1
|
|
70
|
-
|
|
69
|
+
keyed as `recommend` keys it, by `commands.question_key(belief)` once `recommend` has it (0.2.1:
|
|
70
|
+
the fit's input seed key, so identical fits agree), else by the fit id as in 0.2.0."""
|
|
71
71
|
if id_given:
|
|
72
72
|
return task
|
|
73
73
|
key_of = getattr(C, "question_key", None)
|
|
@@ -307,7 +307,7 @@ def candidate_entry(rec: Recommendation, c: Any) -> dict[str, Any]:
|
|
|
307
307
|
|
|
308
308
|
|
|
309
309
|
def rescue_entry(session: Session, cfg: Configuration) -> dict[str, Any]:
|
|
310
|
-
"""The rescue workflow as a candidate when the search did not offer it (
|
|
310
|
+
"""The rescue workflow as a candidate when the search did not offer it (0.2.3): predicted as a
|
|
311
311
|
candidate is, under the recommendation's task, rule and rescue, with origin `rescue`."""
|
|
312
312
|
rec = session.rec
|
|
313
313
|
bands: dict[str, Any] = {}
|
|
@@ -334,7 +334,7 @@ def context_payload(session: Session) -> dict[str, Any]:
|
|
|
334
334
|
offer = [c for c in rec.candidates if c.config.id not in retired]
|
|
335
335
|
top = offer[:TOP_CANDIDATES]
|
|
336
336
|
ids = {c.config.id for c in top}
|
|
337
|
-
# every option's workflows (the pair's exploration one too) and the rescue workflow, whatever the cutoff
|
|
337
|
+
# every option's workflows (the pair's exploration one too) and the rescue workflow, whatever the cutoff
|
|
338
338
|
wanted = {rec.usual.config.id, *(i for ch in core["choices"]
|
|
339
339
|
for i in (ch["config"], *ch.get("members", ()), ch.get("explore_config")) if i)}
|
|
340
340
|
if rec.rescue_config is not None:
|
loopmath/builder/predict.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
A posted configuration is checked (shape, unknown model, width below 1, a gate whose `on_fail` names no
|
|
4
4
|
piece, a piece without a setting) and gets its id as loopmath computes it (`workflows.ids.config_id`, via
|
|
5
|
-
|
|
5
|
+
`configuration_from_any`), whatever id it was sent with. It is predicted with the fit's
|
|
6
6
|
`predict_many` path under the recommendation's task, rule and rescue (`engine.predict_with_medians`, which
|
|
7
7
|
also gives the run cost median the numbers show) and formatted by `Recommendation.numbers()`, so a
|
|
8
8
|
candidate's numbers here equal its numbers in `recommend --json`. Answers are cached by config id.
|
|
@@ -26,7 +26,7 @@ from .context import Session, known_models, numbers_of, with_numbers
|
|
|
26
26
|
CACHE_SIZE = 512
|
|
27
27
|
MAX_MANY = 200 # configurations in one `predict_many` body
|
|
28
28
|
PIECE_KEYS = ("id", "role")
|
|
29
|
-
# 0.2.1
|
|
29
|
+
# 0.2.1: `predict_with_medians` fills `bands` by id and `numbers()` reads `rec.bands`
|
|
30
30
|
_BANDS = "bands" in inspect.signature(engine.predict_with_medians).parameters
|
|
31
31
|
|
|
32
32
|
|
|
@@ -136,7 +136,7 @@ def clean(obj: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
|
136
136
|
return {**obj, "id": None, "settings": settings}, notes
|
|
137
137
|
|
|
138
138
|
|
|
139
|
-
#
|
|
139
|
+
# 0.2.4: a review gate that names a repairer at a round limit of 1 is one of loopmath's own shapes (reviewed
|
|
140
140
|
# once; the search builds it and may recommend it). The workflow file check warns about it; the builder page instead
|
|
141
141
|
# says on the gate what 1 round means, so that warning is not shown here.
|
|
142
142
|
REVIEW_ONCE = "budget_rounds is 1"
|
|
@@ -154,7 +154,7 @@ def parse_config(obj: Any, models: dict[str, int]
|
|
|
154
154
|
data, warnings = clean(obj)
|
|
155
155
|
try:
|
|
156
156
|
cfg = configuration_from_any(data)
|
|
157
|
-
except Exception as exc: # noqa: BLE001 -
|
|
157
|
+
except Exception as exc: # noqa: BLE001 - the workflows decoder raises several kinds; the page gets the words
|
|
158
158
|
return None, [err(None, f"config: {exc}")], []
|
|
159
159
|
errors = notes_of(validate_workflow(cfg.workflow) + validate_settings(cfg.workflow, cfg.settings), cfg)
|
|
160
160
|
for p in cfg.workflow.pieces:
|
|
@@ -251,7 +251,7 @@ def failed(cfg: Configuration, exc: Exception) -> dict[str, Any]:
|
|
|
251
251
|
|
|
252
252
|
def answer_each(session: Session, cfgs: list[Configuration]) -> list[dict[str, Any]]:
|
|
253
253
|
"""`answer_all` in one engine call; when that call fails, one call per configuration, so the configuration
|
|
254
|
-
that fails gets its own error and the others their answers, however many were cached
|
|
254
|
+
that fails gets its own error and the others their answers, however many were cached."""
|
|
255
255
|
try:
|
|
256
256
|
return answer_all(session, cfgs)
|
|
257
257
|
except Exception as exc: # noqa: BLE001 - retried one by one below
|
loopmath/cli.py
CHANGED
|
@@ -184,8 +184,8 @@ def analyze(args: argparse.Namespace) -> int:
|
|
|
184
184
|
seed=args.seed,
|
|
185
185
|
ci=args.ci,
|
|
186
186
|
)
|
|
187
|
-
result["ratio_pair"] = _extremes_ratio(df, result, price_mod)
|
|
188
187
|
result["min_n"] = args.min_n
|
|
188
|
+
result["ratio_pair"] = _extremes_ratio(df, result, price_mod)
|
|
189
189
|
|
|
190
190
|
if getattr(args, "json", False):
|
|
191
191
|
from .output import emit_json
|
loopmath/cli_registry.py
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
"""The 0.1.0 command registry (design/0.1/02-commands.md).
|
|
2
2
|
|
|
3
3
|
Every command and flag of the spec is declared here, and each command points to a
|
|
4
|
-
handler `module:function` in the owning
|
|
5
|
-
when their command runs, so a
|
|
6
|
-
another command.
|
|
4
|
+
handler `module:function` in the owning package. Handlers are imported only
|
|
5
|
+
when their command runs, so a package that fails to import never breaks `--help` or
|
|
6
|
+
another command. A new flag is declared here first.
|
|
7
7
|
"""
|
|
8
8
|
|
|
9
9
|
from __future__ import annotations
|
|
@@ -226,7 +226,8 @@ def _add_recording(sub) -> None:
|
|
|
226
226
|
q.add_argument("--session", action="append", default=[], metavar="[PIECE=]ID", help="a session that worked on the run: a "
|
|
227
227
|
"Claude Code session id, a Codex thread id, or self for this Claude Code session (repeatable); "
|
|
228
228
|
"PIECE= names its piece; run sessions lists the ids. Only the part of a session inside the window "
|
|
229
|
-
"counts
|
|
229
|
+
"counts; a Codex session counts the usage it logged inside the window: each reading's increase at "
|
|
230
|
+
"the reading's time, a reading on either edge included")
|
|
230
231
|
q.add_argument("--cwd", action="append", default=[], metavar="[PIECE=]PATH", help="a folder an agent with no session id "
|
|
231
232
|
"worked in; its log is matched at finish by folder and time (repeatable)")
|
|
232
233
|
q.add_argument("--since", default=None, metavar="TS", help="when the work began (e.g. 2h or a local ISO time); with "
|
|
@@ -374,7 +375,7 @@ def _add_viewing(sub) -> None:
|
|
|
374
375
|
p.add_argument("run_id", nargs="?", default=None, metavar="RUN", help="show this one run in full, as --run RUN")
|
|
375
376
|
p.add_argument("--type", dest="task_type", default=None, metavar="T", help="only this task type (see loopmath task-types)")
|
|
376
377
|
p.add_argument("--repo", default=None, metavar="R", help="only this repository")
|
|
377
|
-
p.add_argument("--since", default=None, metavar="D", help="only runs started since then: 90d, 2w, 12h, or a date or time such as 2026-
|
|
378
|
+
p.add_argument("--since", default=None, metavar="D", help="only runs started since then: 90d, 2w, 12h, or a date or time such as 2026-01-15 or 2026-01-15T09:30 (local time)")
|
|
378
379
|
p.add_argument("--slate", default=None, metavar="SLT", help="only this slate")
|
|
379
380
|
p.add_argument("--run", default=None, metavar="RUN", help="show this one run in full; RUN is an id from the run column of loopmath runs")
|
|
380
381
|
p.add_argument("--wide", action="store_true", help="also show each run's cfg_ configuration id")
|
|
@@ -460,7 +461,7 @@ def register(sub) -> None:
|
|
|
460
461
|
_add_recording(sub)
|
|
461
462
|
_add_learning(sub)
|
|
462
463
|
_add_viewing(sub)
|
|
463
|
-
importlib.import_module("loopmath.builder.cli").add(sub) # `builder` (0.2.1
|
|
464
|
+
importlib.import_module("loopmath.builder.cli").add(sub) # `builder` (0.2.1)
|
|
464
465
|
|
|
465
466
|
|
|
466
467
|
# Handler targets, for the registry test: every one must import and be callable.
|
loopmath/cli_support.py
CHANGED
|
@@ -70,22 +70,16 @@ def _extremes_ratio(df, result, price_mod) -> dict:
|
|
|
70
70
|
and mixing an adjusted dollar figure with an unadjusted token figure would
|
|
71
71
|
make the ratio of the two meaningless.
|
|
72
72
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
two halves could rest on DIFFERENT
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
token sum, the dollar sum, AND the accepted-run count from that one same
|
|
84
|
-
restricted set, so numerator and denominator always describe the same
|
|
85
|
-
runs.
|
|
86
|
-
|
|
87
|
-
The reviewer asked instead for the whole comparison to be withheld
|
|
88
|
-
whenever a single value anywhere is missing. That is NOT what this does:
|
|
73
|
+
Each configuration is restricted to the rows where BOTH `total_tokens`
|
|
74
|
+
and `usd` are known, and the token sum, the dollar sum AND the
|
|
75
|
+
accepted-run count all come from that one restricted set, so numerator
|
|
76
|
+
and denominator always describe the same runs. Summing each stream over
|
|
77
|
+
its own known values (`.sum(skipna=True)`) against a count of EVERY
|
|
78
|
+
accepted run would understate both ratios for a configuration with some
|
|
79
|
+
unpriced or untokenized runs, and the two halves could rest on DIFFERENT
|
|
80
|
+
subsets, making their ratio (the amplifier itself) meaningless.
|
|
81
|
+
|
|
82
|
+
The comparison is not withheld when a single value anywhere is missing:
|
|
89
83
|
the amplifier is the headline finding of the report, and dropping it
|
|
90
84
|
entirely because one run out of sixty lacks a price would hide more than
|
|
91
85
|
it protects. Disclosing the exact basis is the honest middle ground: when
|
|
@@ -93,26 +87,15 @@ def _extremes_ratio(df, result, price_mod) -> dict:
|
|
|
93
87
|
known-outcome row count, `basis_note` on the returned dict says exactly
|
|
94
88
|
how many runs the numbers rest on and why, for both configurations if
|
|
95
89
|
they differ, and naming neither when nothing was dropped (measured
|
|
96
|
-
against
|
|
97
|
-
real run -- see the
|
|
98
|
-
now and why it changed).
|
|
90
|
+
against real data: nothing is, so this key is not set in the
|
|
91
|
+
real run -- see the note above `drops` for what "dropped" means).
|
|
99
92
|
"""
|
|
100
93
|
table = result.get("table")
|
|
101
94
|
n_configs = 0 if table is None else len(table)
|
|
102
95
|
if n_configs < 2:
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
"no workflow configuration met the minimum run count, so there "
|
|
107
|
-
"is nothing to compare"
|
|
108
|
-
)
|
|
109
|
-
}
|
|
110
|
-
return {
|
|
111
|
-
"unavailable": (
|
|
112
|
-
"only one workflow configuration met the minimum run count, so "
|
|
113
|
-
"there is nothing to compare"
|
|
114
|
-
)
|
|
115
|
-
}
|
|
96
|
+
from .report.eligibility import unavailable_lines
|
|
97
|
+
|
|
98
|
+
return {"unavailable": "; ".join(unavailable_lines(result, n_configs))}
|
|
116
99
|
ordered = table.sort_values("cost_per_accepted")
|
|
117
100
|
cheap = ordered.iloc[0]["arm"]
|
|
118
101
|
dear = ordered.iloc[-1]["arm"]
|
|
@@ -124,19 +107,17 @@ def _extremes_ratio(df, result, price_mod) -> dict:
|
|
|
124
107
|
# both-known restriction actually dropped a ROW, in `(dear, cheap)`
|
|
125
108
|
# order; feeds `basis_note` below.
|
|
126
109
|
#
|
|
127
|
-
#
|
|
128
|
-
#
|
|
129
|
-
#
|
|
110
|
+
# Whether anything was dropped is decided on ROW counts (`len(sub)`
|
|
111
|
+
# against `len(sub_all)`), not on ACCEPTED counts (`n_acc` against
|
|
112
|
+
# `n_acc_all`). The numerator (`tokens`, `usd` below) is a sum over every
|
|
130
113
|
# row in `sub` -- accepted AND rejected alike, per the comment on
|
|
131
|
-
# `sub_all
|
|
132
|
-
#
|
|
133
|
-
#
|
|
134
|
-
#
|
|
135
|
-
#
|
|
136
|
-
#
|
|
137
|
-
#
|
|
138
|
-
# the actual population the numerator sums over, and wording the note as
|
|
139
|
-
# runs actually used out of runs considered rather than "accepted" runs.
|
|
114
|
+
# `sub_all`. A REJECTED row that has a known acceptance outcome but is
|
|
115
|
+
# missing its token count or dollar cost is dropped from `sub` by the
|
|
116
|
+
# both-known filter below, shrinking the numerator, yet it is in neither
|
|
117
|
+
# accepted count, so comparing accepted counts would never fire
|
|
118
|
+
# `basis_note` and the understated spend would go unreported. The note
|
|
119
|
+
# is worded as runs actually used out of runs considered, not "accepted"
|
|
120
|
+
# runs.
|
|
140
121
|
drops: list[tuple[str, int, int]] = []
|
|
141
122
|
for arm in (dear, cheap):
|
|
142
123
|
# `sub_all`: every row for this configuration with a KNOWN acceptance
|
|
@@ -155,7 +136,7 @@ def _extremes_ratio(df, result, price_mod) -> dict:
|
|
|
155
136
|
)
|
|
156
137
|
}
|
|
157
138
|
|
|
158
|
-
#
|
|
139
|
+
# Restrict to rows where BOTH streams are known, then take the
|
|
159
140
|
# token sum, the dollar sum, AND the accepted count from this SAME
|
|
160
141
|
# restricted set -- never a subset for one half and the whole set for
|
|
161
142
|
# the other.
|
|
@@ -220,11 +201,11 @@ def _extremes_ratio(df, result, price_mod) -> dict:
|
|
|
220
201
|
pair["label"] = f"{dear} vs {cheap}"
|
|
221
202
|
pair["basis"] = "per accepted run"
|
|
222
203
|
if drops:
|
|
223
|
-
#
|
|
224
|
-
#
|
|
225
|
-
#
|
|
226
|
-
#
|
|
227
|
-
#
|
|
204
|
+
# Name exactly what was dropped rather than staying silent about it;
|
|
205
|
+
# report both configurations' counts when they differ, one when only
|
|
206
|
+
# one configuration lost any rows. Worded as runs actually used out
|
|
207
|
+
# of runs considered (row counts), not accepted counts -- see the
|
|
208
|
+
# note above `drops` for why that distinction matters.
|
|
228
209
|
pieces = [f"{n_used} of {n_considered} runs considered ({a})" for a, n_used, n_considered in drops]
|
|
229
210
|
pair["basis_note"] = (
|
|
230
211
|
"computed on " + " and ".join(pieces) + " where both the token count "
|