loopmath 0.2.2__py3-none-any.whl → 0.2.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loopmath/__init__.py +1 -1
- loopmath/adapters/atrium.py +1 -1
- loopmath/adapters/bb.py +1 -1
- loopmath/adapters/opencode.py +1 -1
- loopmath/adapters/orca.py +1 -1
- loopmath/adapters/otel_genai.py +1 -1
- loopmath/adapters/paseo.py +1 -1
- loopmath/belief/check_pymc.py +7 -3
- loopmath/belief/commands.py +41 -1
- loopmath/belief/design.py +5 -3
- loopmath/belief/fit.py +6 -4
- loopmath/belief/priors.py +46 -7
- loopmath/belief/state.py +4 -1
- loopmath/builder/context.py +20 -3
- loopmath/builder/predict.py +8 -1
- loopmath/cli.py +73 -30
- loopmath/cli_graph.py +28 -19
- loopmath/cli_registry.py +31 -10
- loopmath/cli_support.py +117 -4
- loopmath/e0/verb.py +16 -8
- loopmath/fit_bayes.py +57 -0
- loopmath/graph/dataset_build.py +2 -2
- loopmath/graph/html_common.py +56 -25
- loopmath/graph/html_css.py +5 -2
- loopmath/graph/html_data.py +35 -12
- loopmath/graph/html_force.py +9 -1
- loopmath/graph/html_render.py +1 -1
- loopmath/graph/html_swimlanes.py +28 -6
- loopmath/logmatch/clip.py +16 -3
- loopmath/logmatch/match.py +94 -7
- loopmath/logmatch/settle.py +7 -6
- loopmath/onboard/commands.py +24 -6
- loopmath/onboard/history.py +48 -6
- loopmath/onboard/usual.py +4 -2
- loopmath/output.py +47 -0
- loopmath/priors/__init__.py +31 -8
- loopmath/priors/benchmarks.py +6 -1
- loopmath/priors/benchmarks.toml +7 -2
- loopmath/priors/build.py +84 -33
- loopmath/priors/bundle/README.md +6 -2
- loopmath/priors/bundle/e0.jsonl.gz +0 -0
- loopmath/priors/bundle/lanes.jsonl.gz +0 -0
- loopmath/priors/bundle/manifest.json +133 -42
- loopmath/priors/bundle/rq1.jsonl.gz +0 -0
- loopmath/priors/bundle/sweep.jsonl.gz +0 -0
- loopmath/priors/commands.py +20 -12
- loopmath/priors/e0.py +8 -3
- loopmath/priors/lanes.py +15 -1
- loopmath/priors/ocpdoc.py +45 -0
- loopmath/priors/registry.py +3 -1
- loopmath/priors/rq1.py +13 -3
- loopmath/priors/show.py +111 -1
- loopmath/priors/sweep.py +34 -7
- loopmath/recommend/commands.py +78 -21
- loopmath/recommend/engine.py +18 -8
- loopmath/recommend/message.py +54 -7
- loopmath/recommend/storeread.py +31 -10
- loopmath/report/terminal.py +28 -3
- loopmath/share/commands.py +45 -5
- loopmath/share/export.py +12 -3
- loopmath/skill/commands.py +34 -7
- loopmath/skill/doctor.py +9 -2
- loopmath/skill/install.py +38 -21
- loopmath/skill/skills/loopmath-import-runs/SKILL.md +2 -2
- loopmath/skill/skills/loopmath-onboard/SKILL.md +3 -1
- loopmath/skill/skills/loopmath-record-run/SKILL.md +2 -2
- loopmath/skill/skills/loopmath-update-fit/SKILL.md +2 -2
- loopmath/skill/skills/reference.md +13 -4
- loopmath/store/budget.py +79 -12
- loopmath/store/commands.py +166 -47
- loopmath/store/config.py +48 -3
- loopmath/store/finish.py +2 -2
- loopmath/store/fitjob.py +56 -11
- loopmath/store/home.py +68 -11
- loopmath/store/ids.py +28 -4
- loopmath/store/record.py +192 -31
- loopmath/store/report.py +18 -14
- loopmath/store/run_names.py +45 -0
- loopmath/store/runs.py +14 -0
- loopmath/store/status.py +42 -10
- loopmath/store/verify.py +106 -0
- loopmath/surface_presentation.py +28 -2
- loopmath/types.py +9 -1
- loopmath/views/assets/builder.css +4 -0
- loopmath/views/assets/builder.js +81 -22
- loopmath/views/assets/common.js +14 -4
- loopmath/views/assets/estimates.js +10 -5
- loopmath/views/assets/graph.js +71 -10
- loopmath/views/assets/plans.css +1 -0
- loopmath/views/assets/plans.js +25 -5
- loopmath/views/assets/results.js +58 -19
- loopmath/views/assets/runs.js +1 -1
- loopmath/views/assets/viz.css +3 -1
- loopmath/views/assets/viz.js +19 -3
- loopmath/views/common.py +25 -1
- loopmath/views/posterior.py +140 -8
- loopmath/views/runs.py +37 -11
- loopmath/workflows/commands.py +14 -4
- loopmath/workflows/shapes.py +15 -0
- {loopmath-0.2.2.dist-info → loopmath-0.2.4.dist-info}/METADATA +39 -24
- {loopmath-0.2.2.dist-info → loopmath-0.2.4.dist-info}/RECORD +108 -106
- {loopmath-0.2.2.data → loopmath-0.2.4.data}/data/spec/ocp-v0.2.schema.json +0 -0
- {loopmath-0.2.2.data → loopmath-0.2.4.data}/data/spec/ocp-v0.schema.json +0 -0
- {loopmath-0.2.2.data → loopmath-0.2.4.data}/data/spec/ocp_conformance.py +0 -0
- {loopmath-0.2.2.dist-info → loopmath-0.2.4.dist-info}/WHEEL +0 -0
- {loopmath-0.2.2.dist-info → loopmath-0.2.4.dist-info}/entry_points.txt +0 -0
- {loopmath-0.2.2.dist-info → loopmath-0.2.4.dist-info}/licenses/LICENSE +0 -0
- {loopmath-0.2.2.dist-info → loopmath-0.2.4.dist-info}/top_level.txt +0 -0
loopmath/__init__.py
CHANGED
loopmath/adapters/atrium.py
CHANGED
|
@@ -470,7 +470,7 @@ class AtriumAdapter(Adapter):
|
|
|
470
470
|
return {
|
|
471
471
|
"ocp": "0.2",
|
|
472
472
|
"producer": {
|
|
473
|
-
"name": "
|
|
473
|
+
"name": "loopmath/adapter-atrium",
|
|
474
474
|
"framework": "atrium",
|
|
475
475
|
"source_contract": "atrium/timeline+task-run-segments",
|
|
476
476
|
"capabilities": {
|
loopmath/adapters/bb.py
CHANGED
|
@@ -408,7 +408,7 @@ class BbAdapter(Adapter):
|
|
|
408
408
|
)
|
|
409
409
|
ignored_snapshots = sum(thread["ignored_token_snapshots"] for thread in selected)
|
|
410
410
|
producer: dict[str, Any] = {
|
|
411
|
-
"name": "
|
|
411
|
+
"name": "loopmath/adapter-bb",
|
|
412
412
|
"framework": "bb",
|
|
413
413
|
"source_contract": "bb/sqlite",
|
|
414
414
|
"capabilities": {
|
loopmath/adapters/opencode.py
CHANGED
|
@@ -386,7 +386,7 @@ def _emit_document(
|
|
|
386
386
|
else:
|
|
387
387
|
source_contract = "opencode export"
|
|
388
388
|
producer: dict[str, Any] = {
|
|
389
|
-
"name": "
|
|
389
|
+
"name": "loopmath/adapter-opencode",
|
|
390
390
|
"framework": "opencode",
|
|
391
391
|
"source_contract": source_contract,
|
|
392
392
|
"capabilities": {
|
loopmath/adapters/orca.py
CHANGED
loopmath/adapters/otel_genai.py
CHANGED
|
@@ -383,7 +383,7 @@ class OtelGenAIAdapter(Adapter):
|
|
|
383
383
|
return {
|
|
384
384
|
"ocp": "0.2",
|
|
385
385
|
"producer": {
|
|
386
|
-
"name": "
|
|
386
|
+
"name": "loopmath/adapter-otel-genai",
|
|
387
387
|
"framework": "+".join(harnesses) if harnesses else "unknown",
|
|
388
388
|
"source_contract": "OTLP/JSON file exporter",
|
|
389
389
|
"capabilities": {
|
loopmath/adapters/paseo.py
CHANGED
loopmath/belief/check_pymc.py
CHANGED
|
@@ -21,9 +21,13 @@ from __future__ import annotations
|
|
|
21
21
|
import time
|
|
22
22
|
|
|
23
23
|
import numpy as np
|
|
24
|
-
|
|
25
|
-
import
|
|
26
|
-
|
|
24
|
+
|
|
25
|
+
from ..fit_bayes import use_store_compiledir
|
|
26
|
+
|
|
27
|
+
use_store_compiledir() # before pytensor is imported: its compile folder goes in the store's cache
|
|
28
|
+
import pymc as pm # noqa: E402
|
|
29
|
+
import pytensor # noqa: E402
|
|
30
|
+
import pytensor.tensor as pt # noqa: E402
|
|
27
31
|
|
|
28
32
|
from .gaussian import HYPER_SD, SIGMA_PRIOR_SD, GaussianHead
|
|
29
33
|
|
loopmath/belief/commands.py
CHANGED
|
@@ -121,6 +121,36 @@ def _print_summary(s: dict) -> None:
|
|
|
121
121
|
f"(table in pymc_check.json)")
|
|
122
122
|
|
|
123
123
|
|
|
124
|
+
def _in_pymc_check(exc: BaseException) -> bool:
|
|
125
|
+
"""Whether `exc` was raised inside the `--full` PyMC check (check_pymc, pymc or pytensor)."""
|
|
126
|
+
tb = exc.__traceback__
|
|
127
|
+
while tb is not None:
|
|
128
|
+
if tb.tb_frame.f_globals.get("__name__") == "loopmath.belief.check_pymc":
|
|
129
|
+
return True
|
|
130
|
+
tb = tb.tb_next
|
|
131
|
+
return False
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _pymc_check_failed(home: Path, exc: BaseException) -> int:
|
|
135
|
+
"""`fit --full` whose PyMC check failed (a compiler error, say): remove the unfinished fit
|
|
136
|
+
folder and say what happened in one line, without a traceback."""
|
|
137
|
+
from .fit import FitBusy, fit_lock, remove_partials
|
|
138
|
+
|
|
139
|
+
try:
|
|
140
|
+
with fit_lock(home, LOCK_WAIT_S): # under the lock every .partial is from a fit that stopped
|
|
141
|
+
remove_partials(home)
|
|
142
|
+
except FitBusy:
|
|
143
|
+
pass # the next fit removes it
|
|
144
|
+
lines = [ln.strip() for ln in str(exc).splitlines() if ln.strip()]
|
|
145
|
+
said = ([ln for ln in lines if ln.startswith("ld:")] # the linker's reason, then the first compiler error
|
|
146
|
+
or [ln for ln in lines if ln.startswith("error:") or " error: " in ln])
|
|
147
|
+
detail = (said[0] if said else (lines or [type(exc).__name__])[-1])[:200]
|
|
148
|
+
kind = "could not compile its model on this machine" if type(exc).__name__ == "CompileError" else "stopped"
|
|
149
|
+
return output.fail(f"fit --full: the PyMC check {kind} ({type(exc).__name__}: {detail}). No fit was "
|
|
150
|
+
"written and fits/latest did not move; loopmath fit without --full fits as usual.",
|
|
151
|
+
output.EXIT_USER)
|
|
152
|
+
|
|
153
|
+
|
|
124
154
|
def fit(args: argparse.Namespace) -> int:
|
|
125
155
|
home = output.home(getattr(args, "home", None))
|
|
126
156
|
without = tuple(getattr(args, "without", None) or ())
|
|
@@ -140,10 +170,16 @@ def fit(args: argparse.Namespace) -> int:
|
|
|
140
170
|
else:
|
|
141
171
|
print(f"fit queued: {started.get('reason', 'a fit is running')}")
|
|
142
172
|
return output.EXIT_OK
|
|
173
|
+
import contextlib
|
|
174
|
+
import sys
|
|
175
|
+
|
|
143
176
|
from .fit import FitBusy, NothingToFit, UnknownSource, fit as run_fit
|
|
144
177
|
|
|
178
|
+
# --full: a failed compile makes pytensor print where it left the C code on stdout, which is for the result
|
|
179
|
+
quiet = contextlib.redirect_stdout(sys.stderr) if full else contextlib.nullcontext()
|
|
145
180
|
try:
|
|
146
|
-
|
|
181
|
+
with quiet:
|
|
182
|
+
path = run_fit(home, no_prior=no_prior, without=without, full=full, wait_s=LOCK_WAIT_S)
|
|
147
183
|
except FitBusy:
|
|
148
184
|
return output.fail(f"another fit has held {home / 'fits'} for more than {int(LOCK_WAIT_S)} s; "
|
|
149
185
|
"try again later or use --background", output.EXIT_LOCKED)
|
|
@@ -151,6 +187,10 @@ def fit(args: argparse.Namespace) -> int:
|
|
|
151
187
|
return output.fail(str(exc), output.EXIT_NOT_FOUND)
|
|
152
188
|
except NothingToFit as exc:
|
|
153
189
|
return output.fail(str(exc), output.EXIT_USER)
|
|
190
|
+
except Exception as exc:
|
|
191
|
+
if not (full and _in_pymc_check(exc)):
|
|
192
|
+
raise
|
|
193
|
+
return _pymc_check_failed(home, exc)
|
|
154
194
|
summary = _summary(path)
|
|
155
195
|
if as_json:
|
|
156
196
|
output.emit_json(SCHEMA, summary)
|
loopmath/belief/design.py
CHANGED
|
@@ -464,10 +464,12 @@ def source_label(doc: dict) -> str:
|
|
|
464
464
|
ext = run.get("ext") or {}
|
|
465
465
|
share = ext.get("dev.loopmath.share")
|
|
466
466
|
if isinstance(share, dict):
|
|
467
|
-
# the importer writes `source` and `task.org` as `shared:<org_hash>` already: never prefix twice
|
|
467
|
+
# the importer writes `source` and `task.org` as `shared:<org_hash>` already: never prefix twice. With no
|
|
468
|
+
# org at all the source is `shared`, the name `fit --without shared` takes (0.2.3, 22X review N1)
|
|
468
469
|
org = (share.get("source") or share.get("org") or share.get("org_hash") or (run.get("task") or {}).get("org")
|
|
469
|
-
or "
|
|
470
|
-
|
|
470
|
+
or "")
|
|
471
|
+
name = str(org).removeprefix("shared:")
|
|
472
|
+
return f"shared:{name}" if name and name != "shared" else "shared"
|
|
471
473
|
src = (run.get("task") or {}).get("source")
|
|
472
474
|
kind = src.get("kind") if isinstance(src, dict) else src
|
|
473
475
|
kind = str(kind or "").strip().lower()
|
loopmath/belief/fit.py
CHANGED
|
@@ -72,7 +72,7 @@ REASON_NOTES = {
|
|
|
72
72
|
"attempt without usable cost": "crashed or no usage reported",
|
|
73
73
|
"attempt outside the workflow": "a model attempt at no piece of the run's workflow",
|
|
74
74
|
"duplicate run id": "the same run found twice",
|
|
75
|
-
SHIPPED_COPY: "
|
|
75
|
+
SHIPPED_COPY: "this run ships with loopmath and is also in your store, so it counts once, as yours",
|
|
76
76
|
"no configuration": "the run records no workflow and settings",
|
|
77
77
|
"workflow ref not resolved": "a workflow this version does not know",
|
|
78
78
|
"workflow has no pieces": "an empty workflow",
|
|
@@ -526,7 +526,8 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
526
526
|
with fit_lock(home, wait_s):
|
|
527
527
|
remove_partials(home)
|
|
528
528
|
config = _read_config(home)
|
|
529
|
-
|
|
529
|
+
raw_weight = config.get("benchmark_prior_weight") # unset: each benchmark's own weight (spec 04 section 5)
|
|
530
|
+
weight = None if raw_weight is None else float(raw_weight)
|
|
530
531
|
try:
|
|
531
532
|
features, features_error = FeatureSet.from_config(config.get("features")), None
|
|
532
533
|
except FeatureConfigError as exc: # a hand-edited config.toml: fit with the built-ins, say why
|
|
@@ -550,9 +551,10 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
550
551
|
raise UnknownSource(unknown_source_message(unknown, known))
|
|
551
552
|
if not any(hr.rows for hr in heads_rows.values()):
|
|
552
553
|
raise NothingToFit(_nothing_to_fit(no_prior, without, dropped))
|
|
553
|
-
specs = []
|
|
554
|
+
specs, bench_weights = [], {}
|
|
554
555
|
if not no_prior and "benchmark" not in without:
|
|
555
556
|
specs = prior_data.benchmark_factors(benchmarks, weight=weight)
|
|
557
|
+
bench_weights = prior_data.benchmark_weights(benchmarks, weight=weight)
|
|
556
558
|
seed_key = input_key(heads_rows, specs, {"no_prior": no_prior, "without": sorted(without), "eb": eb,
|
|
557
559
|
"benchmark_prior_weight": weight})
|
|
558
560
|
forest = Forest()
|
|
@@ -604,7 +606,7 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
604
606
|
"seed_key": seed_key, "timebox_effort": design_rows.TIMEBOX_EFFORT_COST,
|
|
605
607
|
"timebox_terms": list(design_rows.TIMEBOX_LEVELS),
|
|
606
608
|
"options": {"no_prior": no_prior, "without": list(without), "full": full, "eb": eb,
|
|
607
|
-
"benchmark_prior_weight": weight},
|
|
609
|
+
"benchmark_prior_weight": weight, "benchmark_weights": bench_weights},
|
|
608
610
|
"runs_by_source": runs_by_source,
|
|
609
611
|
"n_runs": {"prior": sum(v for k, v in runs_by_source.items() if k != "user"),
|
|
610
612
|
"user": runs_by_source.get("user", 0)},
|
loopmath/belief/priors.py
CHANGED
|
@@ -10,8 +10,11 @@
|
|
|
10
10
|
version nodes: success head, the logit gap to the benchmark's reference model with variance
|
|
11
11
|
`1 / (w p (1 - p))`; cost head, the log cost ratio with variance `1 / w`; tokens head
|
|
12
12
|
(`kind = "tokens"`, published total tokens for the evaluation), the log token ratio
|
|
13
|
-
with variance `1 / w`. `w` is
|
|
14
|
-
|
|
13
|
+
with variance `1 / w`. `w` is one model's share of the benchmark's weight `w_b` (0.2.3,
|
|
14
|
+
lane 23K): the `weight` of its `[[benchmark]]` table (default 5, about 5 runs), shared by
|
|
15
|
+
the model's k results on that benchmark, so each gets `w_b / k` and a model with five
|
|
16
|
+
efforts pulls as hard in total as a model with one. Config `benchmark_prior_weight`, when
|
|
17
|
+
set, is one `w_b` for every benchmark; 0 turns the factors off.
|
|
15
18
|
"""
|
|
16
19
|
|
|
17
20
|
from __future__ import annotations
|
|
@@ -91,8 +94,30 @@ def _model_terms(model: str, effort: str | None, sign: float) -> list[tuple[str,
|
|
|
91
94
|
return terms
|
|
92
95
|
|
|
93
96
|
|
|
94
|
-
def
|
|
95
|
-
"""
|
|
97
|
+
def benchmark_weights(path: Path | None = None, *, weight: float | None = None) -> dict[str, float]:
|
|
98
|
+
"""Each benchmark's weight `w_b`: `weight` for every benchmark when it is given (config
|
|
99
|
+
`benchmark_prior_weight`), else the benchmark's own `weight` field, else 5. Empty for a missing file."""
|
|
100
|
+
path = path or benchmark_path()
|
|
101
|
+
if not path.is_file():
|
|
102
|
+
return {}
|
|
103
|
+
data = tomllib.loads(path.read_text(encoding="utf-8"))
|
|
104
|
+
return {str(b["id"]): _weight_of(b, weight) for b in data.get("benchmark") or [] if b.get("id")}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _weight_of(bench: dict, weight: float | None) -> float:
|
|
108
|
+
if weight is not None:
|
|
109
|
+
return float(weight)
|
|
110
|
+
own = bench.get("weight")
|
|
111
|
+
return float(own) if isinstance(own, (int, float)) and not isinstance(own, bool) else BENCHMARK_PRIOR_WEIGHT
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def benchmark_factors(path: Path | None = None, *, weight: float | None = None) -> list[FactorSpec]:
|
|
115
|
+
"""Prior factors from `benchmarks.toml`. A missing file gives no factors.
|
|
116
|
+
|
|
117
|
+
`weight` (config `benchmark_prior_weight`) overrides every benchmark's own weight; None reads
|
|
118
|
+
each benchmark's `weight`. A model's k results on one benchmark share its weight: each factor
|
|
119
|
+
has weight `w_b / k` (B1). A benchmark with weight 0 or less gives no factors.
|
|
120
|
+
"""
|
|
96
121
|
path = path or benchmark_path()
|
|
97
122
|
if not path.is_file():
|
|
98
123
|
return []
|
|
@@ -103,11 +128,15 @@ def benchmark_factors(path: Path | None = None, *, weight: float = BENCHMARK_PRI
|
|
|
103
128
|
for bid, bench in benches.items():
|
|
104
129
|
kind = str(bench.get("kind") or "success")
|
|
105
130
|
ref = canonical_model_id(str(bench.get("reference") or ""))
|
|
131
|
+
w_b = _weight_of(bench, weight)
|
|
132
|
+
if w_b <= 0:
|
|
133
|
+
continue
|
|
106
134
|
rows = [r for r in results if r["benchmark"] == bid]
|
|
107
135
|
ref_rows = [r for r in rows if canonical_model_id(str(r["model"])) == ref]
|
|
108
136
|
if not ref_rows:
|
|
109
137
|
continue
|
|
110
138
|
ref_value = float(ref_rows[0]["value"])
|
|
139
|
+
by_model: dict[str, list[tuple[str, list, float, dict]]] = {} # model -> (head, terms, mean, row) per result
|
|
111
140
|
for r in rows:
|
|
112
141
|
model = canonical_model_id(str(r["model"]))
|
|
113
142
|
if model == ref:
|
|
@@ -117,14 +146,24 @@ def benchmark_factors(path: Path | None = None, *, weight: float = BENCHMARK_PRI
|
|
|
117
146
|
terms = _cancel(terms)
|
|
118
147
|
if not terms:
|
|
119
148
|
continue
|
|
120
|
-
note = f"{bid}: {model} {value} against {ref} {ref_value} ({r.get('url', '')}, {r.get('date', '')})"
|
|
121
149
|
if kind == "success":
|
|
122
150
|
p = min(max(value, 0.01), 0.99)
|
|
123
151
|
p_ref = min(max(ref_value, 0.01), 0.99)
|
|
124
152
|
gap = math.log(p / (1 - p)) - math.log(p_ref / (1 - p_ref))
|
|
125
|
-
|
|
153
|
+
by_model.setdefault(model, []).append(("success", terms, gap, r))
|
|
126
154
|
elif kind in ("cost", "tokens") and value > 0 and ref_value > 0: # Tokens feed the tokens head only
|
|
127
|
-
|
|
155
|
+
by_model.setdefault(model, []).append((kind, terms, math.log(value / ref_value), r))
|
|
156
|
+
for model, items in by_model.items():
|
|
157
|
+
w = w_b / len(items) # one budget per model and benchmark (B1)
|
|
158
|
+
share = f"weight {w_b:g} over {len(items)}" if len(items) > 1 else f"weight {w_b:g}"
|
|
159
|
+
for head, terms, mean, r in items:
|
|
160
|
+
note = (f"{bid}: {model} {float(r['value'])} against {ref} {ref_value} ({share}; "
|
|
161
|
+
f"{r.get('url', '')}, {r.get('date', '')})")
|
|
162
|
+
if head == "success":
|
|
163
|
+
p = min(max(float(r["value"]), 0.01), 0.99)
|
|
164
|
+
out.append(FactorSpec("success", terms, mean, 1.0 / (w * p * (1 - p)), note))
|
|
165
|
+
else:
|
|
166
|
+
out.append(FactorSpec(head, terms, mean, 1.0 / w, note))
|
|
128
167
|
return out
|
|
129
168
|
|
|
130
169
|
|
loopmath/belief/state.py
CHANGED
|
@@ -792,10 +792,13 @@ class FitState:
|
|
|
792
792
|
gate_pass=res[hg] if hg is not None else None, rounds=res[hr],
|
|
793
793
|
cost_per_round=Money(usd=res[hpr[0]], tokens=res[hpr[1]]) if hpr else None)
|
|
794
794
|
for piece, (hu, ht, hg, hr, hpr) in handles.items()}
|
|
795
|
+
# The typical run (0.2.3): the median of the same simulated runs, one per draw, the 80% range is read from
|
|
796
|
+
run_usd = dataclasses.replace(res[h_run[1]], median=float(np.median(rd.sim_usd)))
|
|
797
|
+
run_tokens = dataclasses.replace(res[h_run[2]], median=float(np.median(rd.sim_tokens)))
|
|
795
798
|
pred = Prediction(
|
|
796
799
|
config=cfg.id,
|
|
797
800
|
p_success=res[h_run[0]],
|
|
798
|
-
cost=Money(usd=
|
|
801
|
+
cost=Money(usd=run_usd, tokens=run_tokens),
|
|
799
802
|
ell=Money(usd=res[h_run[3]], tokens=res[h_run[4]]),
|
|
800
803
|
rounds=res[h_run[5]],
|
|
801
804
|
per_piece=per_piece,
|
loopmath/builder/context.py
CHANGED
|
@@ -21,6 +21,7 @@ from typing import Any
|
|
|
21
21
|
from ..output import EXIT_NOT_FOUND, EXIT_OK, fail, home as home_dir
|
|
22
22
|
from ..recommend import commands as C
|
|
23
23
|
from ..recommend import engine, storeread
|
|
24
|
+
from ..store import run_names
|
|
24
25
|
from ..recommend.curve import accepted_within
|
|
25
26
|
from ..recommend.engine import Recommendation, r6
|
|
26
27
|
from ..recommend.storeread import Conf
|
|
@@ -137,7 +138,7 @@ def build_session(args: argparse.Namespace) -> tuple[Session | None, int]:
|
|
|
137
138
|
rec = engine.recommend(belief, asked, rule, usual=usual, usual_from=usual_from, configs=configs,
|
|
138
139
|
settings=settings, diff=C.diff_fn(), keep=[*(u.id for u in user),
|
|
139
140
|
*(r.id for r in recorded)],
|
|
140
|
-
**offered_kw(offered))
|
|
141
|
+
own_runs=storeread.own_runs(home), **offered_kw(offered))
|
|
141
142
|
rec.task = task
|
|
142
143
|
start = None
|
|
143
144
|
if getattr(args, "start", None):
|
|
@@ -221,7 +222,7 @@ def recorded_runs(home: Any) -> list[tuple[Configuration, int]]:
|
|
|
221
222
|
run_of[str(cfg)] = str(row["run"])
|
|
222
223
|
out = []
|
|
223
224
|
for cfg_id, n in counts.items():
|
|
224
|
-
doc = storeread.read_json(home / "runs" /
|
|
225
|
+
doc = storeread.read_json(home / "runs" / run_names.file_name(run_of[cfg_id])) if cfg_id in run_of else None
|
|
225
226
|
conf = doc.get("run", {}).get("configuration") if isinstance(doc, dict) else None
|
|
226
227
|
cfg = storeread.config_from_any(conf) if conf else None
|
|
227
228
|
if cfg is not None:
|
|
@@ -305,6 +306,17 @@ def candidate_entry(rec: Recommendation, c: Any) -> dict[str, Any]:
|
|
|
305
306
|
"origin": c.origin, **rec.search_fields(c.config.id)}
|
|
306
307
|
|
|
307
308
|
|
|
309
|
+
def rescue_entry(session: Session, cfg: Configuration) -> dict[str, Any]:
|
|
310
|
+
"""The rescue workflow as a candidate when the search did not offer it (22W N1, 0.2.3): predicted as a
|
|
311
|
+
candidate is, under the recommendation's task, rule and rescue, with origin `rescue`."""
|
|
312
|
+
rec = session.rec
|
|
313
|
+
bands: dict[str, Any] = {}
|
|
314
|
+
preds, medians = engine.predict_with_medians(session.belief, session.task, [cfg], session.rule, rec.rescue, bands)
|
|
315
|
+
view = with_numbers(rec, medians=medians, bands=bands)
|
|
316
|
+
return {"config": cfg.to_dict(), "label": rec.label(cfg), "numbers": numbers_of(view, preds[0]),
|
|
317
|
+
"origin": "rescue", **rec.search_fields(cfg.id)}
|
|
318
|
+
|
|
319
|
+
|
|
308
320
|
def context_payload(session: Session) -> dict[str, Any]:
|
|
309
321
|
"""`GET /api/context` (spec 02, `builder`), built once per session."""
|
|
310
322
|
if session.context is not None:
|
|
@@ -329,6 +341,10 @@ def context_payload(session: Session) -> dict[str, Any]:
|
|
|
329
341
|
wanted.add(rec.rescue_config.id)
|
|
330
342
|
top += [c for c in offer[TOP_CANDIDATES:]
|
|
331
343
|
if c.config.id not in ids and (c.origin in KEEP_ORIGINS or c.config.id in wanted)]
|
|
344
|
+
candidates = [candidate_entry(rec, c) for c in top]
|
|
345
|
+
fix = rec.rescue_config
|
|
346
|
+
if fix is not None and fix.id not in retired and rec.by_id(fix.id) is None:
|
|
347
|
+
candidates.append(rescue_entry(session, fix))
|
|
332
348
|
session.context = {
|
|
333
349
|
"schema": SCHEMA,
|
|
334
350
|
"task": C.task_block(rec.task, session.belief),
|
|
@@ -339,9 +355,10 @@ def context_payload(session: Session) -> dict[str, Any]:
|
|
|
339
355
|
"rescue": core["rescue"],
|
|
340
356
|
"reference": reference,
|
|
341
357
|
"choices": choices,
|
|
342
|
-
"candidates":
|
|
358
|
+
"candidates": candidates,
|
|
343
359
|
"catalog": catalog(session),
|
|
344
360
|
"start": session.start.to_dict() if session.start is not None else None,
|
|
361
|
+
"own_runs": rec.own_runs, # 0.2.3: the user's recorded runs; 0 leads each run cost with the typical run
|
|
345
362
|
}
|
|
346
363
|
return session.context
|
|
347
364
|
|
loopmath/builder/predict.py
CHANGED
|
@@ -136,6 +136,12 @@ def clean(obj: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
|
|
136
136
|
return {**obj, "id": None, "settings": settings}, notes
|
|
137
137
|
|
|
138
138
|
|
|
139
|
+
# P3-21 (0.2.4): a review gate that names a repairer at a round limit of 1 is one of loopmath's own shapes (reviewed
|
|
140
|
+
# once; the search builds it and may recommend it). The workflow file check warns about it; the builder page instead
|
|
141
|
+
# says on the gate what 1 round means, so that warning is not shown here.
|
|
142
|
+
REVIEW_ONCE = "budget_rounds is 1"
|
|
143
|
+
|
|
144
|
+
|
|
139
145
|
def parse_config(obj: Any, models: dict[str, int]
|
|
140
146
|
) -> tuple[Configuration | None, list[dict[str, Any]], list[dict[str, Any]]]:
|
|
141
147
|
"""(configuration, errors, warnings); the configuration is None when it does not read at all."""
|
|
@@ -158,7 +164,8 @@ def parse_config(obj: Any, models: dict[str, int]
|
|
|
158
164
|
errors.append(err(p.id, f"piece {p.id!r}: model {s.model!r} is a retired or unknown model; "
|
|
159
165
|
f"pick one of {known}"))
|
|
160
166
|
if not errors:
|
|
161
|
-
|
|
167
|
+
shown = [w for w in workflow_warnings(cfg.workflow) if REVIEW_ONCE not in w]
|
|
168
|
+
warnings += notes_of(shown + settings_warnings(cfg.settings), cfg)
|
|
162
169
|
return cfg, errors, warnings
|
|
163
170
|
|
|
164
171
|
|
loopmath/cli.py
CHANGED
|
@@ -5,27 +5,26 @@ Verbs:
|
|
|
5
5
|
scan explain where implicit scanning occurs
|
|
6
6
|
adapt emit OCP v0.2 from a registered ADE adapter
|
|
7
7
|
validate-prices check a price table's schema, staleness and todo flags
|
|
8
|
-
analyze-e0 the frozen E0 corpus walkdown
|
|
9
8
|
prices show the active price table
|
|
10
9
|
graph the workflow graph of one or more workspaces (ocp, json, dot, run, html: self-contained interactive HTML viewer)
|
|
11
|
-
research
|
|
12
|
-
verify-receipts
|
|
10
|
+
research research data folders: research fit, research transfer-test (the bayes extra), research analyze-e0 (the e0 extra)
|
|
11
|
+
verify-receipts check that the receipts in your store are whole and scored right (or give a research ledger file)
|
|
13
12
|
|
|
14
13
|
0.1.0 verbs (design/0.1/02-commands.md, registered in cli_registry.py):
|
|
15
|
-
plan: task-types workflows recommend plan
|
|
16
|
-
record: run start|attempt|artifact|finish|
|
|
14
|
+
plan: task-types workflows recommend builder plan
|
|
15
|
+
record: run start|record|import|attempt|artifact|finish|sessions outcome budget config
|
|
17
16
|
learn: fit onboard share
|
|
18
17
|
view: runs posterior status report doctor skill ocp
|
|
19
18
|
|
|
20
|
-
No verb in this CLI
|
|
21
|
-
($LOOPMATH_HOME, default ~/.loopmath, which also
|
|
22
|
-
|
|
19
|
+
No verb in this CLI connects to another machine; builder serves on 127.0.0.1 only.
|
|
20
|
+
Verbs write only to the store ($LOOPMATH_HOME, default ~/.loopmath, which also
|
|
21
|
+
holds the parse cache and the fit --full compile folder), to the skill folders
|
|
22
|
+
skill install names, and to paths the user names.
|
|
23
23
|
"""
|
|
24
24
|
|
|
25
25
|
from __future__ import annotations
|
|
26
26
|
|
|
27
27
|
import argparse
|
|
28
|
-
import importlib.util as _importlib_util
|
|
29
28
|
import sys
|
|
30
29
|
import time
|
|
31
30
|
from types import ModuleType
|
|
@@ -79,6 +78,7 @@ class _CliModule(ModuleType):
|
|
|
79
78
|
sys.modules[__name__].__class__ = _CliModule
|
|
80
79
|
|
|
81
80
|
DEFAULT_SINCE_DAYS = 14.0
|
|
81
|
+
_ANALYZE_SCHEMA = "loopmath.analyze/1"
|
|
82
82
|
|
|
83
83
|
|
|
84
84
|
def _counted_noun(count: int, singular: str, plural: str | None = None) -> str:
|
|
@@ -185,6 +185,18 @@ def analyze(args: argparse.Namespace) -> int:
|
|
|
185
185
|
ci=args.ci,
|
|
186
186
|
)
|
|
187
187
|
result["ratio_pair"] = _extremes_ratio(df, result, price_mod)
|
|
188
|
+
result["min_n"] = args.min_n
|
|
189
|
+
|
|
190
|
+
if getattr(args, "json", False):
|
|
191
|
+
from .output import emit_json
|
|
192
|
+
|
|
193
|
+
grading = list(grade_mod.grading_report(records, coverage)) if args.grading else None
|
|
194
|
+
emit_json(_ANALYZE_SCHEMA, _cli_support.analyze_payload(
|
|
195
|
+
result, diag=diag, coverage=coverage, price_lines=price_lines, grading=grading,
|
|
196
|
+
walkdown=surface_mod.walkdown(result)))
|
|
197
|
+
if not args.quiet:
|
|
198
|
+
print(f"ran in {time.time() - t0:.1f} s", file=sys.stderr)
|
|
199
|
+
return 0
|
|
188
200
|
|
|
189
201
|
print(
|
|
190
202
|
terminal.render(
|
|
@@ -246,7 +258,7 @@ def _add_graph(sub) -> None:
|
|
|
246
258
|
p.add_argument("--workspace", action="append", default=[], help="native workspace name to include (repeatable; required unless --ocp is used)")
|
|
247
259
|
p.add_argument("--ocp", action="append", default=[], metavar="FILE", help="read an OCP document (v0.2 or later) as a graph source (repeatable)")
|
|
248
260
|
p.add_argument("--format", type=_cli_support._graph_format, choices=GRAPH_FORMATS, default="ocp", help="ocp (OCP v0.3, default), json (internal dagr_graph), dot (Graphviz), run (herdr-dagr contract v3 run file), html (self-contained interactive HTML viewer)")
|
|
249
|
-
p.add_argument("--out", default=None, help="write here instead of stdout (
|
|
261
|
+
p.add_argument("--out", default=None, help="write here instead of stdout (any path; missing folders are created; an existing file is replaced)")
|
|
250
262
|
p.add_argument("--privacy", choices=("metadata_only", "full"), default="metadata_only", help="OCP privacy profile (default metadata_only)")
|
|
251
263
|
p.add_argument("--logs", default=None, help="parse this path instead of the default log roots")
|
|
252
264
|
p.add_argument(
|
|
@@ -259,7 +271,8 @@ def _add_graph(sub) -> None:
|
|
|
259
271
|
p.add_argument("--limit", type=int, default=None, help="stop after N files per harness")
|
|
260
272
|
p.add_argument("--no-cache", action="store_true", help="reparse everything, ignoring the cache")
|
|
261
273
|
p.add_argument("--prices", default=None, help="price table to use instead of the packaged one")
|
|
262
|
-
p.add_argument("--quiet", action="store_true", help="suppress progress output on stderr
|
|
274
|
+
p.add_argument("--quiet", action="store_true", help="suppress progress output on stderr")
|
|
275
|
+
p.add_argument("--verbose", action="store_true", help="also print every count of what was skipped, excluded or could not be placed")
|
|
263
276
|
p.set_defaults(func=graph_verb)
|
|
264
277
|
|
|
265
278
|
|
|
@@ -279,7 +292,7 @@ def adapt_verb(args: argparse.Namespace) -> int:
|
|
|
279
292
|
|
|
280
293
|
out_path = None
|
|
281
294
|
if args.out:
|
|
282
|
-
out_path, err = _resolve_out(args.out)
|
|
295
|
+
out_path, err = _resolve_out(args.out, "out.ocp.json")
|
|
283
296
|
if err:
|
|
284
297
|
print(err, file=sys.stderr)
|
|
285
298
|
return 2
|
|
@@ -302,9 +315,18 @@ def adapt_verb(args: argparse.Namespace) -> int:
|
|
|
302
315
|
|
|
303
316
|
adapter = adapter_type()
|
|
304
317
|
adapter.configure_services(default_adapter_services())
|
|
305
|
-
|
|
318
|
+
try:
|
|
319
|
+
doc = adapter.emit(selection)
|
|
320
|
+
except (OSError, ValueError) as exc: # a store path it cannot read: say so, no traceback
|
|
321
|
+
print(f"error: adapt {args.name}: {exc}", file=sys.stderr)
|
|
322
|
+
return 1
|
|
306
323
|
if not isinstance(doc, dict):
|
|
307
324
|
raise TypeError(f"adapter {args.name!r} emit() must return a dict")
|
|
325
|
+
if not doc.get("nodes") and not doc.get("attempts"):
|
|
326
|
+
where = ", ".join(str(p) for p in selection.stores) if selection.stores else "its default location"
|
|
327
|
+
print(f"error: adapt {args.name} found no sessions in {where}; nothing written. Point --store at the "
|
|
328
|
+
f"{args.name} data, or widen --since, --until, --session and --workspace", file=sys.stderr)
|
|
329
|
+
return 1
|
|
308
330
|
from .ocp import version_at_least
|
|
309
331
|
|
|
310
332
|
if not version_at_least(doc, "0.2"):
|
|
@@ -323,7 +345,7 @@ def adapt_verb(args: argparse.Namespace) -> int:
|
|
|
323
345
|
def _add_adapt(sub) -> None:
|
|
324
346
|
p = sub.add_parser("adapt", help="emit OCP v0.2 from an ADE store (loopmath ocp migrate converts it to v0.3)")
|
|
325
347
|
p.add_argument("name", metavar="NAME", help="registered adapter name")
|
|
326
|
-
p.add_argument("--out", default=None, help="write here instead of stdout (
|
|
348
|
+
p.add_argument("--out", default=None, help="write here instead of stdout (any path; missing folders are created; an existing file is replaced)")
|
|
327
349
|
p.add_argument("--store", action="append", default=[], metavar="PATH", help="use this store path instead of discovery (repeatable)")
|
|
328
350
|
p.add_argument("--session", action="append", default=[], metavar="ID", help="include this session id (repeatable; default: all)")
|
|
329
351
|
p.add_argument("--workspace", action="append", default=[], metavar="WORKSPACE", help="include this workspace (repeatable; default: all)")
|
|
@@ -371,6 +393,9 @@ def _add_analyze(sub) -> None:
|
|
|
371
393
|
p.add_argument("--tokens", action="store_true", help="rank by output tokens instead of dollars")
|
|
372
394
|
p.add_argument("--grading", action="store_true", help="print which rule graded what")
|
|
373
395
|
p.add_argument("--quiet", action="store_true", help="suppress progress and timing on stderr")
|
|
396
|
+
p.add_argument("--home", default=None, metavar="PATH", help="store root whose cache/ holds the log parse cache "
|
|
397
|
+
"(default: $LOOPMATH_HOME or ~/.loopmath; LOOPMATH_CACHE_DIR wins); analyze writes nothing else")
|
|
398
|
+
p.add_argument("--json", action="store_true", help="print one JSON object on stdout")
|
|
374
399
|
p.set_defaults(func=analyze)
|
|
375
400
|
|
|
376
401
|
|
|
@@ -385,11 +410,18 @@ def _add_validate_prices(sub) -> None:
|
|
|
385
410
|
def _add_fit(sub) -> None:
|
|
386
411
|
from .research_defaults import E1A_OBSERVE, SEED
|
|
387
412
|
|
|
388
|
-
research = sub.add_parser(
|
|
413
|
+
research = sub.add_parser(
|
|
414
|
+
"research",
|
|
415
|
+
help="verbs for research data folders that are not part of the package (loopmath research --help)",
|
|
416
|
+
description="Verbs for research data folders that are not part of the package: sweep run records "
|
|
417
|
+
"(research fit, research transfer-test; they need the bayes extra) and a session corpus "
|
|
418
|
+
"(research analyze-e0; it needs the e0 extra). Name a folder with --sweep-dir or --corpus, "
|
|
419
|
+
"LOOPMATH_SWEEP_DIR or LOOPMATH_E0_CORPUS, or config research.sweep_dir or research.e0_corpus.",
|
|
420
|
+
)
|
|
389
421
|
rsub = research.add_subparsers(dest="research_command", required=True)
|
|
390
422
|
for name, fn, helptext in (
|
|
391
|
-
("fit", fit_verb, "fit the
|
|
392
|
-
("transfer-test", transfer_test_verb, "score held-out
|
|
423
|
+
("fit", fit_verb, "fit the research cost model on sweep run records"),
|
|
424
|
+
("transfer-test", transfer_test_verb, "fit on some cells of the sweep and score the held-out ones"),
|
|
393
425
|
):
|
|
394
426
|
p = rsub.add_parser(name, help=helptext)
|
|
395
427
|
p.add_argument("--sweep-dir", default=None, help="sweep run records directory (default: LOOPMATH_SWEEP_DIR, then research.sweep_dir in config)")
|
|
@@ -405,10 +437,20 @@ def _add_fit(sub) -> None:
|
|
|
405
437
|
else:
|
|
406
438
|
p.add_argument("--ci", type=float, default=0.80, help="central interval for held-out coverage (default 0.80)")
|
|
407
439
|
p.set_defaults(func=fn)
|
|
440
|
+
from .e0 import verb as _e0_verb
|
|
441
|
+
|
|
442
|
+
_e0_verb.register(rsub)
|
|
408
443
|
|
|
409
444
|
|
|
410
445
|
def verify_receipts_verb(args: argparse.Namespace) -> int:
|
|
411
|
-
"""
|
|
446
|
+
"""No ledger: check the store's receipts. A ledger path: verify that research receipt ledger."""
|
|
447
|
+
if args.ledger is None:
|
|
448
|
+
from .store.home import Store
|
|
449
|
+
from .store.verify import verify_lines, verify_store
|
|
450
|
+
|
|
451
|
+
res = verify_store(Store(args.home))
|
|
452
|
+
print("\n".join(verify_lines(res)))
|
|
453
|
+
return 1 if res["failed"] else 0
|
|
412
454
|
from .receipts import ReceiptViolation, verify_receipts
|
|
413
455
|
|
|
414
456
|
try:
|
|
@@ -421,18 +463,20 @@ def verify_receipts_verb(args: argparse.Namespace) -> int:
|
|
|
421
463
|
|
|
422
464
|
|
|
423
465
|
def _add_verify_receipts(sub) -> None:
|
|
424
|
-
p = sub.add_parser("verify-receipts", help="
|
|
466
|
+
p = sub.add_parser("verify-receipts", help="check that the receipts in your store are whole and scored right "
|
|
467
|
+
"(or give a research ledger file)")
|
|
425
468
|
p.add_argument(
|
|
426
469
|
"ledger",
|
|
427
470
|
nargs="?",
|
|
428
|
-
default=
|
|
429
|
-
help="receipt ledger (
|
|
471
|
+
default=None,
|
|
472
|
+
help="a research receipt ledger to verify instead (e.g. e2-receipts.jsonl)",
|
|
430
473
|
)
|
|
431
474
|
p.add_argument(
|
|
432
475
|
"--frozen",
|
|
433
476
|
default=None,
|
|
434
|
-
help="checksum file (default: FROZEN.txt beside the ledger)",
|
|
477
|
+
help="with a ledger: its checksum file (default: FROZEN.txt beside the ledger)",
|
|
435
478
|
)
|
|
479
|
+
p.add_argument("--home", default=None, metavar="PATH", help="the store (default: $LOOPMATH_HOME or ~/.loopmath)")
|
|
436
480
|
p.set_defaults(func=verify_receipts_verb)
|
|
437
481
|
|
|
438
482
|
|
|
@@ -455,15 +499,14 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
455
499
|
|
|
456
500
|
_registry.register(sub)
|
|
457
501
|
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
from .e0 import verb as _e0_verb
|
|
502
|
+
# The E0 walkdown draws figures, so running it needs matplotlib (the `e0`
|
|
503
|
+
# extra; SPEC section 2's light core is numpy and pandas). It is listed under
|
|
504
|
+
# `loopmath research`; the old top-level name stays callable, off the list.
|
|
505
|
+
# It registers from e0/verb.py without importing the walkdown, so no other
|
|
506
|
+
# command pays for pandas and matplotlib.
|
|
507
|
+
from .e0 import verb as _e0_verb
|
|
465
508
|
|
|
466
|
-
|
|
509
|
+
_e0_verb.register(sub, listed=False)
|
|
467
510
|
|
|
468
511
|
try:
|
|
469
512
|
_add_fit(sub)
|