loopmath 0.4.1__py3-none-any.whl → 0.4.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loopmath/__init__.py +1 -1
- loopmath/belief/design.py +70 -14
- loopmath/belief/fit.py +11 -3
- loopmath/belief/state.py +7 -3
- loopmath/cli_registry.py +5 -1
- loopmath/harness/herdr/loopmath-harness/herdr-plugin.toml +1 -1
- loopmath/harness/mod/loopmath-harness/.claude-plugin/plugin.json +1 -1
- loopmath/recommend/search.py +5 -4
- loopmath/store/config.py +6 -2
- {loopmath-0.4.1.dist-info → loopmath-0.4.2.dist-info}/METADATA +3 -1
- {loopmath-0.4.1.dist-info → loopmath-0.4.2.dist-info}/RECORD +18 -18
- {loopmath-0.4.1.data → loopmath-0.4.2.data}/data/spec/ocp-v0.2.schema.json +0 -0
- {loopmath-0.4.1.data → loopmath-0.4.2.data}/data/spec/ocp-v0.schema.json +0 -0
- {loopmath-0.4.1.data → loopmath-0.4.2.data}/data/spec/ocp_conformance.py +0 -0
- {loopmath-0.4.1.dist-info → loopmath-0.4.2.dist-info}/WHEEL +0 -0
- {loopmath-0.4.1.dist-info → loopmath-0.4.2.dist-info}/entry_points.txt +0 -0
- {loopmath-0.4.1.dist-info → loopmath-0.4.2.dist-info}/licenses/LICENSE +0 -0
- {loopmath-0.4.1.dist-info → loopmath-0.4.2.dist-info}/top_level.txt +0 -0
loopmath/__init__.py
CHANGED
loopmath/belief/design.py
CHANGED
|
@@ -45,7 +45,20 @@ MODEL_OPTIONS: dict[str, tuple[Any, ...]] = {"belief.success_head": ("sampled",)
|
|
|
45
45
|
# the store level and benchmark transfer (level.py)
|
|
46
46
|
"belief.level": ("shared", "store"),
|
|
47
47
|
"belief.shipped_success": ("keep", "drop"),
|
|
48
|
-
"belief.benchmark_factors": ("reference", "offset")
|
|
48
|
+
"belief.benchmark_factors": ("reference", "offset"),
|
|
49
|
+
# the seat weights of the success and score rows (`seat_weights`)
|
|
50
|
+
"belief.seat_weights": ("role-v1",)}
|
|
51
|
+
# Seat weights (0.4.2): how much each piece's model chain, effort and harness counts in a success or score row
|
|
52
|
+
# (`run_rest`). `average` is the rule up to 0.4.1: every piece at 1/n. A role rule gives each role one weight,
|
|
53
|
+
# relative to the writer and not rescaled to sum to 1, shared equally by the pieces of that role. `role-v1`: the
|
|
54
|
+
# writer (`implementer`) 1, the planner 0.22, the reviewers 0.11. These three numbers are a placeholder from one
|
|
55
|
+
# experiment on 8 coding tasks and will change. A fit records its rule in `model_options` under SEAT_KEY (nothing
|
|
56
|
+
# under `average`), so a fit is always read with the rule it was made under.
|
|
57
|
+
SEAT_KEY = "belief.seat_weights"
|
|
58
|
+
SEAT_AVERAGE = "average"
|
|
59
|
+
SEAT_RULE = "role-v1" # what the config value `role` means in this loopmath
|
|
60
|
+
SEAT_WRITER = "implementer"
|
|
61
|
+
SEAT_RULES: dict[str, dict[str, float]] = {"role-v1": {SEAT_WRITER: 1.0, "planner": 0.22, "reviewer": 0.11}}
|
|
49
62
|
# The version the seed key hashes (spec 04 section 3). It stays 3 from 0.2.8: the same data seeds the same
|
|
50
63
|
# draws as in 0.2.7, so every number that is not a token count is 0.2.7's.
|
|
51
64
|
SEED_VERSION = 3
|
|
@@ -450,19 +463,59 @@ def gate_rest(st: Structure, gate: GateInfo, k: int) -> tuple[Term, ...]:
|
|
|
450
463
|
+ role_terms(role, m) + step_terms(st.flow, gate.judged, role, m))
|
|
451
464
|
|
|
452
465
|
|
|
453
|
-
def
|
|
466
|
+
def seat_rule(config: Mapping[str, Any] | None) -> str:
|
|
467
|
+
"""The seat rule a fit is made under, from a config.toml table: `[belief] seat_weights`, `"role"` (SEAT_RULE)
|
|
468
|
+
or `"average"`. Unset or malformed, it follows the level as `shipped_success` does (level.py): the role rule
|
|
469
|
+
under the store level, and `average` under `level = "shared"`, which stays 0.3.5's fit exactly."""
|
|
470
|
+
b = (config or {}).get("belief")
|
|
471
|
+
b = b if isinstance(b, Mapping) else {}
|
|
472
|
+
value = b.get("seat_weights")
|
|
473
|
+
if value == "role":
|
|
474
|
+
return SEAT_RULE
|
|
475
|
+
if value == SEAT_AVERAGE:
|
|
476
|
+
return SEAT_AVERAGE
|
|
477
|
+
return SEAT_AVERAGE if b.get("level") == "shared" else SEAT_RULE
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
def fit_seat_rule(meta: Mapping[str, Any]) -> str:
|
|
481
|
+
"""The seat rule a fit was made under, from its meta.json: `model_options[SEAT_KEY]`, else `average` (every
|
|
482
|
+
fit before 0.4.2, and a later fit under `average`, which writes no option)."""
|
|
483
|
+
options = meta.get("model_options")
|
|
484
|
+
value = options.get(SEAT_KEY) if isinstance(options, Mapping) else None
|
|
485
|
+
return value if isinstance(value, str) and value in SEAT_RULES else SEAT_AVERAGE
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def seat_weights(st: Structure, seats: str = SEAT_RULE) -> dict[str, float]:
|
|
489
|
+
"""piece -> the weight of its model chain, effort and harness in the success and score rows (`run_rest`).
|
|
490
|
+
|
|
491
|
+
Under a role rule (SEAT_RULES) each role has one weight, shared equally by the pieces of that role: one
|
|
492
|
+
writer counts 1, one planner 0.22, one reviewer 0.11 and each of two reviewers 0.055. The weights are
|
|
493
|
+
relative to the writer and do not sum to 1. The rule covers workflows made of writers, planners and
|
|
494
|
+
reviewers with at least one writer. Any other workflow (a piece of another role such as `tester`, `referee`
|
|
495
|
+
or `worker`, or no writer) keeps `average`, 1/n for each of its n pieces, as does every workflow under
|
|
496
|
+
`seats = "average"`. A workflow of one piece is at 1 under every rule."""
|
|
497
|
+
n = max(1, len(st.pieces))
|
|
498
|
+
weights = SEAT_RULES.get(seats)
|
|
499
|
+
roles = [st.roles[p] for p in st.pieces]
|
|
500
|
+
if weights is None or SEAT_WRITER not in roles or any(r not in weights for r in roles):
|
|
501
|
+
return {p: 1.0 / n for p in st.pieces}
|
|
502
|
+
return {p: weights[r] / roles.count(r) for p, r in zip(st.pieces, roles)}
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def run_rest(st: Structure, seats: str = SEAT_RULE) -> tuple[Term, ...]:
|
|
454
506
|
"""Success and score rows without the task part: topology, control, and per piece its model chain,
|
|
455
|
-
effort and harness
|
|
507
|
+
effort and harness at its seat weight (`seat_weights` under the rule `seats`: by role from 0.4.2, 1/n
|
|
508
|
+
under `average`) with its harness factors at that weight, plus its role and role x family
|
|
456
509
|
(weight 1/g, g the copies in its group, so parallel copies carry the role effect once, as one wide piece
|
|
457
510
|
does), and in a flow run its step term at the role x family weight (`step_terms`)."""
|
|
458
511
|
terms: list[Term] = [(f"topology:{st.shape or st.workflow_id}", None, 1.0)]
|
|
459
512
|
terms += _control_terms(st.k_max, max(st.widths.values() or [1]))
|
|
460
|
-
|
|
513
|
+
weight = seat_weights(st, seats)
|
|
461
514
|
for piece in st.pieces:
|
|
462
515
|
h, m, e = _s(st.settings[piece])
|
|
463
|
-
terms += setting_terms(h, m, e,
|
|
516
|
+
terms += setting_terms(h, m, e, weight[piece])
|
|
464
517
|
if st.factors:
|
|
465
|
-
terms += factor_terms(h, st.factors,
|
|
518
|
+
terms += factor_terms(h, st.factors, weight[piece])
|
|
466
519
|
g = 1.0 / (st.copies.get(piece) or 1)
|
|
467
520
|
terms += role_terms(st.roles[piece], m, g) + step_terms(st.flow, piece, st.roles[piece], m, g)
|
|
468
521
|
return tuple(terms)
|
|
@@ -479,8 +532,9 @@ def gate_row(task: Task, source: str, st: Structure, gate: GateInfo, k: int,
|
|
|
479
532
|
return task_terms(task, source, features) + list(gate_rest(st, gate, k))
|
|
480
533
|
|
|
481
534
|
|
|
482
|
-
def run_row(task: Task, source: str, st: Structure, features: FeatureSet | None = None
|
|
483
|
-
|
|
535
|
+
def run_row(task: Task, source: str, st: Structure, features: FeatureSet | None = None,
|
|
536
|
+
seats: str = SEAT_RULE) -> list[Term]:
|
|
537
|
+
return task_terms(task, source, features) + list(run_rest(st, seats))
|
|
484
538
|
|
|
485
539
|
|
|
486
540
|
def _control_terms(k_max: int, width: int) -> list[Term]:
|
|
@@ -495,9 +549,10 @@ def _control_terms(k_max: int, width: int) -> list[Term]:
|
|
|
495
549
|
|
|
496
550
|
|
|
497
551
|
def rows_for_config(task: Task, config: Configuration, source: str = "user",
|
|
498
|
-
features: FeatureSet | None = None, factors: Sequence[str] = ()
|
|
552
|
+
features: FeatureSet | None = None, factors: Sequence[str] = (),
|
|
553
|
+
seats: str = SEAT_RULE) -> dict:
|
|
499
554
|
"""Prediction rows: per (piece, round), per (gate, round), one run row, and the structure. `factors`: the
|
|
500
|
-
harness factors on every piece (`spec/harness.md`)."""
|
|
555
|
+
harness factors on every piece (`spec/harness.md`); `seats`: the seat rule of the run row (`seat_weights`)."""
|
|
501
556
|
st = structure(config, factors)
|
|
502
557
|
cost = {}
|
|
503
558
|
for piece in st.pieces:
|
|
@@ -509,7 +564,7 @@ def rows_for_config(task: Task, config: Configuration, source: str = "user",
|
|
|
509
564
|
rounds = st.k_max if st.gate_loop.get(gi) is not None else 1
|
|
510
565
|
for k in range(1, rounds + 1):
|
|
511
566
|
gate[(gi, k)] = gate_row(task, source, st, g, k, features)
|
|
512
|
-
return {"structure": st, "cost": cost, "gate": gate, "run": run_row(task, source, st, features)}
|
|
567
|
+
return {"structure": st, "cost": cost, "gate": gate, "run": run_row(task, source, st, features, seats)}
|
|
513
568
|
|
|
514
569
|
|
|
515
570
|
# ---------------------------------------------------------------- reading run documents
|
|
@@ -1030,8 +1085,9 @@ def _gate_observations(doc: dict, st: Structure, by_piece_round: dict) -> list[t
|
|
|
1030
1085
|
return out
|
|
1031
1086
|
|
|
1032
1087
|
|
|
1033
|
-
def rows_for_run(run_doc: dict) -> dict:
|
|
1034
|
-
"""Fitting rows for one run document: {cost, tokens, gate, success, scores} plus its structure
|
|
1088
|
+
def rows_for_run(run_doc: dict, seats: str = SEAT_RULE) -> dict:
|
|
1089
|
+
"""Fitting rows for one run document: {cost, tokens, gate, success, scores} plus its structure; `seats` is
|
|
1090
|
+
the seat rule of the success and score rows (`seat_weights`)."""
|
|
1035
1091
|
pr = parse_run(run_doc)
|
|
1036
1092
|
st = structure(pr.config, pr.factors, pr.flow)
|
|
1037
1093
|
out: dict[str, Any] = {"run": pr.run_id, "source": pr.source, "structure": st,
|
|
@@ -1044,7 +1100,7 @@ def rows_for_run(run_doc: dict) -> dict:
|
|
|
1044
1100
|
out["tokens"].append((terms, math.log(a.tokens), a.weight))
|
|
1045
1101
|
for gi, k, passed in pr.gates:
|
|
1046
1102
|
out["gate"].append((gate_row(pr.task, pr.source, st, st.gates[gi], k), 1.0 if passed else 0.0))
|
|
1047
|
-
row = run_row(pr.task, pr.source, st)
|
|
1103
|
+
row = run_row(pr.task, pr.source, st, seats=seats)
|
|
1048
1104
|
if pr.evidence is not None and pr.evidence.z is not None:
|
|
1049
1105
|
out["success"] = (row, float(pr.evidence.z), float(pr.evidence.q))
|
|
1050
1106
|
for name, value in (pr.evidence.scores if pr.evidence else {}).items():
|
loopmath/belief/fit.py
CHANGED
|
@@ -251,9 +251,12 @@ def _flow_run(doc: dict) -> bool:
|
|
|
251
251
|
|
|
252
252
|
def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tuple[str, ...] = (),
|
|
253
253
|
now: datetime | None = None, features: FeatureSet | None = None,
|
|
254
|
-
q_by_tier: dict[str, float] | None = None, own_success: bool = False
|
|
254
|
+
q_by_tier: dict[str, float] | None = None, own_success: bool = False,
|
|
255
|
+
seats: str = design_rows.SEAT_RULE):
|
|
255
256
|
"""Rows of every head from run documents, plus counts of what was used and dropped.
|
|
256
257
|
|
|
258
|
+
`seats` (config `belief.seat_weights`, `design.seat_rule`): the seat rule of the success and score rows.
|
|
259
|
+
|
|
257
260
|
`own_success` (config `belief.shipped_success = "drop"`): shipped runs give no success or score rows, so
|
|
258
261
|
the success level comes from the user's runs and shared imports only (`level.py`).
|
|
259
262
|
|
|
@@ -329,7 +332,7 @@ def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tup
|
|
|
329
332
|
for gi, k, passed in pr.gates:
|
|
330
333
|
heads["gate"].add(gate_row(pr.task, pr.source, st, st.gates[gi], k, features), 1.0 if passed else 0.0,
|
|
331
334
|
1.0, pr.run_id, pr.source, ttype)
|
|
332
|
-
row = run_row(pr.task, pr.source, st, features)
|
|
335
|
+
row = run_row(pr.task, pr.source, st, features, seats)
|
|
333
336
|
if _flow_run(doc):
|
|
334
337
|
flow_runs.append((doc, pr, row, ttype))
|
|
335
338
|
continue
|
|
@@ -721,6 +724,7 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
721
724
|
# `outcome.q.<tier>`, read as `run finish` reads it; unset, the built-in q (0.2.6, fix (1))
|
|
722
725
|
q_by_tier = {tier: Config(home / "config.toml", config).q_for_tier(tier) for tier in TIER_Q}
|
|
723
726
|
level = level_settings(config)
|
|
727
|
+
seats = design_rows.seat_rule(config) # `belief.seat_weights`: by role, or `average` (1/n, 0.4.1's rows)
|
|
724
728
|
|
|
725
729
|
def all_docs() -> Iterator[dict | tuple[str | None, dict]]:
|
|
726
730
|
if docs is not None:
|
|
@@ -734,7 +738,8 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
734
738
|
|
|
735
739
|
(heads_rows, score_info, dropped, runs_by_source, config_support, checks, overlap,
|
|
736
740
|
labels, tally, variants) = collect_rows(all_docs(), without=without, now=now, features=features,
|
|
737
|
-
q_by_tier=q_by_tier, own_success=level.own_success
|
|
741
|
+
q_by_tier=q_by_tier, own_success=level.own_success,
|
|
742
|
+
seats=seats)
|
|
738
743
|
removed = {k[len("without "):] for k in dropped if k.startswith("without ")}
|
|
739
744
|
unknown, known = unknown_sources(without, set(runs_by_source) | removed, bundle_dir)
|
|
740
745
|
if unknown:
|
|
@@ -876,6 +881,9 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
876
881
|
"belief.shipped_success": level.shipped_success,
|
|
877
882
|
"belief.benchmark_factors": level.benchmarks})
|
|
878
883
|
meta["options"]["level"] = level.key()
|
|
884
|
+
if seats != design_rows.SEAT_AVERAGE: # rows by a role rule: 0.4.1 and older refuse the fit by name
|
|
885
|
+
meta["design_version"] = design_rows.OPT_IN_DESIGN_VERSION
|
|
886
|
+
meta.setdefault("model_options", {})[design_rows.SEAT_KEY] = seats
|
|
879
887
|
if "recipe" in fitted.get("cost", {}):
|
|
880
888
|
from .cost_recipe import METHOD, OPTION
|
|
881
889
|
|
loopmath/belief/state.py
CHANGED
|
@@ -62,8 +62,8 @@ from ..types import (
|
|
|
62
62
|
Prediction, ScorePrediction, Task,
|
|
63
63
|
)
|
|
64
64
|
from .compose import Intervals, RunDraws, compose, cost_per_round, interval, ranges
|
|
65
|
-
from .design import (Structure, cost_rest, gate_rest, other_design, run_rest, split_factor_node,
|
|
66
|
-
task_features, task_terms)
|
|
65
|
+
from .design import (Structure, cost_rest, fit_seat_rule, gate_rest, other_design, run_rest, split_factor_node,
|
|
66
|
+
structure, task_features, task_terms)
|
|
67
67
|
from .forest import LEVEL_ALIASES, canonical_model_id, default_scale, display_level, level_of, node_key, scale_group
|
|
68
68
|
from .block import BlockFactor, blocks, leading_diagonal
|
|
69
69
|
from .gaussian import _cho, cov_factor
|
|
@@ -444,6 +444,9 @@ class FitState:
|
|
|
444
444
|
self.price_offsets: dict[str, float] = dict((self.meta.get("price_offsets") or {}).get("models") or {})
|
|
445
445
|
# spec 04 section 3 (0.2.8): each priced model's cache-read rate, $ per token; a draw's rate stays above it
|
|
446
446
|
self.rate_floors: dict[str, float] = dict(self.meta.get("rate_floors") or {})
|
|
447
|
+
# the seat rule the fit's success and score rows were made under (`design.seat_weights`): predictions
|
|
448
|
+
# use the same one, so a fit from before 0.4.2 keeps its 1/n rows until the next fit
|
|
449
|
+
self.seat_rule: str = fit_seat_rule(self.meta)
|
|
447
450
|
if heads is None:
|
|
448
451
|
heads = {}
|
|
449
452
|
for name, hmeta in (self.meta.get("heads") or {}).items():
|
|
@@ -532,7 +535,8 @@ class FitState:
|
|
|
532
535
|
gate[(gi, k)] = gate_rest(st, g, k)
|
|
533
536
|
floors = {piece: float(self.rate_floors.get(canonical_model_id(st.settings[piece].model), 0.0))
|
|
534
537
|
for piece in st.pieces}
|
|
535
|
-
return {"config": config, "st": st, "cost": cost, "gate": gate, "run": run_rest(st),
|
|
538
|
+
return {"config": config, "st": st, "cost": cost, "gate": gate, "run": run_rest(st, self.seat_rule),
|
|
539
|
+
"floors": floors}
|
|
536
540
|
|
|
537
541
|
# ------------------------------------------------------------ the task as the fit reads it
|
|
538
542
|
|
loopmath/cli_registry.py
CHANGED
|
@@ -538,7 +538,11 @@ def _add_recording(sub) -> None:
|
|
|
538
538
|
"benchmark in the shipped benchmarks.toml has its own weight, in runs, shared by "
|
|
539
539
|
"a model's results on it; a number here is one weight for every benchmark, and 0 "
|
|
540
540
|
"turns benchmark priors off. belief.level is store by default: the store's pass "
|
|
541
|
-
"rate is learned from its own runs only; shared is 0.3.5's fit."
|
|
541
|
+
"rate is learned from its own runs only; shared is 0.3.5's fit. "
|
|
542
|
+
"belief.seat_weights is role by default: in a multi-piece workflow the writer's "
|
|
543
|
+
"model, effort and harness count 1, the planner's 0.22 and the reviewers' 0.11 "
|
|
544
|
+
"between them (placeholder values); average counts every piece the same, as "
|
|
545
|
+
"0.4.1 did.")
|
|
542
546
|
q.add_argument("key", metavar="KEY")
|
|
543
547
|
q.add_argument("value", metavar="VALUE")
|
|
544
548
|
_common(q)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "loopmath-harness",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.2",
|
|
4
4
|
"description": "Configure, start and watch a loopmath harness tune from Claude Code. Runs the loopmath CLI; makes no model call itself.",
|
|
5
5
|
"author": { "name": "loopmath" },
|
|
6
6
|
"types": "./types/index.d.ts",
|
loopmath/recommend/search.py
CHANGED
|
@@ -36,7 +36,7 @@ from typing import Any, Callable, Iterable, Sequence
|
|
|
36
36
|
import numpy as np
|
|
37
37
|
from scipy.special import expit, ndtr
|
|
38
38
|
|
|
39
|
-
from ..belief.design import Structure, _piece_reach, cost_rest, gate_rest, run_rest, structure
|
|
39
|
+
from ..belief.design import SEAT_RULE, Structure, _piece_reach, cost_rest, gate_rest, run_rest, structure
|
|
40
40
|
from ..belief.forest import canonical_model_id
|
|
41
41
|
from ..belief.state import MEAN_NODES, MIN_SCORE_SWITCH, N_DRAWS, PREDICT_SOURCE, _GW, _GX, transform
|
|
42
42
|
from ..types import AcceptanceRule, Configuration, Piece, Prediction, Setting, Task, Workflow
|
|
@@ -454,6 +454,7 @@ class Tables:
|
|
|
454
454
|
self.tp = fs._task_parts(task)
|
|
455
455
|
self.effort = fs.effort_for(task) if hasattr(fs, "effort_for") else 1.0 # spec 04 section 1, under a timebox
|
|
456
456
|
self.prices = getattr(fs, "price_offsets", None) # spec 04 section 2, the price offset per model
|
|
457
|
+
self.seats = getattr(fs, "seat_rule", SEAT_RULE) # the fit's seat rule, as its own predictions use it
|
|
457
458
|
self.half_u = 0.5 * fs.heads["cost"].sigma ** 2
|
|
458
459
|
self.draw_idx = draw_idx
|
|
459
460
|
tg = self.tp["gate"]
|
|
@@ -544,7 +545,7 @@ class Tables:
|
|
|
544
545
|
"""The run row as a case part plus one setting part per piece: {piece: parts}, case terms.
|
|
545
546
|
Cases with the same pieces and flow share the per-piece parts; a hit is checked on the last setting."""
|
|
546
547
|
st = case.st
|
|
547
|
-
r0 = run_rest(st)
|
|
548
|
+
r0 = run_rest(st, self.seats)
|
|
548
549
|
mkey = (st.workflow_id, tuple(st.pieces), tuple(st.roles[p] for p in st.pieces),
|
|
549
550
|
tuple(sorted(case.workflow.edges)), len(r0))
|
|
550
551
|
hit = self._run_memo.get(mkey)
|
|
@@ -552,7 +553,7 @@ class Tables:
|
|
|
552
553
|
vary, parts = hit
|
|
553
554
|
for p in st.pieces:
|
|
554
555
|
with _Varied(st, p, self.settings[-1]) as v:
|
|
555
|
-
row = run_rest(v)
|
|
556
|
+
row = run_rest(v, self.seats)
|
|
556
557
|
want = list(r0)
|
|
557
558
|
for i, t in zip(vary[p], parts[p][-1]):
|
|
558
559
|
want[i] = t
|
|
@@ -565,7 +566,7 @@ class Tables:
|
|
|
565
566
|
rows = []
|
|
566
567
|
for s in self.settings:
|
|
567
568
|
with _Varied(st, p, s) as v:
|
|
568
|
-
rows.append(run_rest(v))
|
|
569
|
+
rows.append(run_rest(v, self.seats))
|
|
569
570
|
if any(len(r) != len(r0) for r in rows):
|
|
570
571
|
raise NotExact("run rows change length with a setting")
|
|
571
572
|
vary[p] = [i for i in range(len(r0)) if any(r[i] != r0[i] for r in rows)]
|
loopmath/store/config.py
CHANGED
|
@@ -16,7 +16,9 @@ declared task features `features.<key>` with `features.min_tasks`
|
|
|
16
16
|
`eb`, 0.3.1's success head; `sampled` opts in to the success head by NUTS over
|
|
17
17
|
its effects and scales, belief.integrated and belief.sampled), and `belief.level` (unset or `store`, from 0.4.0:
|
|
18
18
|
each store's own success level, benchmarks by offset; `shared` is 0.3.5's fit, belief.level) with its two sds,
|
|
19
|
-
`belief.benchmark_effort_sd` and `belief.cost_level_sd` (0.5 each under `store`).
|
|
19
|
+
`belief.benchmark_effort_sd` and `belief.cost_level_sd` (0.5 each under `store`), and `belief.seat_weights` (unset or
|
|
20
|
+
`role`: a multi-piece run's success and score rows carry the writer's setting at 1, the planner's at 0.22 and the
|
|
21
|
+
reviewers' at 0.11 between them, placeholder values; `average` is 0.4.1's 1/n, also the default under `shared`).
|
|
20
22
|
|
|
21
23
|
Reading uses `tomllib` (Python 3.11+), else a reader for the
|
|
22
24
|
subset this module writes. The writer emits plain tables and JSON-compatible
|
|
@@ -60,6 +62,8 @@ CHOICES: dict[str, tuple[str, ...]] = {
|
|
|
60
62
|
# unset or `eb`: 0.3.1's success head; `sampled` opts in (belief.integrated, belief.sampled)
|
|
61
63
|
"belief.success_head": ("eb", "sampled"),
|
|
62
64
|
"belief.level": ("store", "shared"), # unset: `store` (0.4.0); `shared` is 0.3.5's fit (belief.level)
|
|
65
|
+
# unset: `role` under the store level, `average` under `shared` (0.4.2; belief.design.seat_rule)
|
|
66
|
+
"belief.seat_weights": ("role", "average"),
|
|
63
67
|
}
|
|
64
68
|
LABELER_KINDS = ("claude", "codex", "command")
|
|
65
69
|
LABELER_KEYS = ("onboard.labeler", "labeler") # `labeler` is the spec 02 name; D39 stores `onboard.labeler`
|
|
@@ -80,7 +84,7 @@ STRING_ROOTS = ("research",) # values stored as given, never parsed as JSON
|
|
|
80
84
|
KEYS = ("org", "acceptance_rule", "rules.<name>", "goal", "rescue.kind", "rescue.person_usd_per_hour",
|
|
81
85
|
"rescue.hours", "rescue.decay", "rescue.max_attempts", "rescue.min_chance", "models.allowed", "harnesses",
|
|
82
86
|
"subtypes", "benchmark_prior_weight", "belief.success_head", "belief.level", "belief.benchmark_effort_sd",
|
|
83
|
-
"belief.cost_level_sd", "explore.default_pick",
|
|
87
|
+
"belief.cost_level_sd", "belief.seat_weights", "explore.default_pick",
|
|
84
88
|
"explore.auto_payback_runs", "referee.model",
|
|
85
89
|
"budget.usd", "budget.period", "outcome.q.<tier>", "onboard.labeler", "onboard.skip",
|
|
86
90
|
"usual.<type>.<repo>", "usual_meta.<type>.<repo>", "research.sweep_dir", "research.e0_corpus",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: loopmath
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.2
|
|
4
4
|
Summary: Plan agent workflows, record how they went, and learn from your own Claude Code and Codex logs
|
|
5
5
|
Author: loopmath
|
|
6
6
|
License-Expression: MIT
|
|
@@ -112,6 +112,8 @@ Reading the history uses up to 8 worker processes (one per CPU, at most 8). Set
|
|
|
112
112
|
|
|
113
113
|
loopmath ships with a prior built from our own sweeps and experiments, so `recommend` gives an answer before you have any history. Your own runs then move it. Your store's overall pass rate is its own, with a wide prior that only your runs move: the shipped runs give their costs and gate rows but no success rows, and the public benchmarks say which models are stronger (`belief.level`, `store` by default; `loopmath config set belief.level shared`, then `loopmath fit`, gives 0.3.5's fit).
|
|
114
114
|
|
|
115
|
+
In a workflow with several pieces, the piece that writes the change counts most in the predicted chance of success: its model, effort and harness count 1, the planner's 0.22 and the reviewers' 0.11 between them. These three weights are a placeholder from one experiment on 8 coding tasks; they are fixed, not learned from your runs, and will change. `loopmath config set belief.seat_weights average`, then `loopmath fit`, counts every piece the same, as 0.4.1 did.
|
|
116
|
+
|
|
115
117
|
The prior ships inside the package: our runs, in the shared form (no titles, paths, commands or free text), and the published benchmark results. They are the same files we fit with, and each release updates them. `onboard` and the results page say in one line which prior they start from, and `loopmath prior show` lists it source by source.
|
|
116
118
|
|
|
117
119
|
## The loop
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
loopmath/__init__.py,sha256=
|
|
1
|
+
loopmath/__init__.py,sha256=kpqp9WNMPSjE7S0rPorsQD9ma0VCeQqF2hJ0_8O6l08,244
|
|
2
2
|
loopmath/__main__.py,sha256=X6nV3FjHmxDbIjccOYrBts_0KyJwOmDEPI5URzMO_-8,126
|
|
3
3
|
loopmath/adapter_host.py,sha256=KBLHsP-LzrRDzOaDQMJRDq-NfCJeT5nY_CQAPLqyKQ4,10284
|
|
4
4
|
loopmath/cli.py,sha256=cApXAOg1xMcBsOpgUZkrGvXP3IlxFmXnyTRGqlikVNk,32240
|
|
5
5
|
loopmath/cli_errors.py,sha256=pf7kuCoA--zV0L4wwj8v3JA3KrJFaDIcdLUUuSF2sRo,4839
|
|
6
6
|
loopmath/cli_graph.py,sha256=30VWDHnTEwAB_IX6CL_FluvnfAjydwrjDm0GRlBcLlQ,13492
|
|
7
|
-
loopmath/cli_registry.py,sha256=
|
|
7
|
+
loopmath/cli_registry.py,sha256=F4eHyQX-a-N_P74l501jiDh382PBCnyWhOcC2wbqQts,92383
|
|
8
8
|
loopmath/cli_support.py,sha256=gD_33ELLH4oUuYtWbxoCSYr2LUpJuh3ay80sn1KpTUA,23199
|
|
9
9
|
loopmath/explain.py,sha256=fQN2ANW4fniyWdm0k14Cvioicm_Nncx7-lanVPteTUw,3166
|
|
10
10
|
loopmath/fit.py,sha256=8eelfN7KqrHVQFLvWkq0iCgHLD7Jun8shuslUFuXfmU,14680
|
|
@@ -73,9 +73,9 @@ loopmath/belief/commands.py,sha256=Qui1HXTbcdkRVDTrGVHNw2fX47b5qjpMOrbcshVW8tw,1
|
|
|
73
73
|
loopmath/belief/compose.py,sha256=tM4sRlvG2HIepkbVDwmsXO7v2-HCxsIo-WucamM4zkg,12390
|
|
74
74
|
loopmath/belief/cost_recipe.py,sha256=kSa_SOr9VqNxUym3trvjQRsVf4fWVMo_-qFxyb6qqzQ,8726
|
|
75
75
|
loopmath/belief/cost_sources.py,sha256=NpwwJkv3TtSLnLkX5avdsfrfN19jzD_S7DwPtFWQak4,3487
|
|
76
|
-
loopmath/belief/design.py,sha256
|
|
76
|
+
loopmath/belief/design.py,sha256=-ZKQzDZQTkIEYrwKPz3RtmCB_JcLOikeFZdTHSQtXvY,55344
|
|
77
77
|
loopmath/belief/features.py,sha256=4nStB7E1a3cgAXdh7xXCCH4Ddf9T9vNi0AFnx5DP_LQ,11615
|
|
78
|
-
loopmath/belief/fit.py,sha256=
|
|
78
|
+
loopmath/belief/fit.py,sha256=RnajZyhSrtg0iW6fT-2yFZYCXDT3HzR1oUHgNE4qQ5U,50696
|
|
79
79
|
loopmath/belief/forest.py,sha256=UK9CQSOZU-F6U85X0_MB7HEMpxn9qxi4t9HlnZDgKPI,9504
|
|
80
80
|
loopmath/belief/gaussian.py,sha256=YgN-SUOzSKj-dXndjPaaDMOw2oCx_NVj3hoGtn2B_IM,22090
|
|
81
81
|
loopmath/belief/integrated.py,sha256=tscZbSN7jCAjmJcVnwHaO-L-DV-NgTFor2OvFiBUGXs,7009
|
|
@@ -86,7 +86,7 @@ loopmath/belief/pricing.py,sha256=vTKf7iheX9zW9TjpbNOk9BhLhxN0BZNLoSZXD0eKo6k,79
|
|
|
86
86
|
loopmath/belief/priors.py,sha256=nfovOtwso11Xch2QTrXwzA3tPWFY9Diywvrzv5VBRVY,11593
|
|
87
87
|
loopmath/belief/rules.py,sha256=BRii7j8V3FYS3heX3Yj-UIaVRLXe5a0A42NpYeZ0jDM,6558
|
|
88
88
|
loopmath/belief/sampled.py,sha256=2b0riMci2ZK6AHR1w1j7mBOYxOkwxrvKlFOurVCrJCg,19931
|
|
89
|
-
loopmath/belief/state.py,sha256=
|
|
89
|
+
loopmath/belief/state.py,sha256=Uchw-u8KjhUORkC6TkCFIOA_tlP5-lnZEeY6sftDHNI,59906
|
|
90
90
|
loopmath/builder/__init__.py,sha256=E9_JLObtnM_r2to6bqH13xV3LHN00zVIRH1nzJTCv7s,105
|
|
91
91
|
loopmath/builder/cli.py,sha256=NPMkFE0vIOAsEMCtGVkvzdzmszrNd7uZ3CNX_E-zV9M,2429
|
|
92
92
|
loopmath/builder/context.py,sha256=1NCKZSx9Fn64pSixnLJqV80fPDtoPusttnj5ZnjhDrU,18387
|
|
@@ -201,11 +201,11 @@ loopmath/graph/schema.py,sha256=qWb2FZnaGU5q0CcfNpStrKwAO0Hb_p3HnMCj8fLlO7Y,6601
|
|
|
201
201
|
loopmath/graph/token_completeness.py,sha256=PWFJHbo5EZhC3vCwkLLRF-yi-hpGjP3Zw3ofoC-rvcw,916
|
|
202
202
|
loopmath/harness/__init__.py,sha256=gtDVACNxywJ1FJ6F3UBHa4qi8WDa8uAYrixYCI_62wk,6748
|
|
203
203
|
loopmath/harness/layouts.py,sha256=BLped3LFoMiJq5dJG89pHuRbBQQPVyMJiW4MX6gs5Y0,12142
|
|
204
|
-
loopmath/harness/herdr/loopmath-harness/herdr-plugin.toml,sha256=
|
|
204
|
+
loopmath/harness/herdr/loopmath-harness/herdr-plugin.toml,sha256=y9XgbwR_yBhytZ0j4ePqnSdhE2mopgOBaMJVhpZGAP0,1327
|
|
205
205
|
loopmath/harness/mod/loopmath-harness/.gitignore,sha256=OL7gSmPfqQOsSHBOkhxXX1tefIZYRfeK3Q4BoXQRi8I,144
|
|
206
206
|
loopmath/harness/mod/loopmath-harness/README.md,sha256=nAv9iCsNESht7eXozmukEGZ-cI8X8OUKSrpNiUQnO1c,2122
|
|
207
207
|
loopmath/harness/mod/loopmath-harness/.claude-plugin/marketplace.json,sha256=DgMKNP6g7CYiFi2-0ZDF5jFLR_m-RhPAciE2Rvb2GRk,173
|
|
208
|
-
loopmath/harness/mod/loopmath-harness/.claude-plugin/plugin.json,sha256=
|
|
208
|
+
loopmath/harness/mod/loopmath-harness/.claude-plugin/plugin.json,sha256=tGEJu7Atv4Xpv3GL80P47uKJajxL9xfEZOkmBPmVaV8,705
|
|
209
209
|
loopmath/harness/mod/loopmath-harness/hooks/events.ts,sha256=KI9AmK7p2D8HU_B_54HS5hvGKXUckREbg1JS8tOqUJU,14557
|
|
210
210
|
loopmath/harness/mod/loopmath-harness/hooks/hooks.json,sha256=2ELXiUdtZygtDwS35f3Gij2YHM7GXdec-i0R7tE_uCg,34
|
|
211
211
|
loopmath/harness/mod/loopmath-harness/hooks/register.test.ts,sha256=6zDRR6MKMg9V-NlPZ8Si4uVHFD2CSh1Bx6HK2jPU8xY,26319
|
|
@@ -314,7 +314,7 @@ loopmath/recommend/engine.py,sha256=YjW-D8jVYOvKdL6DtvfOgmdEb6-Xkqacmw36Q1kJNqs,
|
|
|
314
314
|
loopmath/recommend/gain.py,sha256=PPd6D3XKyPPwcE13X0Q3T82LoNEQF45KVBuabhAvKPw,7446
|
|
315
315
|
loopmath/recommend/message.py,sha256=3gADyqWzHHAeN-vFCFB2NkH3V6WNjrn9p9353jk4iXE,20062
|
|
316
316
|
loopmath/recommend/receipts.py,sha256=6peAikOTJdsKaNWnsn9D_xEwlEdyVNVvqunPJUCPbTQ,16137
|
|
317
|
-
loopmath/recommend/search.py,sha256=
|
|
317
|
+
loopmath/recommend/search.py,sha256=TlQUhHsy4Sme51HE9aEyLnbFH9-bJimDsW5qjeSDrOY,52175
|
|
318
318
|
loopmath/recommend/stats.py,sha256=ci-3jCYCHDAdaNCgXqVvu3dBRh5V7zuJP4j5Rmt3lAA,1676
|
|
319
319
|
loopmath/recommend/storeread.py,sha256=lQV37aJQ181uXRytRi1-aoDlGTvoXdUGwbn7TU_zZkE,13888
|
|
320
320
|
loopmath/recommend/variants.py,sha256=fY_3rEQMu59ibabIzOlRrJG2yQ-KfqRE91dzPQ5jndQ,8880
|
|
@@ -351,7 +351,7 @@ loopmath/skill/skills/loopmath/SKILL.md,sha256=jxp_dkTnITGzUSShiplJynl3J5QN8phk-
|
|
|
351
351
|
loopmath/store/__init__.py,sha256=vCGuqOSRKmKoT4eLbjEl8kxJRwdHWqxouGTp6gqPLFE,708
|
|
352
352
|
loopmath/store/budget.py,sha256=5ra1EuTJOQ5K7xCaN2_cdqQWbPvRkyLjcBuD5K32PvY,8526
|
|
353
353
|
loopmath/store/commands.py,sha256=dWTRI6JFseDE_FGP_78V_KLaqZEaW88-i54xZhwfx_U,89043
|
|
354
|
-
loopmath/store/config.py,sha256=
|
|
354
|
+
loopmath/store/config.py,sha256=atj6q2g2ZvCyYIC5e4VizC_UG6UOpA0mcJQz3Q_zqIo,30739
|
|
355
355
|
loopmath/store/finish.py,sha256=9EPw8ndtnQ89NGIQdS9qw_IhwTk2i6lYLSpV7GhStxQ,36832
|
|
356
356
|
loopmath/store/fitjob.py,sha256=GHBF61GyT9V-SGne-drAdaEzp2J5X8lSpgHitjHtwH0,11809
|
|
357
357
|
loopmath/store/home.py,sha256=s8ZLkTmR5uvb0CUKJlnmABOx87W75p4pbfT81n4CYY4,31865
|
|
@@ -410,12 +410,12 @@ loopmath/workflows/catalog/plan_implement.toml,sha256=qxAs6G3JGOUuW6TnhiaUXzaFfy
|
|
|
410
410
|
loopmath/workflows/catalog/plan_implement_review.toml,sha256=rRtiactglcJAynFWbrjnI-Ftbl3qe971ov9HJ3Vr7uU,862
|
|
411
411
|
loopmath/workflows/catalog/solo.toml,sha256=aJWBEEfS4DNCLqznDI3yp-7s1ZXx_rdqsEIPX7n0-nM,373
|
|
412
412
|
loopmath/workflows/catalog/swarm.toml,sha256=1hbQR41nl7VS_XN1z2fmwLCNw6kn2syowpwwauFUL3o,850
|
|
413
|
-
loopmath-0.4.
|
|
414
|
-
loopmath-0.4.
|
|
415
|
-
loopmath-0.4.
|
|
416
|
-
loopmath-0.4.
|
|
417
|
-
loopmath-0.4.
|
|
418
|
-
loopmath-0.4.
|
|
419
|
-
loopmath-0.4.
|
|
420
|
-
loopmath-0.4.
|
|
421
|
-
loopmath-0.4.
|
|
413
|
+
loopmath-0.4.2.data/data/spec/ocp-v0.2.schema.json,sha256=T4P5gg6H2dW-28MRbg9fSNct_ejXezI69-SaKR9WogU,31855
|
|
414
|
+
loopmath-0.4.2.data/data/spec/ocp-v0.schema.json,sha256=tOGqmvAjh0Ap09zV3QLQp3PovQB8inpVNgxSnrPS354,15279
|
|
415
|
+
loopmath-0.4.2.data/data/spec/ocp_conformance.py,sha256=VzRGKvC6f-64U72xrDOSJEeYa92Bpj3tEf5x0uJXqhk,893
|
|
416
|
+
loopmath-0.4.2.dist-info/licenses/LICENSE,sha256=Df5w_-_yYMWXFWcPR1Cj0pDXWBCE5_xdFN9WCNGHuGQ,1065
|
|
417
|
+
loopmath-0.4.2.dist-info/METADATA,sha256=vzjGbxbuBgoMgAjCiwvaRCFUUKdrbjFmmT90oj87Dvg,44247
|
|
418
|
+
loopmath-0.4.2.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
|
|
419
|
+
loopmath-0.4.2.dist-info/entry_points.txt,sha256=WQwYqfGAwvoQ_vu6_lwQpq3V8xbK3i9SyNKPDdQ1VmQ,72
|
|
420
|
+
loopmath-0.4.2.dist-info/top_level.txt,sha256=SH9Y0Z7thA0zx1PAgHAwc1vOJaS6o3yS5HazS-LzXOg,9
|
|
421
|
+
loopmath-0.4.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|