loopmath 0.3.4__py3-none-any.whl → 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loopmath/__init__.py +1 -1
- loopmath/belief/design.py +108 -27
- loopmath/belief/fit.py +149 -42
- loopmath/belief/forest.py +9 -3
- loopmath/belief/level.py +132 -0
- loopmath/belief/outcome.py +108 -10
- loopmath/belief/priors.py +52 -0
- loopmath/belief/rules.py +10 -7
- loopmath/cli.py +43 -3
- loopmath/cli_registry.py +367 -5
- loopmath/compare/__init__.py +259 -33
- loopmath/compare/commands.py +5 -3
- loopmath/compare/units.py +9 -5
- loopmath/flows/__init__.py +10 -0
- loopmath/flows/check.py +717 -0
- loopmath/flows/commands.py +1306 -0
- loopmath/flows/edits.py +725 -0
- loopmath/flows/folder.py +133 -0
- loopmath/flows/judges.py +843 -0
- loopmath/flows/next.py +120 -0
- loopmath/flows/plan.py +822 -0
- loopmath/flows/prefill.py +335 -0
- loopmath/flows/record.py +474 -0
- loopmath/flows/records.py +384 -0
- loopmath/flows/simulate.py +816 -0
- loopmath/flows/status.py +374 -0
- loopmath/flows/tomlview.py +666 -0
- loopmath/flows/walk.py +869 -0
- loopmath/harness/herdr/loopmath-harness/herdr-plugin.toml +38 -0
- loopmath/harness/mod/loopmath-harness/.claude-plugin/marketplace.json +6 -0
- loopmath/harness/mod/loopmath-harness/.claude-plugin/plugin.json +21 -0
- loopmath/harness/mod/loopmath-harness/.gitignore +3 -0
- loopmath/harness/mod/loopmath-harness/README.md +40 -0
- loopmath/harness/mod/loopmath-harness/hooks/events.ts +338 -0
- loopmath/harness/mod/loopmath-harness/hooks/hooks.json +1 -0
- loopmath/harness/mod/loopmath-harness/hooks/register.test.ts +469 -0
- loopmath/harness/mod/loopmath-harness/hooks/register.tsx +417 -0
- loopmath/harness/mod/loopmath-harness/types/index.d.ts +79 -0
- loopmath/harness/tune/__init__.py +13 -0
- loopmath/harness/tune/commands.py +1518 -0
- loopmath/harness/tune/config.py +391 -0
- loopmath/harness/tune/contracts.py +778 -0
- loopmath/harness/tune/delivery.py +217 -0
- loopmath/harness/tune/design.py +231 -0
- loopmath/harness/tune/editors.py +203 -0
- loopmath/harness/tune/edits.py +928 -0
- loopmath/harness/tune/effects.py +193 -0
- loopmath/harness/tune/engine.py +1243 -0
- loopmath/harness/tune/events.py +322 -0
- loopmath/harness/tune/executor.py +270 -0
- loopmath/harness/tune/herdr.py +611 -0
- loopmath/harness/tune/isolation.py +677 -0
- loopmath/harness/tune/ledger.py +165 -0
- loopmath/harness/tune/library.py +126 -0
- loopmath/harness/tune/modinstall.py +180 -0
- loopmath/harness/tune/plan.py +378 -0
- loopmath/harness/tune/prompts.py +192 -0
- loopmath/harness/tune/reflect.py +131 -0
- loopmath/harness/tune/schema.py +406 -0
- loopmath/harness/tune/score.py +445 -0
- loopmath/harness/tune/sessions.py +158 -0
- loopmath/harness/tune/skilldoc.py +180 -0
- loopmath/harness/tune/state.py +200 -0
- loopmath/harness/tune/stream.py +320 -0
- loopmath/harness_runs/__init__.py +9 -6
- loopmath/harness_runs/ocp.py +352 -36
- loopmath/harness_runs/rows.py +224 -12
- loopmath/ocp/__init__.py +6 -3
- loopmath/ocp/canonical.py +69 -8
- loopmath/ocp/commands.py +3 -2
- loopmath/ocp/conformance.py +632 -34
- loopmath/ocp/emit.py +100 -15
- loopmath/ocp/finite.py +93 -0
- loopmath/ocp/migrate.py +17 -9
- loopmath/ocp/schema/ocp-v0.4.schema.json +2778 -0
- loopmath/prices.toml +29 -3
- loopmath/priors/reduce.py +6 -2
- loopmath/priors/show.py +9 -4
- loopmath/priors/validate.py +4 -1
- loopmath/recommend/commands.py +4 -1
- loopmath/skill/commands.py +2 -2
- loopmath/skill/doctor.py +7 -4
- loopmath/skill/guide.py +2 -1
- loopmath/skill/guides/fit.md +2 -2
- loopmath/skill/guides/flow.md +409 -0
- loopmath/skill/guides/reference.md +1 -1
- loopmath/skill/install.py +3 -1
- loopmath/skill/skills/loopmath/SKILL.md +3 -2
- loopmath/store/commands.py +150 -27
- loopmath/store/config.py +20 -5
- loopmath/store/finish.py +87 -13
- loopmath/store/home.py +78 -8
- loopmath/store/ids.py +29 -0
- loopmath/store/report.py +6 -3
- loopmath/store/runs.py +70 -0
- loopmath/types.py +79 -0
- loopmath/update.py +299 -0
- loopmath/views/posterior.py +12 -6
- loopmath/workflows/candidates.py +3 -2
- loopmath/workflows/format.py +75 -6
- loopmath/workflows/ocp.py +17 -3
- {loopmath-0.3.4.dist-info → loopmath-0.4.0.dist-info}/METADATA +29 -8
- {loopmath-0.3.4.dist-info → loopmath-0.4.0.dist-info}/RECORD +110 -54
- {loopmath-0.3.4.data → loopmath-0.4.0.data}/data/spec/ocp-v0.2.schema.json +0 -0
- {loopmath-0.3.4.data → loopmath-0.4.0.data}/data/spec/ocp-v0.schema.json +0 -0
- {loopmath-0.3.4.data → loopmath-0.4.0.data}/data/spec/ocp_conformance.py +0 -0
- {loopmath-0.3.4.dist-info → loopmath-0.4.0.dist-info}/WHEEL +0 -0
- {loopmath-0.3.4.dist-info → loopmath-0.4.0.dist-info}/entry_points.txt +0 -0
- {loopmath-0.3.4.dist-info → loopmath-0.4.0.dist-info}/licenses/LICENSE +0 -0
- {loopmath-0.3.4.dist-info → loopmath-0.4.0.dist-info}/top_level.txt +0 -0
loopmath/__init__.py
CHANGED
loopmath/belief/design.py
CHANGED
|
@@ -16,7 +16,7 @@ import json
|
|
|
16
16
|
import math
|
|
17
17
|
from collections.abc import Iterable, Mapping
|
|
18
18
|
from dataclasses import dataclass, field
|
|
19
|
-
from datetime import datetime
|
|
19
|
+
from datetime import datetime, timedelta
|
|
20
20
|
from functools import lru_cache
|
|
21
21
|
from typing import Any, Sequence
|
|
22
22
|
|
|
@@ -40,7 +40,11 @@ OPT_IN_DESIGN_VERSION = 5
|
|
|
40
40
|
# The model options this loopmath's `fit` writes, each with the values it writes there (unset or `eb` writes
|
|
41
41
|
# version 4, with no options). A version 5 fit with any other key or value was made by a loopmath with an option
|
|
42
42
|
# this one has no nodes for: it is refused, as 0.3.1 refuses version 5, rather than read without them.
|
|
43
|
-
MODEL_OPTIONS: dict[str, tuple[Any, ...]] = {"belief.success_head": ("sampled",)
|
|
43
|
+
MODEL_OPTIONS: dict[str, tuple[Any, ...]] = {"belief.success_head": ("sampled",),
|
|
44
|
+
# the store level and benchmark transfer (level.py)
|
|
45
|
+
"belief.level": ("shared", "store"),
|
|
46
|
+
"belief.shipped_success": ("keep", "drop"),
|
|
47
|
+
"belief.benchmark_factors": ("reference", "offset")}
|
|
44
48
|
# The version the seed key hashes (spec 04 section 3). It stays 3 from 0.2.8: the same data seeds the same
|
|
45
49
|
# draws as in 0.2.7, so every number that is not a token count is 0.2.7's.
|
|
46
50
|
SEED_VERSION = 3
|
|
@@ -200,6 +204,27 @@ def role_terms(role: str, model: str, weight: float = 1.0) -> tuple[Term, ...]:
|
|
|
200
204
|
return ((f"role:{role}", None, weight), (f"role_family:{role}|{fam}", f"role:{role}", weight))
|
|
201
205
|
|
|
202
206
|
|
|
207
|
+
def _step_key(text: str) -> str:
|
|
208
|
+
return text.replace("%", "%25").replace("#", "%23").replace("|", "%7C")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def step_node(flow: str, piece: str, family: str) -> str:
|
|
212
|
+
"""The node of a flow's piece on a model family (SPEC 3.10): `step:<flow>#<piece>|<family>`, with `%`, `#` and
|
|
213
|
+
`|` in the flow and piece ids written `%25`, `%23` and `%7C`, so two (flow, piece) pairs never share a node.
|
|
214
|
+
Ids without those characters read as written."""
|
|
215
|
+
return f"step:{_step_key(flow)}#{_step_key(piece)}|{family}"
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
@lru_cache(maxsize=65536)
|
|
219
|
+
def step_terms(flow: str, piece: str, role: str, model: str, weight: float = 1.0) -> tuple[Term, ...]:
|
|
220
|
+
"""The step term of a flow run's piece, a child of the piece's `role_family` node (`role_terms`), at that
|
|
221
|
+
node's weight. Empty outside a flow (`flow` empty), so a run without `run.flow` keeps its rows."""
|
|
222
|
+
if not flow:
|
|
223
|
+
return ()
|
|
224
|
+
fam = model_path(model)[1]
|
|
225
|
+
return ((step_node(flow, piece, fam), f"role_family:{role}|{fam}", weight),)
|
|
226
|
+
|
|
227
|
+
|
|
203
228
|
def _s(setting: Setting) -> tuple[str, str, str]:
|
|
204
229
|
return (setting.harness or "unknown", canonical_model_id(setting.model), (setting.effort or "default").lower())
|
|
205
230
|
|
|
@@ -232,6 +257,7 @@ class Structure:
|
|
|
232
257
|
shape: str = "" # topology and position key (`shape_key`); empty means the workflow id
|
|
233
258
|
copies: dict[str, int] = field(default_factory=dict) # piece -> copies in its group (`copy_groups`)
|
|
234
259
|
factors: tuple[str, ...] = () # the run's harness factors, sorted (`spec/harness.md`); () without a record
|
|
260
|
+
flow: str = "" # the run's flow id (`run.flow.id`), for the step terms (SPEC 3.10); empty outside a flow
|
|
235
261
|
|
|
236
262
|
|
|
237
263
|
def topological_pieces(workflow: Workflow) -> list[str]:
|
|
@@ -315,8 +341,9 @@ def default_gate_rule(role: str) -> str:
|
|
|
315
341
|
return {"reviewer": "review_approve", "tester": "tests_pass", "referee": "referee_pick"}.get(role, "gate")
|
|
316
342
|
|
|
317
343
|
|
|
318
|
-
def structure(config: Configuration, factors: Iterable[str] = ()) -> Structure:
|
|
319
|
-
"""The configuration's structure; `factors` are the harness factors every piece runs with (sorted, unique)
|
|
344
|
+
def structure(config: Configuration, factors: Iterable[str] = (), flow: str | None = None) -> Structure:
|
|
345
|
+
"""The configuration's structure; `factors` are the harness factors every piece runs with (sorted, unique), and
|
|
346
|
+
`flow` the run's flow id (`ParsedRun.flow`; None outside a flow)."""
|
|
320
347
|
wf = config.workflow
|
|
321
348
|
order = topological_pieces(wf)
|
|
322
349
|
roles = {p.id: canonical_role(p.role) for p in wf.pieces}
|
|
@@ -362,7 +389,8 @@ def structure(config: Configuration, factors: Iterable[str] = ()) -> Structure:
|
|
|
362
389
|
for i in gidx:
|
|
363
390
|
gate_loop[i] = len(loops) - 1
|
|
364
391
|
return Structure(wf.id, order, roles, settings, widths, gates, loops, k_max, piece_loop, gate_loop,
|
|
365
|
-
shape_key(wf.id, roles.values()), copy_groups(order, roles, reach), tuple(sorted(set(factors)))
|
|
392
|
+
shape_key(wf.id, roles.values()), copy_groups(order, roles, reach), tuple(sorted(set(factors))),
|
|
393
|
+
flow or "")
|
|
366
394
|
|
|
367
395
|
|
|
368
396
|
# ---------------------------------------------------------------- rows
|
|
@@ -384,12 +412,12 @@ def _effort_scaled(terms: tuple[Term, ...], weight: float) -> tuple[Term, ...]:
|
|
|
384
412
|
def cost_rest(st: Structure, piece: str, k: int, setting: Setting | None = None, *,
|
|
385
413
|
source: str | None = None, effort: float = 1.0,
|
|
386
414
|
prices: Mapping[str, float] | None = None) -> tuple[Term, ...]:
|
|
387
|
-
"""Cost and tokens rows without the task part: setting, role
|
|
388
|
-
and with a `source` the position x source node
|
|
389
|
-
node `fsrc:<family>|<source>` (spec 04 section 1); the
|
|
390
|
-
harness, at 1 (`factor_terms`). `effort` is the value of the
|
|
391
|
-
effort and shape (topology, position) terms (`effort_weight`: below 1 under a
|
|
392
|
-
fsrc terms stay at 1. `prices` is a fit's price offsets (`FitState.price_offsets`, spec 04 section 2): the
|
|
415
|
+
"""Cost and tokens rows without the task part: setting, role (and in a flow run its step term, `step_terms`),
|
|
416
|
+
topology, position, round, control, and with a `source` the position x source node
|
|
417
|
+
`psrc:<shape>#<pos>|<source>` and the family x source node `fsrc:<family>|<source>` (spec 04 section 1); the
|
|
418
|
+
structure's harness factors under the piece's harness, at 1 (`factor_terms`). `effort` is the value of the
|
|
419
|
+
terms a timebox caps, the effort and shape (topology, position) terms (`effort_weight`: below 1 under a
|
|
420
|
+
timebox); the role, step, psrc and fsrc terms stay at 1. `prices` is a fit's price offsets (`FitState.price_offsets`, spec 04 section 2): the
|
|
393
421
|
model's `price:offset` term goes last. The cost head has that node and the tokens head does not, so a
|
|
394
422
|
prediction's shared row prices only its cost."""
|
|
395
423
|
s = setting or st.settings[piece]
|
|
@@ -400,7 +428,8 @@ def cost_rest(st: Structure, piece: str, k: int, setting: Setting | None = None,
|
|
|
400
428
|
shape_terms = ((f"topology:{shape}", None, 1.0), (f"position:{position}", f"topology:{shape}", 1.0))
|
|
401
429
|
if effort != 1.0:
|
|
402
430
|
st_terms, shape_terms = _effort_scaled(st_terms, float(effort)), _effort_scaled(shape_terms, float(effort))
|
|
403
|
-
|
|
431
|
+
role = st.roles[piece]
|
|
432
|
+
terms = st_terms + role_terms(role, m) + step_terms(st.flow, piece, role, m) + shape_terms
|
|
404
433
|
if st.factors:
|
|
405
434
|
terms += factor_terms(h, st.factors)
|
|
406
435
|
if source is not None:
|
|
@@ -412,17 +441,19 @@ def cost_rest(st: Structure, piece: str, k: int, setting: Setting | None = None,
|
|
|
412
441
|
|
|
413
442
|
|
|
414
443
|
def gate_rest(st: Structure, gate: GateInfo, k: int) -> tuple[Term, ...]:
|
|
415
|
-
"""Gate rows without the task part: gate rule, round, and the judged piece's setting and role
|
|
444
|
+
"""Gate rows without the task part: gate rule, round, and the judged piece's setting and role, with its step
|
|
445
|
+
term in a flow run (`step_terms`)."""
|
|
416
446
|
h, m, e = _s(st.settings[gate.judged])
|
|
447
|
+
role = st.roles[gate.judged]
|
|
417
448
|
return (((f"gate:{gate.rule}", None, 1.0),) + tuple(round_terms(k)) + setting_terms(h, m, e, 1.0, False)
|
|
418
|
-
+ role_terms(st.
|
|
449
|
+
+ role_terms(role, m) + step_terms(st.flow, gate.judged, role, m))
|
|
419
450
|
|
|
420
451
|
|
|
421
452
|
def run_rest(st: Structure) -> tuple[Term, ...]:
|
|
422
453
|
"""Success and score rows without the task part: topology, control, and per piece its model chain,
|
|
423
454
|
effort and harness (weight 1/n) with its harness factors at that weight, plus its role and role x family
|
|
424
455
|
(weight 1/g, g the copies in its group, so parallel copies carry the role effect once, as one wide piece
|
|
425
|
-
does)."""
|
|
456
|
+
does), and in a flow run its step term at the role x family weight (`step_terms`)."""
|
|
426
457
|
terms: list[Term] = [(f"topology:{st.shape or st.workflow_id}", None, 1.0)]
|
|
427
458
|
terms += _control_terms(st.k_max, max(st.widths.values() or [1]))
|
|
428
459
|
n = max(1, len(st.pieces))
|
|
@@ -431,7 +462,8 @@ def run_rest(st: Structure) -> tuple[Term, ...]:
|
|
|
431
462
|
terms += setting_terms(h, m, e, 1.0 / n)
|
|
432
463
|
if st.factors:
|
|
433
464
|
terms += factor_terms(h, st.factors, 1.0 / n)
|
|
434
|
-
|
|
465
|
+
g = 1.0 / (st.copies.get(piece) or 1)
|
|
466
|
+
terms += role_terms(st.roles[piece], m, g) + step_terms(st.flow, piece, st.roles[piece], m, g)
|
|
435
467
|
return tuple(terms)
|
|
436
468
|
|
|
437
469
|
|
|
@@ -507,6 +539,7 @@ class ParsedRun:
|
|
|
507
539
|
dropped: dict[str, int] = field(default_factory=dict)
|
|
508
540
|
checks: dict[str, int] = field(default_factory=dict) # check name -> attempts that ran no model
|
|
509
541
|
harness: dict | None = None # the run's harness record (`harness.run_record`), None without one
|
|
542
|
+
flow: str | None = None # the run's flow id (`run.flow.id`), None without `run.flow`
|
|
510
543
|
|
|
511
544
|
@property
|
|
512
545
|
def factors(self) -> tuple[str, ...]:
|
|
@@ -667,18 +700,44 @@ def config_from_doc(cfg: dict) -> Configuration:
|
|
|
667
700
|
|
|
668
701
|
|
|
669
702
|
def rule_from_doc(doc: dict, default: AcceptanceRule = DEFAULT_RULE) -> AcceptanceRule:
|
|
703
|
+
"""The run's stored rule. A score target no finite float holds, or a `window_days` no timedelta holds, is
|
|
704
|
+
refused by name (`Unusable`, a ValueError, which every reader of a malformed rule catches); any other
|
|
705
|
+
malformed score raises ValueError or TypeError as before."""
|
|
670
706
|
r = (doc.get("run") or {}).get("acceptance_rule")
|
|
671
707
|
if not isinstance(r, dict) or not r.get("name"):
|
|
672
708
|
return default
|
|
673
709
|
score = r.get("score")
|
|
674
710
|
st = None
|
|
675
711
|
if isinstance(score, dict) and score.get("name") and score.get("target") is not None:
|
|
676
|
-
st = ScoreTarget(name=str(score["name"]), target=
|
|
712
|
+
st = ScoreTarget(name=str(score["name"]), target=_rule_target(score["target"]),
|
|
677
713
|
better=str(score.get("better") or "higher"), scale=str(score.get("scale") or "linear"))
|
|
678
714
|
return AcceptanceRule(name=str(r["name"]), definition=str(r.get("definition") or r["name"]),
|
|
679
715
|
requires=tuple(r.get("requires") if r.get("requires") is not None else ("tests",)),
|
|
680
716
|
score=st, excludes_events=tuple(r.get("excludes_events") or ("revert", "incident")),
|
|
681
|
-
window_days=
|
|
717
|
+
window_days=_rule_window(r.get("window_days")))
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
def _rule_target(value: Any) -> float:
|
|
721
|
+
"""A stored score target as a float; one past the float range, infinite or NaN is refused by name."""
|
|
722
|
+
try:
|
|
723
|
+
target = float(value)
|
|
724
|
+
except OverflowError:
|
|
725
|
+
target = math.inf
|
|
726
|
+
if not math.isfinite(target):
|
|
727
|
+
raise Unusable("score target not a finite number")
|
|
728
|
+
return target
|
|
729
|
+
|
|
730
|
+
|
|
731
|
+
def _rule_window(value: Any) -> int:
|
|
732
|
+
"""A stored `window_days` (0 or none: 14); a day count past 999999999 either way (`timedelta.max.days`), or
|
|
733
|
+
an infinite float, is refused by name."""
|
|
734
|
+
try:
|
|
735
|
+
days = int(value or 14)
|
|
736
|
+
except OverflowError:
|
|
737
|
+
raise Unusable("window_days past 999999999") from None
|
|
738
|
+
if abs(days) > timedelta.max.days:
|
|
739
|
+
raise Unusable("window_days past 999999999")
|
|
740
|
+
return days
|
|
682
741
|
|
|
683
742
|
|
|
684
743
|
_PRICES = None
|
|
@@ -792,19 +851,23 @@ def _check_name(a: dict, node: dict) -> str | None:
|
|
|
792
851
|
|
|
793
852
|
|
|
794
853
|
def run_evidence(doc: dict, rule: AcceptanceRule, *, now: datetime,
|
|
795
|
-
q_by_tier: dict[str, float] | None = None
|
|
796
|
-
|
|
854
|
+
q_by_tier: dict[str, float] | None = None,
|
|
855
|
+
quality: tuple[float | None, float | None] | None = None) -> Evidence:
|
|
856
|
+
"""The run's outcome as the fit reads it: `rules.evidence` under `rule` (None `q_by_tier`: the built-in q;
|
|
857
|
+
`quality`: a flow run's combined quality, as `outcome_evidence` takes it)."""
|
|
797
858
|
from . import rules
|
|
798
859
|
|
|
799
|
-
return rules.evidence(doc, rule, now=now, q_by_tier=q_by_tier)
|
|
860
|
+
return rules.evidence(doc, rule, now=now, q_by_tier=q_by_tier, quality=quality)
|
|
800
861
|
|
|
801
862
|
|
|
802
863
|
def parse_run(doc: dict, *, now: datetime | None = None, rule: AcceptanceRule | None = None,
|
|
803
|
-
origin: str | None = None, q_by_tier: dict[str, float] | None = None
|
|
864
|
+
origin: str | None = None, q_by_tier: dict[str, float] | None = None,
|
|
865
|
+
quality: tuple[float | None, float | None] | None = None) -> ParsedRun:
|
|
804
866
|
"""Everything the fit needs from one finished OCP v0.3 run document. Raises Unusable.
|
|
805
867
|
|
|
806
868
|
`origin` is where the fit found the document (`data_source`); None reads its label. `q_by_tier` is
|
|
807
|
-
`outcome.q.<tier>` from config.toml, passed to `outcome_evidence` (None: the built-in q)
|
|
869
|
+
`outcome.q.<tier>` from config.toml, passed to `outcome_evidence` (None: the built-in q), and so is
|
|
870
|
+
`quality`, a flow run's combined quality.
|
|
808
871
|
"""
|
|
809
872
|
run = doc.get("run") or {}
|
|
810
873
|
cfg = run.get("configuration")
|
|
@@ -869,7 +932,19 @@ def parse_run(doc: dict, *, now: datetime | None = None, rule: AcceptanceRule |
|
|
|
869
932
|
from .rules import resolve_rule
|
|
870
933
|
|
|
871
934
|
use_rule = rule or resolve_rule(doc) # one resolver for every reader (0.2.7); raises on a malformed score
|
|
872
|
-
evidence = run_evidence(doc, use_rule, now=now or datetime.now().astimezone(), q_by_tier=q_by_tier
|
|
935
|
+
evidence = run_evidence(doc, use_rule, now=now or datetime.now().astimezone(), q_by_tier=q_by_tier,
|
|
936
|
+
quality=quality)
|
|
937
|
+
flow = (run.get("flow") or {}).get("id") if isinstance(run.get("flow"), dict) else None
|
|
938
|
+
return ParsedRun(str(run.get("id") or task.id), task, config, source, attempts, gates, evidence,
|
|
939
|
+
run_score_meta(doc, evidence, use_rule), use_rule, dropped, checks, harness,
|
|
940
|
+
flow if isinstance(flow, str) and flow else None)
|
|
941
|
+
|
|
942
|
+
|
|
943
|
+
def run_score_meta(doc: dict, evidence: Evidence, rule: AcceptanceRule) -> dict[str, dict]:
|
|
944
|
+
"""Each score's unit, direction and scale, from the run's first signal of that name. A flow run's combined
|
|
945
|
+
quality (`outcome_evidence`) is a fraction in the rule's direction; a score name a judge scored with no
|
|
946
|
+
combined value has no entry."""
|
|
947
|
+
run = doc.get("run") or {}
|
|
873
948
|
score_meta: dict[str, dict] = {}
|
|
874
949
|
for s in run.get("signals") or []:
|
|
875
950
|
if s.get("kind") == "score" and isinstance(s.get("value"), (int, float)) and not isinstance(s.get("value"), bool):
|
|
@@ -881,8 +956,14 @@ def parse_run(doc: dict, *, now: datetime | None = None, rule: AcceptanceRule |
|
|
|
881
956
|
score_meta.setdefault(str(name), dict(meta))
|
|
882
957
|
for name in (evidence.scores or {}):
|
|
883
958
|
score_meta.setdefault(str(name), {"unit": None, "better": "higher", "scale": "linear"})
|
|
884
|
-
|
|
885
|
-
|
|
959
|
+
elif isinstance(run.get("flow"), dict) and rule.score is not None:
|
|
960
|
+
from ..flows import judges
|
|
961
|
+
|
|
962
|
+
if any(judges.judged(doc, rule.score.name)):
|
|
963
|
+
score_meta.pop(rule.score.name, None)
|
|
964
|
+
if rule.score.name in (evidence.scores or {}):
|
|
965
|
+
score_meta[rule.score.name] = {"unit": None, "better": rule.score.better, "scale": "fraction"}
|
|
966
|
+
return score_meta
|
|
886
967
|
|
|
887
968
|
|
|
888
969
|
def _gate_observations(doc: dict, st: Structure, by_piece_round: dict) -> list[tuple[int, int, bool]]:
|
|
@@ -951,7 +1032,7 @@ def _gate_observations(doc: dict, st: Structure, by_piece_round: dict) -> list[t
|
|
|
951
1032
|
def rows_for_run(run_doc: dict) -> dict:
|
|
952
1033
|
"""Fitting rows for one run document: {cost, tokens, gate, success, scores} plus its structure."""
|
|
953
1034
|
pr = parse_run(run_doc)
|
|
954
|
-
st = structure(pr.config, pr.factors)
|
|
1035
|
+
st = structure(pr.config, pr.factors, pr.flow)
|
|
955
1036
|
out: dict[str, Any] = {"run": pr.run_id, "source": pr.source, "structure": st,
|
|
956
1037
|
"cost": [], "tokens": [], "gate": [], "success": None, "scores": {}}
|
|
957
1038
|
for a in pr.attempts:
|
loopmath/belief/fit.py
CHANGED
|
@@ -39,6 +39,8 @@ from .forest import Forest, default_scale, fixed_sd, scale_group
|
|
|
39
39
|
from .pricing import PRICE_NODE, PRICE_PRIOR_SD, STREAMS, fit_offsets, price_term, rate_floors
|
|
40
40
|
from .gaussian import N_DRAWS, Factors, GaussianHead, HeadFit, LogisticHead, Prior
|
|
41
41
|
from .integrated import SUCCESS_HEAD_KEY, integrate_success, success_method
|
|
42
|
+
from .level import INTERCEPT_SD, USER_LEVEL, LevelSettings, level_settings, shipped
|
|
43
|
+
from .outcome import unreadable_signals
|
|
42
44
|
from .state import SCORE_CLAMP, inverse_transform, transform # noqa: F401 (re-exported)
|
|
43
45
|
|
|
44
46
|
FIT_LOCK = ".fit.lock"
|
|
@@ -72,6 +74,7 @@ class UnknownFit(LookupError):
|
|
|
72
74
|
FIXED_SOURCES = ("benchmark", "shared", "user")
|
|
73
75
|
|
|
74
76
|
SHIPPED_COPY = "shipped copy of a stored run"
|
|
77
|
+
RUN_NOT_OBJECT = "unreadable: run not an object"
|
|
75
78
|
|
|
76
79
|
REASON_NOTES = {
|
|
77
80
|
"attempt without usable cost": "crashed or no usage reported",
|
|
@@ -189,11 +192,71 @@ def _excluded(source: str, without: tuple[str, ...]) -> bool:
|
|
|
189
192
|
return False
|
|
190
193
|
|
|
191
194
|
|
|
195
|
+
def admitted(docs: Iterable[dict | tuple[str | None, dict]], *, without: tuple[str, ...] = (),
|
|
196
|
+
now: datetime | None = None, q_by_tier: dict[str, float] | None = None,
|
|
197
|
+
dropped: Counter | None = None, overlap: Counter | None = None):
|
|
198
|
+
"""The documents `collect_rows` reads, as `(origin, document, ParsedRun)` in order, with its admission rules:
|
|
199
|
+
origins in `without`, the store's copy over a shipped one, unusable documents, repeated run ids and sources in
|
|
200
|
+
`without`. `dropped` and `overlap` take the counts `collect_rows` reports. A flow run is parsed with no
|
|
201
|
+
combined quality (`quality=(None, None)`), so admission fits nothing. A document or `run` that is not an
|
|
202
|
+
object is unreadable before any rule reads its run id; a run whose signals no reader can walk
|
|
203
|
+
(`outcome.unreadable_signals`) is unreadable before it is parsed."""
|
|
204
|
+
dropped = Counter() if dropped is None else dropped
|
|
205
|
+
overlap = Counter() if overlap is None else overlap
|
|
206
|
+
now = now or datetime.now().astimezone()
|
|
207
|
+
seen: set[str] = set()
|
|
208
|
+
stored: set[str] = set() # run ids of the user's store, whose copy wins
|
|
209
|
+
for item in docs:
|
|
210
|
+
origin, doc = item if isinstance(item, tuple) else (None, item)
|
|
211
|
+
if not isinstance(doc, dict) or not isinstance(doc.get("run", {}), dict):
|
|
212
|
+
dropped[RUN_NOT_OBJECT] += 1 # no run id to read, so before every rule that reads one
|
|
213
|
+
continue
|
|
214
|
+
if origin is not None and _excluded(origin, without):
|
|
215
|
+
if origin != "user" and prior_data.run_id_of(doc) in stored:
|
|
216
|
+
overlap[origin] += 1
|
|
217
|
+
dropped[f"without {origin}"] += 1 # counted, not parsed; `--without user` frees the shipped copies
|
|
218
|
+
continue
|
|
219
|
+
if origin == "user":
|
|
220
|
+
stored.add(prior_data.run_id_of(doc))
|
|
221
|
+
elif origin is not None:
|
|
222
|
+
if prior_data.run_id_of(doc) in stored:
|
|
223
|
+
overlap[origin] += 1
|
|
224
|
+
dropped[SHIPPED_COPY] += 1
|
|
225
|
+
continue
|
|
226
|
+
shape = unreadable_signals(doc)
|
|
227
|
+
if shape is not None:
|
|
228
|
+
dropped[shape] += 1
|
|
229
|
+
continue
|
|
230
|
+
try:
|
|
231
|
+
pr = parse_run(doc, now=now, origin=origin, q_by_tier=q_by_tier, quality=(None, None))
|
|
232
|
+
except Unusable as exc:
|
|
233
|
+
dropped[str(exc)] += 1
|
|
234
|
+
continue
|
|
235
|
+
except (KeyError, TypeError, ValueError) as exc:
|
|
236
|
+
dropped[f"unreadable: {type(exc).__name__}"] += 1
|
|
237
|
+
continue
|
|
238
|
+
if pr.run_id in seen:
|
|
239
|
+
dropped["duplicate run id"] += 1
|
|
240
|
+
continue
|
|
241
|
+
seen.add(pr.run_id)
|
|
242
|
+
if _excluded(pr.source, without):
|
|
243
|
+
dropped[f"without {pr.source}"] += 1
|
|
244
|
+
continue
|
|
245
|
+
yield origin, doc, pr
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _flow_run(doc: dict) -> bool:
|
|
249
|
+
return isinstance((doc.get("run") or {}).get("flow"), dict)
|
|
250
|
+
|
|
251
|
+
|
|
192
252
|
def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tuple[str, ...] = (),
|
|
193
253
|
now: datetime | None = None, features: FeatureSet | None = None,
|
|
194
|
-
q_by_tier: dict[str, float] | None = None):
|
|
254
|
+
q_by_tier: dict[str, float] | None = None, own_success: bool = False):
|
|
195
255
|
"""Rows of every head from run documents, plus counts of what was used and dropped.
|
|
196
256
|
|
|
257
|
+
`own_success` (config `belief.shipped_success = "drop"`): shipped runs give no success or score rows, so
|
|
258
|
+
the success level comes from the user's runs and shared imports only (`level.py`).
|
|
259
|
+
|
|
197
260
|
`docs` yields documents, or `(origin, document)` pairs from `fit()`: `user` for the store
|
|
198
261
|
(read first), the manifest source for a shipped run, None to read the document's label
|
|
199
262
|
(spec 04 section 1). A shipped run whose id is in the store is left out as
|
|
@@ -208,6 +271,13 @@ def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tup
|
|
|
208
271
|
built-ins by default); only admitted values keep their `feature:` terms, and `tally`
|
|
209
272
|
(features.FeatureTally) holds the support, the admitted values and the recorded horizons.
|
|
210
273
|
`q_by_tier` (`outcome.q.<tier>` from config.toml) reaches each run's outcome; None is the built-in q.
|
|
274
|
+
|
|
275
|
+
Runs are admitted by `admitted`. A flow run's success and score rows follow the other runs' rows: its score
|
|
276
|
+
is its unpooled estimate under the current judges fit of the admitted flow runs (`flows.judges.quality_map`),
|
|
277
|
+
the numbers `flows.judges.current_quality` gives every other reader. Each judge score the reader skips is
|
|
278
|
+
counted under its reason's count (`flows.judges.DROP_COUNTS`: unranged, unattributed, outside-range,
|
|
279
|
+
nonfinite and nonnumeric), one per judge, or per signal when unattributed, and a run whose fit was refused for
|
|
280
|
+
its number of judges as `judge scores of a fit refused for too many judges`.
|
|
211
281
|
"""
|
|
212
282
|
now = now or datetime.now().astimezone()
|
|
213
283
|
features = features or FeatureSet()
|
|
@@ -221,42 +291,14 @@ def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tup
|
|
|
221
291
|
checks: Counter = Counter()
|
|
222
292
|
runs_by_source: Counter = Counter()
|
|
223
293
|
config_support: dict[str, Counter] = defaultdict(Counter)
|
|
224
|
-
seen: set[str] = set()
|
|
225
|
-
stored: set[str] = set() # run ids of the user's store, whose copy wins
|
|
226
294
|
overlap: Counter = Counter()
|
|
227
295
|
labels: Counter = Counter()
|
|
228
296
|
model_runs: dict[str, set[str]] = defaultdict(set) # the price offset's inputs (spec 04 section 2)
|
|
229
297
|
model_streams: dict[str, list[float]] = defaultdict(lambda: [0.0] * len(STREAMS))
|
|
230
298
|
variants: dict[str, dict] = {} # the harness records read, by digest (`spec/harness.md`)
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
if origin != "user" and prior_data.run_id_of(doc) in stored:
|
|
235
|
-
overlap[origin] += 1
|
|
236
|
-
dropped[f"without {origin}"] += 1 # counted, not parsed; `--without user` frees the shipped copies
|
|
237
|
-
continue
|
|
238
|
-
if origin == "user":
|
|
239
|
-
stored.add(prior_data.run_id_of(doc))
|
|
240
|
-
elif origin is not None:
|
|
241
|
-
if prior_data.run_id_of(doc) in stored:
|
|
242
|
-
overlap[origin] += 1
|
|
243
|
-
dropped[SHIPPED_COPY] += 1
|
|
244
|
-
continue
|
|
245
|
-
try:
|
|
246
|
-
pr = parse_run(doc, now=now, origin=origin, q_by_tier=q_by_tier)
|
|
247
|
-
except Unusable as exc:
|
|
248
|
-
dropped[str(exc)] += 1
|
|
249
|
-
continue
|
|
250
|
-
except (KeyError, TypeError, ValueError) as exc:
|
|
251
|
-
dropped[f"unreadable: {type(exc).__name__}"] += 1
|
|
252
|
-
continue
|
|
253
|
-
if pr.run_id in seen:
|
|
254
|
-
dropped["duplicate run id"] += 1
|
|
255
|
-
continue
|
|
256
|
-
seen.add(pr.run_id)
|
|
257
|
-
if _excluded(pr.source, without):
|
|
258
|
-
dropped[f"without {pr.source}"] += 1
|
|
259
|
-
continue
|
|
299
|
+
flow_runs: list[tuple] = [] # (document, ParsedRun, run row, task type): success and score rows come last
|
|
300
|
+
for _, doc, pr in admitted(docs, without=without, now=now, q_by_tier=q_by_tier, dropped=dropped,
|
|
301
|
+
overlap=overlap):
|
|
260
302
|
if pr.source == "user" and pr.task.extra.get("source_label"):
|
|
261
303
|
labels[pr.task.extra["source_label"]] += 1
|
|
262
304
|
for reason, n in pr.dropped.items():
|
|
@@ -266,7 +308,7 @@ def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tup
|
|
|
266
308
|
tally.add(pr.task, task_features(pr.task, features), pr.run_id, pr.source)
|
|
267
309
|
chain = [n for n, _, _ in task_terms(pr.task, pr.source) if n.split(":", 1)[0] in TASK_LEVELS]
|
|
268
310
|
config_support[pr.config.id].update(chain + ["all"])
|
|
269
|
-
st = structure(pr.config, pr.factors)
|
|
311
|
+
st = structure(pr.config, pr.factors, pr.flow)
|
|
270
312
|
if pr.harness is not None:
|
|
271
313
|
_add_variant(variants, pr, st)
|
|
272
314
|
ttype = pr.task.type
|
|
@@ -288,7 +330,12 @@ def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tup
|
|
|
288
330
|
heads["gate"].add(gate_row(pr.task, pr.source, st, st.gates[gi], k, features), 1.0 if passed else 0.0,
|
|
289
331
|
1.0, pr.run_id, pr.source, ttype)
|
|
290
332
|
row = run_row(pr.task, pr.source, st, features)
|
|
333
|
+
if _flow_run(doc):
|
|
334
|
+
flow_runs.append((doc, pr, row, ttype))
|
|
335
|
+
continue
|
|
291
336
|
ev = pr.evidence
|
|
337
|
+
if own_success and shipped(pr.source):
|
|
338
|
+
ev = None
|
|
292
339
|
if ev is not None and ev.z is not None:
|
|
293
340
|
heads["success"].add(row, float(ev.z), float(ev.q), pr.run_id, pr.source, ttype)
|
|
294
341
|
if configured_q: # z and the tier do not depend on q, so this read differs from `ev` in q only
|
|
@@ -297,6 +344,30 @@ def collect_rows(docs: Iterable[dict | tuple[str | None, dict]], *, without: tup
|
|
|
297
344
|
meta = pr.score_meta.get(name) or {}
|
|
298
345
|
score_meta[name][(meta.get("scale") or "linear", meta.get("better") or "higher", meta.get("unit"))] += 1
|
|
299
346
|
score_raw[name].append((row, float(value), pr.run_id, pr.source, ttype))
|
|
347
|
+
if flow_runs:
|
|
348
|
+
from ..flows import judges
|
|
349
|
+
|
|
350
|
+
quality = judges.quality_map([(doc, pr.rule) for doc, pr, _, _ in flow_runs])
|
|
351
|
+
for doc, pr, row, ttype in flow_runs:
|
|
352
|
+
if own_success and shipped(pr.source):
|
|
353
|
+
continue # as for every other shipped run: no success or score rows
|
|
354
|
+
value = quality.get(pr.run_id, (None, None))
|
|
355
|
+
ev = design_rows.run_evidence(doc, pr.rule, now=now, q_by_tier=q_by_tier, quality=value)
|
|
356
|
+
for reason in judges.judged(doc, pr.rule.score.name).reasons if pr.rule.score is not None else ():
|
|
357
|
+
dropped[judges.DROP_COUNTS[reason.split(":", 1)[0]]] += 1
|
|
358
|
+
if getattr(value, "reason", None):
|
|
359
|
+
dropped[judges.REFUSED] += 1
|
|
360
|
+
if ev.z is not None:
|
|
361
|
+
heads["success"].add(row, float(ev.z), float(ev.q), pr.run_id, pr.source, ttype)
|
|
362
|
+
if configured_q:
|
|
363
|
+
heads["success"].seed_w.append(
|
|
364
|
+
float(design_rows.run_evidence(doc, pr.rule, now=now, quality=value).q))
|
|
365
|
+
metas = design_rows.run_score_meta(doc, ev, pr.rule)
|
|
366
|
+
for name, v in (ev.scores or {}).items():
|
|
367
|
+
meta = metas.get(name) or {}
|
|
368
|
+
score_meta[name][(meta.get("scale") or "linear", meta.get("better") or "higher",
|
|
369
|
+
meta.get("unit"))] += 1
|
|
370
|
+
score_raw[name].append((row, float(v), pr.run_id, pr.source, ttype))
|
|
300
371
|
for name, items in score_raw.items():
|
|
301
372
|
scale, better, unit = score_meta[name].most_common(1)[0][0]
|
|
302
373
|
hr = HeadRows(f"score:{name}", "gaussian", "score")
|
|
@@ -394,11 +465,28 @@ def build_matrix(rows: list[list[Term]], forest: Forest, extra_nodes: Iterable[t
|
|
|
394
465
|
return X, list(local)
|
|
395
466
|
|
|
396
467
|
|
|
397
|
-
def make_prior(node_ids: list[str], kind: str) -> Prior:
|
|
468
|
+
def make_prior(node_ids: list[str], kind: str, level: LevelSettings | None = None) -> Prior:
|
|
469
|
+
"""Under a store level (`level.py`) every `source:` node is fixed at sd `level_sd`, and the intercept is held at 0
|
|
470
|
+
on the logit and score heads. The cost-kind heads (cost, rate) keep 0.3.5's free intercept, which carries the
|
|
471
|
+
shipped runs' level of log dollars and log dollars per token, so each source node is a deviation from it and a
|
|
472
|
+
provider with no runs is priced at that level, not at e^0."""
|
|
398
473
|
groups: list[str] = []
|
|
399
474
|
gidx = np.full(len(node_ids), -1, dtype=np.int64)
|
|
400
475
|
fvar = np.ones(len(node_ids))
|
|
476
|
+
store = level is not None and level.store
|
|
401
477
|
for j, node in enumerate(node_ids):
|
|
478
|
+
if store and node.startswith("source:"):
|
|
479
|
+
fvar[j] = level.level_sd ** 2
|
|
480
|
+
continue
|
|
481
|
+
if store and node == "fixed:intercept" and kind != "cost":
|
|
482
|
+
fvar[j] = INTERCEPT_SD ** 2
|
|
483
|
+
continue
|
|
484
|
+
if level is not None and node.startswith("btx:"): # the transfer error, tempered with the benchmark
|
|
485
|
+
fvar[j] = (level.gap_scale * level.transfer_sd) ** 2 / max(level.alpha, 1e-9)
|
|
486
|
+
continue
|
|
487
|
+
if level is not None and node.startswith("btxe:"): # an effort's own transfer error
|
|
488
|
+
fvar[j] = (level.gap_scale * level.effort_sd) ** 2 / max(level.alpha, 1e-9)
|
|
489
|
+
continue
|
|
402
490
|
g = scale_group(node)
|
|
403
491
|
if g is None:
|
|
404
492
|
fvar[j] = fixed_sd(node, kind) ** 2
|
|
@@ -479,11 +567,15 @@ def input_key(heads_rows: dict[str, HeadRows], specs: list, settings: dict) -> s
|
|
|
479
567
|
return h.hexdigest()
|
|
480
568
|
|
|
481
569
|
|
|
482
|
-
def fit_head(hr: HeadRows, forest: Forest, factor_specs: list, *, eb: bool = True
|
|
483
|
-
|
|
570
|
+
def fit_head(hr: HeadRows, forest: Forest, factor_specs: list, *, eb: bool = True,
|
|
571
|
+
level: LevelSettings | None = None) -> dict:
|
|
572
|
+
"""Fit one head; returns arrays and metadata for the state files. Under a store level the head always
|
|
573
|
+
holds `source:user`, so a store without runs predicts at its wide prior level."""
|
|
484
574
|
extra = [(n, p) for spec in factor_specs for n, p, _ in spec.terms]
|
|
575
|
+
if level is not None and level.store:
|
|
576
|
+
extra.append((USER_LEVEL, None))
|
|
485
577
|
X, node_ids = build_matrix(hr.rows, forest, extra)
|
|
486
|
-
prior = make_prior(node_ids, hr.kind)
|
|
578
|
+
prior = make_prior(node_ids, hr.kind, level)
|
|
487
579
|
factors = make_factors(factor_specs, node_ids)
|
|
488
580
|
# leaves first (block.py): no two of the leading nodes share a row or a factor, so the stored
|
|
489
581
|
# precision's Cholesky factor is the block factor, rebuilt in O(c^3) by the state
|
|
@@ -628,6 +720,7 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
628
720
|
|
|
629
721
|
# `outcome.q.<tier>`, read as `run finish` reads it; unset, the built-in q (0.2.6, fix (1))
|
|
630
722
|
q_by_tier = {tier: Config(home / "config.toml", config).q_for_tier(tier) for tier in TIER_Q}
|
|
723
|
+
level = level_settings(config)
|
|
631
724
|
|
|
632
725
|
def all_docs() -> Iterator[dict | tuple[str | None, dict]]:
|
|
633
726
|
if docs is not None:
|
|
@@ -641,7 +734,7 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
641
734
|
|
|
642
735
|
(heads_rows, score_info, dropped, runs_by_source, config_support, checks, overlap,
|
|
643
736
|
labels, tally, variants) = collect_rows(all_docs(), without=without, now=now, features=features,
|
|
644
|
-
q_by_tier=q_by_tier)
|
|
737
|
+
q_by_tier=q_by_tier, own_success=level.own_success)
|
|
645
738
|
removed = {k[len("without "):] for k in dropped if k.startswith("without ")}
|
|
646
739
|
unknown, known = unknown_sources(without, set(runs_by_source) | removed, bundle_dir)
|
|
647
740
|
if unknown:
|
|
@@ -652,13 +745,19 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
652
745
|
if not no_prior and "benchmark" not in without:
|
|
653
746
|
specs = prior_data.benchmark_factors(benchmarks, weight=weight)
|
|
654
747
|
bench_weights = prior_data.benchmark_weights(benchmarks, weight=weight)
|
|
655
|
-
|
|
656
|
-
|
|
748
|
+
if level.benchmarks == "offset": # success factors by offset and scale; cost and tokens as before
|
|
749
|
+
specs = [s for s in specs if s.head != "success"] + prior_data.benchmark_offset_factors(
|
|
750
|
+
benchmarks, alpha=level.alpha, gap_scale=level.gap_scale, transfer_sd=level.transfer_sd,
|
|
751
|
+
effort_sd=level.effort_sd, weight=weight)
|
|
752
|
+
key_settings = {"no_prior": no_prior, "without": sorted(without), "eb": eb, "benchmark_prior_weight": weight}
|
|
753
|
+
if level.key(): # 0.3.5's defaults keep 0.3.5's seed keys
|
|
754
|
+
key_settings["level"] = repr(sorted(level.key().items()))
|
|
755
|
+
seed_key = input_key(heads_rows, specs, key_settings)
|
|
657
756
|
forest = Forest()
|
|
658
757
|
fitted = {}
|
|
659
758
|
for name, hr in heads_rows.items():
|
|
660
759
|
head_specs = [s for s in specs if s.head == name]
|
|
661
|
-
if not hr.rows:
|
|
760
|
+
if not hr.rows and not (level.store and name == "success"): # a store level fits success unseen
|
|
662
761
|
continue
|
|
663
762
|
nodes = horizon_nodes(hr.rows)
|
|
664
763
|
if nodes: # the horizon priors, --no-prior included (spec 04 section 1)
|
|
@@ -668,7 +767,9 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
668
767
|
if name in KEY_ONLY: # 0.2.7's tokens head: its nodes enter the index where they did, unfitted
|
|
669
768
|
build_matrix(hr.rows, forest, [(n, p) for spec in head_specs for n, p, _ in spec.terms])
|
|
670
769
|
continue
|
|
671
|
-
|
|
770
|
+
# 0.3.5's call when no level key is set
|
|
771
|
+
fitted[name] = (fit_head(hr, forest, head_specs, eb=eb, level=level.for_head(name)) if level.key()
|
|
772
|
+
else fit_head(hr, forest, head_specs, eb=eb))
|
|
672
773
|
fit_id = new_fit_id(fits, now)
|
|
673
774
|
partial = fits / f"{fit_id}.partial"
|
|
674
775
|
partial.mkdir(parents=True)
|
|
@@ -751,6 +852,12 @@ def fit(home: Path, *, no_prior: bool = False, without: tuple[str, ...] = (), fu
|
|
|
751
852
|
meta.setdefault("model_options", {})[SUCCESS_HEAD_KEY] = success_head
|
|
752
853
|
if "success" in fitted:
|
|
753
854
|
meta["heads"]["success"]["method"] = success_head
|
|
855
|
+
if level.key(): # a store level or benchmark transfer: older loopmath refuses the fit (design version 5)
|
|
856
|
+
meta["design_version"] = design_rows.OPT_IN_DESIGN_VERSION
|
|
857
|
+
meta.setdefault("model_options", {}).update({"belief.level": level.level,
|
|
858
|
+
"belief.shipped_success": level.shipped_success,
|
|
859
|
+
"belief.benchmark_factors": level.benchmarks})
|
|
860
|
+
meta["options"]["level"] = level.key()
|
|
754
861
|
meta["seconds"] = round(time.monotonic() - started, 3)
|
|
755
862
|
_atomic_json(partial / "meta.json", meta)
|
|
756
863
|
final = fits / fit_id
|
loopmath/belief/forest.py
CHANGED
|
@@ -9,10 +9,11 @@ data then predicts its parent's belief widened by `phi`, as the tree law says.
|
|
|
9
9
|
Node ids are `level:key`. Keys nest along a chain so that each node has one parent:
|
|
10
10
|
`type:feature`, `repo:feature/owner/name`, `subtype:feature/owner/name/api`, `task:<id>`;
|
|
11
11
|
`provider:anthropic`, `family:opus`, `model:claude-opus-5-5`, `fsrc:opus|sweep`; `effort:high`,
|
|
12
|
-
`family_effort:opus|high`; `role:reviewer`, `role_family:reviewer|opus
|
|
12
|
+
`family_effort:opus|high`; `role:reviewer`, `role_family:reviewer|opus`, `step:lit-brief#critic|opus` (a flow's
|
|
13
|
+
piece under its role and family, flow runs only);
|
|
13
14
|
`topology:implement_review`, `position:implement_review#1`, `psrc:implement_review#1|sweep`;
|
|
14
15
|
`feature:size=s`;
|
|
15
|
-
`source:sweep`; `org:<org>` (crossed, since org is optional); `gate:<rule>`;
|
|
16
|
+
`source:sweep`; `org:<org>` (crossed, since org is optional); `gate:<rule>`; `bench:<id>` (level.py);
|
|
16
17
|
`hfactor:claude-code|skill:pdf`, a harness factor under `harness:claude-code` (`spec/harness.md`).
|
|
17
18
|
Fixed effects (wide priors, no scale fit): `fixed:intercept`, `round:2`, `round:3+`,
|
|
18
19
|
`control:budget`, `control:width`.
|
|
@@ -40,6 +41,7 @@ DEFAULT_SCALES: dict[str, tuple[float, float, float]] = {
|
|
|
40
41
|
"harness": (0.3, 0.5, 0.5),
|
|
41
42
|
"role": (0.5, 0.5, 0.5),
|
|
42
43
|
"role_family": (0.3, 0.5, 0.5),
|
|
44
|
+
"step": (0.3, 0.5, 0.5), # a flow's piece on a family, under its role_family node (SPEC 3.10)
|
|
43
45
|
"topology": (0.5, 0.7, 0.7),
|
|
44
46
|
"position": (0.2, 0.3, 0.3),
|
|
45
47
|
"psrc": (0.7, 0.7, 0.7), # position x source, on the cost and tokens heads only
|
|
@@ -58,6 +60,9 @@ FIXED_SD: dict[str, tuple[float, float, float]] = {
|
|
|
58
60
|
"control": (1.0, 1.0, 1.0),
|
|
59
61
|
"horizon": (10.0, 10.0, 10.0), # wide: the informative prior is a FactorSpec in fit() (spec 04 section 1)
|
|
60
62
|
"price": (10.0, 10.0, 10.0), # wide: fit() holds `price:offset` at 1 by a FactorSpec (spec 04 section 2)
|
|
63
|
+
"bench": (10.0, 5.0, 5.0), # a benchmark's own offset (level.py): free, so a benchmark never sets the level
|
|
64
|
+
"btx": (0.6, 0.6, 0.6), # a model's transfer error on one benchmark (level.py); fit.make_prior sets it
|
|
65
|
+
"btxe": (0.5, 0.5, 0.5), # an effort's own transfer error on one benchmark (level.py); fit.make_prior sets it
|
|
61
66
|
}
|
|
62
67
|
|
|
63
68
|
HYPER_SD = 0.7 # sd of the log-normal hyperprior on each phi, around the defaults above
|
|
@@ -70,6 +75,7 @@ PARENT_LEVEL = {
|
|
|
70
75
|
"model": "family",
|
|
71
76
|
"family_effort": "effort",
|
|
72
77
|
"role_family": "role",
|
|
78
|
+
"step": "role_family",
|
|
73
79
|
"position": "topology",
|
|
74
80
|
"psrc": "position",
|
|
75
81
|
"fsrc": "family",
|
|
@@ -80,7 +86,7 @@ PARENT_LEVEL = {
|
|
|
80
86
|
LEVEL_ALIASES = {
|
|
81
87
|
"model": ("provider", "family", "model", "fsrc"),
|
|
82
88
|
"effort": ("effort", "family_effort"),
|
|
83
|
-
"role": ("role", "role_family"),
|
|
89
|
+
"role": ("role", "role_family", "step"),
|
|
84
90
|
"topology": ("topology", "position", "psrc"),
|
|
85
91
|
"harness": ("harness", "hfactor"),
|
|
86
92
|
}
|