forge-play 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
forge/__init__.py ADDED
@@ -0,0 +1 @@
1
+ """The Forge — a refuse-a-confident-wrong-answer harness (the engine)."""
forge/_ids.py ADDED
@@ -0,0 +1,32 @@
1
+ """forge/core/_ids.py — builder-id charset (vendored from safe-app-store principal.py).
2
+
3
+ The model side keys every per-builder record on a builder_id; principal.py (D2/D11)
4
+ is the one place that charset is defined, and checkpoint_memory imported
5
+ `principal._check_builder_id` from it. Rather than drag the 900-line SAFE identity
6
+ core into a dependency-light library, this vendors the ~10 lines actually used —
7
+ the same "vendor the primitive, not the package" discipline calibration/human_loop/
8
+ friction_floor follow. Bound as `principal` at the import site so the reference is
9
+ unchanged. If principal.py's charset moves, reconcile this copy.
10
+ """
11
+ from __future__ import annotations
12
+
13
+ import re
14
+ from typing import Any
15
+
16
+ _ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
17
+ _MAX_BUILDER_ID_LEN = 128
18
+
19
+
20
+ class PrincipalError(Exception):
21
+ """Fail-closed refusal — a builder_id that is not a str or fails the
22
+ path-safety charset (D11)."""
23
+
24
+
25
+ def _check_builder_id(builder_id: Any) -> str:
26
+ if not isinstance(builder_id, str):
27
+ raise PrincipalError(f"builder_id must be a str, got {type(builder_id).__name__}")
28
+ if not builder_id or not _ID_PATTERN.match(builder_id):
29
+ raise PrincipalError(f"builder_id {builder_id!r} fails the path-safety charset (D11)")
30
+ if len(builder_id) > _MAX_BUILDER_ID_LEN:
31
+ raise PrincipalError(f"builder_id is longer than {_MAX_BUILDER_ID_LEN} characters")
32
+ return builder_id
forge/build_loop.py ADDED
@@ -0,0 +1,206 @@
1
+ #!/usr/bin/env python3
2
+ """forge/build_loop.py — resolve a plan's decisions through memory, and find
3
+ out whether the proposer's confidence was earned.
4
+
5
+ decision_extract → run_checkpoint (per decision, in plan order) → the
6
+ answer substituted back into the plan → a plan with no fork left in it.
7
+
8
+ The model never touches the router: extraction is rule-based, the band comes
9
+ from the maker's own memory, and a fork's `recommended` is only ever a
10
+ suggestion the deferral path may fall back to. What the model DOES get is a
11
+ scorecard. For every fork that carried a `recommended` and a `confidence`:
12
+
13
+ before the ask calibration_ledger.record_prediction(
14
+ claim="<decision_type>: maker picks <recommended>",
15
+ confidence)
16
+ after the ask calibration_ledger.resolve_prediction(
17
+ outcome = (chosen == recommended))
18
+
19
+ Ground truth that arrives on its own — the maker's pick IS the outcome, no
20
+ opinion enters — which is the loop the Forge exists to close and which, until
21
+ this module, nothing outside the ledger's own tests had ever called. A fork
22
+ without a confidence records nothing: no fabricated 0.5.
23
+
24
+ Refusals: a plan whose extraction refused a fork cannot be resolved (the
25
+ reasons are in the error); a plan that still holds a fork after resolution is
26
+ a bug and is refused rather than returned; a memory answer that names no
27
+ option on offer is refused rather than guessed at.
28
+ """
29
+ from __future__ import annotations
30
+
31
+ import argparse
32
+ import json
33
+ import sys
34
+ from dataclasses import dataclass, field
35
+ from pathlib import Path
36
+
37
+ from . import calibration_ledger, checkpoint, checkpoint_memory, decision_extract, plan_shape
38
+ from .decision_extract import ORIGIN_CONFLICT, ORIGIN_ENTRY, ORIGIN_FORK, Extraction
39
+ from .plan_shape import FileWrite, Fork, PlanDoc
40
+
41
+ __all__ = ["BuildLoopError", "Resolved", "resolve", "label_from_chosen"]
42
+
43
+
44
+ class BuildLoopError(Exception):
45
+ """A plan that cannot be resolved, or an answer that cannot be placed."""
46
+
47
+
48
+ @dataclass
49
+ class Resolved:
50
+ plan: PlanDoc
51
+ outcomes: list[checkpoint.CheckpointOutcome] = field(default_factory=list)
52
+ predictions: list[dict] = field(default_factory=list) # {claim, confidence, outcome | already_settled}
53
+ chosen: dict[str, str] = field(default_factory=dict) # decision_type -> chosen label
54
+
55
+ def to_dict(self) -> dict:
56
+ from dataclasses import asdict
57
+ return {"plan": self.plan.to_dict(), "outcomes": [asdict(o) for o in self.outcomes],
58
+ "predictions": self.predictions, "chosen": self.chosen}
59
+
60
+
61
+ def label_from_chosen(chosen: str, labels: tuple[str, ...] | list[str]) -> str | None:
62
+ """A memory answer is `label` (fresh) or `label: rationale` (a prior seal's
63
+ one canonical string — `CheckpointOutcome.chosen` says why it is not split).
64
+ Longest label that the answer starts with wins; nothing matching is None,
65
+ and the caller refuses rather than guesses."""
66
+ c = chosen.strip()
67
+ for lab in sorted(labels, key=len, reverse=True):
68
+ if c == lab or c.startswith(lab + ":") or c.startswith(lab + " "):
69
+ return lab
70
+ return None
71
+
72
+
73
+ def _predict(builder_id: str, x: decision_extract.Extracted, root: Path) -> dict | None:
74
+ if x.recommended is None or x.confidence is None:
75
+ return None
76
+ claim = f"{x.decision.decision_type}: maker picks {x.recommended}"
77
+ try:
78
+ rec = calibration_ledger.record_prediction(builder_id, claim, x.confidence, kind="fork", root=root)
79
+ except calibration_ledger.CalibrationLedgerError as e:
80
+ # The same claim already settled (a re-run of a resolved plan). A
81
+ # settled prediction is history; do not re-open it, do not fabricate.
82
+ return {"claim": claim, "confidence": x.confidence, "already_settled": True, "note": str(e)}
83
+ return {"claim": claim, "confidence": x.confidence, "prediction_id": rec["id"]}
84
+
85
+
86
+ def _settle(builder_id: str, pred: dict | None, hit: bool, root: Path) -> None:
87
+ if pred is None or pred.get("already_settled"):
88
+ return
89
+ rec = calibration_ledger.resolve_prediction(builder_id, pred["prediction_id"], hit, root=root)
90
+ pred["outcome"] = rec["outcome"]
91
+
92
+
93
+ def resolve(
94
+ plan: PlanDoc,
95
+ *,
96
+ builder_id: str,
97
+ responder: checkpoint.Responder,
98
+ root: Path = checkpoint_memory.DEFAULT_CHECKPOINT_ROOT,
99
+ entry=None,
100
+ extraction: Extraction | None = None,
101
+ recognize_threshold: float = checkpoint.DEFAULT_RECOGNIZE_THRESHOLD,
102
+ ) -> Resolved:
103
+ ex = extraction if extraction is not None else decision_extract.extract(plan, entry=entry)
104
+ if ex.refused:
105
+ raise BuildLoopError("plan holds a fork that cannot be asked: " +
106
+ "; ".join(f"entries[{r.index}]: {r.reason}" for r in ex.refused))
107
+
108
+ out = Resolved(plan=plan)
109
+ replacements: dict[int, tuple[FileWrite, ...] | None] = {} # entry index -> what replaces it
110
+ drop: set[int] = set()
111
+
112
+ for x in ex.items:
113
+ pred = _predict(builder_id, x, root)
114
+ outcome = checkpoint.run_checkpoint(
115
+ x.decision, builder_id=builder_id, responder=responder, root=root,
116
+ recognize_threshold=recognize_threshold,
117
+ )
118
+ out.outcomes.append(outcome)
119
+ label = label_from_chosen(outcome.chosen, x.decision.options and [o.label for o in x.decision.options])
120
+ if label is None:
121
+ raise BuildLoopError(
122
+ f"memory answered {outcome.chosen!r} for {x.decision.decision_type!r}, which names none of "
123
+ f"{[o.label for o in x.decision.options]} — refusing to guess")
124
+ out.chosen[x.decision.decision_type] = label
125
+ _settle(builder_id, pred, label == x.recommended, root)
126
+ if pred is not None:
127
+ out.predictions.append(pred)
128
+
129
+ if x.origin == ORIGIN_FORK and x.index is not None:
130
+ fork = plan.entries[x.index]
131
+ assert isinstance(fork, Fork)
132
+ replacements[x.index] = fork.resolves.get(label, ())
133
+ elif x.origin == ORIGIN_CONFLICT:
134
+ keep = int(label.split()[-1]) # "entry N"
135
+ drop.update(i for i in x.indices if i != keep)
136
+ elif x.origin == ORIGIN_ENTRY:
137
+ pass # the entry's major; nothing in the plan changes
138
+
139
+ entries: list = []
140
+ for i, e in enumerate(plan.entries):
141
+ if i in drop:
142
+ continue
143
+ if i in replacements:
144
+ entries.extend(replacements[i] or ())
145
+ continue
146
+ entries.append(e)
147
+ resolved = PlanDoc(app_name=plan.app_name, entries=tuple(entries))
148
+ if resolved.has_forks:
149
+ raise BuildLoopError("a fork survived resolution — refusing to return an unresolved plan")
150
+ out.plan = resolved
151
+ return out
152
+
153
+
154
+ # ── CLI (dev shape) ─────────────────────────────────────────────────────────
155
+
156
+ class _PickResponder:
157
+ def __init__(self, picks: dict[str, str], why: str):
158
+ self._picks, self._why = picks, why
159
+
160
+ def confirm(self, prompt: str) -> bool:
161
+ print(f"[confirm] {prompt}\n[confirm] -> yes", file=sys.stderr)
162
+ return True
163
+
164
+ def choose(self, d: checkpoint.Decision) -> checkpoint.ChoiceResult:
165
+ label = self._picks.get(d.decision_type) or d.options[0].label
166
+ print(f"[choose] {d.surface}\n[choose] -> {label}", file=sys.stderr)
167
+ return checkpoint.ChoiceResult(chosen_label=label, rationale=self._why)
168
+
169
+
170
+ def build_parser() -> argparse.ArgumentParser:
171
+ p = argparse.ArgumentParser(prog="build_loop.py", description="resolve a plan's forks through memory")
172
+ p.add_argument("plan")
173
+ p.add_argument("--builder", required=True, dest="builder_id")
174
+ p.add_argument("--root", default=str(checkpoint_memory.DEFAULT_CHECKPOINT_ROOT))
175
+ p.add_argument("--choose", action="append", default=[], metavar="TYPE=LABEL",
176
+ help="what to pick if asked, per decision_type (default: first option)")
177
+ p.add_argument("--why", default="picked at the command line")
178
+ p.add_argument("--json", action="store_true")
179
+ return p
180
+
181
+
182
+ def main(argv: list[str] | None = None) -> int:
183
+ a = build_parser().parse_args(argv)
184
+ picks = dict(s.split("=", 1) for s in a.choose)
185
+ try:
186
+ plan = plan_shape.load(a.plan)
187
+ res = resolve(plan, builder_id=a.builder_id, responder=_PickResponder(picks, a.why), root=Path(a.root))
188
+ except (plan_shape.PlanShapeError, BuildLoopError, checkpoint_memory.CheckpointMemoryError) as e:
189
+ print(f"REFUSED: {e}", file=sys.stderr)
190
+ return 2
191
+ card = calibration_ledger.scorecard(a.builder_id, root=Path(a.root))
192
+ if a.json:
193
+ print(json.dumps({**res.to_dict(), "scorecard": card}, indent=2, default=str))
194
+ else:
195
+ for o in res.outcomes:
196
+ print(f"{o.decision_type:24} band {o.band:9} → {res.chosen[o.decision_type]}")
197
+ for pr in res.predictions:
198
+ print(f" prediction {pr['claim']!r} @ {pr['confidence']:.2f} → "
199
+ f"{'already settled' if pr.get('already_settled') else pr.get('outcome')}")
200
+ print(f"resolved plan: {len(res.plan.entries)} entries, no forks")
201
+ print(f"scorecard: {card['resolved']} resolved, {card['pending']} pending")
202
+ return 0
203
+
204
+
205
+ if __name__ == "__main__":
206
+ sys.exit(main())
forge/calibration.py ADDED
@@ -0,0 +1,94 @@
1
+ #!/usr/bin/env python3
2
+ # ── SAFE App Store vendor note (The Forge, calibration ledger, 2026-08-11) ────
3
+ # Vendored byte-for-byte (except this header block) from the oakenscrolls-office
4
+ # playground build in THIS repo: `apps/oakenscrolls-office/calibration.py`
5
+ # (b17: OAKOF), Apache-2.0 — same license as this store, zero friction.
6
+ #
7
+ # Why vendor rather than import the playground copy: `apps/` is the contested
8
+ # commons (rule 5) — a store-side organ (D1) must not import a build out of the
9
+ # untrusted playground, and `apps/oakenscrolls-office/` is not on the store's
10
+ # import path anyway. This is pure stdlib (`math`), ~40 lines, small enough to
11
+ # audit in one sitting — the same "vendor the core, not the package" discipline
12
+ # `friction_floor`/`human_loop` already follow. Rule 11: the house already grew
13
+ # this (oakenscrolls-office's confidence mirror, the box's calibration tool);
14
+ # the Forge re-points it at the MODEL's own confidence instead of rebuilding the
15
+ # Brier/log-score/reliability math. Kept diffable against the playground copy on
16
+ # purpose — if oakenscrolls' copy moves, `diff` this against it and reconcile.
17
+ #
18
+ # A "pair" everywhere below is (confidence, outcome): the final P(true) the
19
+ # predictor stated, and whether the claim turned out true.
20
+ """
21
+ calibration.py — pure calibration math. b17: OAKOF
22
+
23
+ Dependency-free, stdlib-only, in the no-egress zone (tests/test_no_egress.py).
24
+ The vendoring pattern is utety/core/mastery.py: the ~40 lines of inference a
25
+ device actually needs, small enough to audit in one sitting.
26
+
27
+ A "pair" everywhere below is (confidence, outcome): the final P(true) the
28
+ person stated, and whether the claim turned out true.
29
+ """
30
+ from __future__ import annotations
31
+
32
+ import math
33
+
34
+ _EPS = 1e-9
35
+
36
+ # House convention (design D3): claims are stated in the direction believed,
37
+ # so confidence lives in [0.5, 0.99]. Bins cover that range in five steps.
38
+ BIN_EDGES = (0.5, 0.6, 0.7, 0.8, 0.9, 1.0)
39
+
40
+
41
+ def brier(confidence: float, outcome: bool) -> float:
42
+ """Squared distance between what you said and what happened. 0 is prophecy,
43
+ 0.25 is coin-flipping at 50%, 1.0 is confidently wrong."""
44
+ return (confidence - (1.0 if outcome else 0.0)) ** 2
45
+
46
+
47
+ def log_score(confidence: float, outcome: bool) -> float:
48
+ """Negative log likelihood of the outcome under your stated belief.
49
+ Punishes confident wrongness much harder than Brier does."""
50
+ p = confidence if outcome else 1.0 - confidence
51
+ return -math.log(max(p, _EPS))
52
+
53
+
54
+ def bins(pairs: list[tuple[float, bool]]) -> list[dict]:
55
+ """Reliability table: for each stated-confidence band, how often were you
56
+ actually right? Empty bands are returned with n=0 so the mirror shows the
57
+ whole range, not just where you've dared to speak."""
58
+ out = []
59
+ for lo, hi in zip(BIN_EDGES, BIN_EDGES[1:]):
60
+ members = [(c, o) for c, o in pairs if lo <= c < hi or (hi == 1.0 and c == 1.0)]
61
+ n = len(members)
62
+ out.append({
63
+ "lo": lo,
64
+ "hi": hi,
65
+ "n": n,
66
+ "mean_confidence": sum(c for c, _ in members) / n if n else None,
67
+ "hit_rate": sum(1 for _, o in members if o) / n if n else None,
68
+ })
69
+ return out
70
+
71
+
72
+ def summary(pairs: list[tuple[float, bool]]) -> dict:
73
+ """The scorecard header. `overconfidence` is mean stated confidence minus
74
+ actual hit rate: positive means you promise more than you deliver."""
75
+ n = len(pairs)
76
+ if not n:
77
+ return {
78
+ "n": 0,
79
+ "brier": None,
80
+ "log_score": None,
81
+ "mean_confidence": None,
82
+ "hit_rate": None,
83
+ "overconfidence": None,
84
+ }
85
+ mean_conf = sum(c for c, _ in pairs) / n
86
+ hit_rate = sum(1 for _, o in pairs if o) / n
87
+ return {
88
+ "n": n,
89
+ "brier": sum(brier(c, o) for c, o in pairs) / n,
90
+ "log_score": sum(log_score(c, o) for c, o in pairs) / n,
91
+ "mean_confidence": mean_conf,
92
+ "hit_rate": hit_rate,
93
+ "overconfidence": mean_conf - hit_rate,
94
+ }
@@ -0,0 +1,288 @@
1
+ #!/usr/bin/env python3
2
+ """forge/calibration_ledger.py — the model's confidence mirror
3
+ (docs/design/the-forge-measure.md, the `calibration` class).
4
+
5
+ The measuring panel refuses a confident wrong ARTIFACT; the checkpoint loop
6
+ refuses a confident wrong DECISION. This refuses a confident wrong PREDICTOR —
7
+ the model itself. It is the panel's `calibration` class, but it is NOT a per-build
8
+ instrument like `census`/`execution`: calibration is a claim about the model
9
+ ACROSS builds, so it is a longitudinal ledger, not a directory measurement.
10
+
11
+ The mechanism the box taught (oakenscrolls-office, the confidence mirror): a
12
+ prediction is a `(confidence, outcome)` pair — the model states P(true) for a
13
+ claim it makes while building ("this file parses", "this design holds", "this
14
+ dependency exists"), and later ground truth arrives (a test runs, the panel
15
+ measures, a human rules). Grade the model's stated confidence against what
16
+ actually happened: `brier`, `log_score`, and the reliability table (`bins`) —
17
+ all from the vendored `forge/calibration.py` (rule 11: the math is
18
+ oakenscrolls', not rebuilt here). The one signal that matters is
19
+ `overconfidence` — mean stated confidence minus actual hit rate; positive means
20
+ the model promises more than it delivers.
21
+
22
+ Persistence is the same per-builder SOIL file the governance layer already uses
23
+ (`forge/soil_store.py`, D6: a builder's records live in that builder's own file
24
+ by construction), under this module's own collection. The overconfidence signal
25
+ routes through `checkpoint_governance.route_nudge` (rule 11: reuse the deduped
26
+ outbox bite 3 / #67 and the panel already feed) — it surfaces a `review` item a
27
+ human should see; it NEVER blocks a build, exactly like every other model-side
28
+ signal.
29
+
30
+ Store-side (D1): `apps/the-forge/` never imports this — a sandboxed build does
31
+ not grade its own model's confidence, any more than it marks its own homework.
32
+ """
33
+ from __future__ import annotations
34
+
35
+ import argparse
36
+ import hashlib
37
+ import json
38
+ from pathlib import Path
39
+
40
+
41
+
42
+ from . import calibration
43
+ from . import soil_store
44
+ from . import checkpoint_governance as governance
45
+
46
+ # Predictions live in this module's own collection inside the builder's SOIL
47
+ # file — alongside (never colliding with) the governance queue collection.
48
+ PREDICTIONS = "calibration_predictions"
49
+
50
+ # The overconfidence signal's defaults. `MIN_N` is the floor below which a
51
+ # scorecard is too thin to route on (two lucky calls are not a calibration);
52
+ # `OVERCONFIDENCE_FLOOR` is how far stated confidence may exceed hit rate before
53
+ # it is worth a human's eyes. Both are overridable per call.
54
+ MIN_N = 5
55
+ OVERCONFIDENCE_FLOOR = 0.10
56
+
57
+ # One standing open flag per builder: route_nudge dedupes by source_ref, so a
58
+ # persistently overconfident model surfaces ONE open `review` item, not a new
59
+ # one every resolve. When a human resolves that item and the model is still
60
+ # overconfident later, a fresh open item can form (route_nudge only dedupes
61
+ # against OPEN rows) — the correct "one live flag at a time" behavior.
62
+ _SIGNAL_SOURCE_REF_PREFIX = "calibration-overconfidence:"
63
+
64
+
65
+ class CalibrationLedgerError(Exception):
66
+ """This module's own refusal — a confidence outside the believed-direction
67
+ range, or a prediction lifecycle violation (re-predicting a settled claim,
68
+ resolving an unknown or already-resolved one). `soil_store`'s
69
+ `SoilStoreError` and `governance`'s errors propagate unwrapped."""
70
+
71
+
72
+ def _store(builder_id: str, root: Path | None):
73
+ kw = {} if root is None else {"root": Path(root)}
74
+ return soil_store.FilesystemSoilStore(builder_id, **kw)
75
+
76
+
77
+ def _check_confidence(confidence: float) -> float:
78
+ """House convention (calibration D3): a claim is stated in the direction
79
+ believed, so confidence is P(the claim is true) in [0.5, 1.0]. A value below
80
+ 0.5 means the model actually believes the claim FALSE — it must be restated
81
+ in the believed direction, not recorded as a weak 'true'. Reject rather than
82
+ silently clamp: a clamp would fabricate a data point the model never made."""
83
+ try:
84
+ c = float(confidence)
85
+ except (TypeError, ValueError) as e:
86
+ raise CalibrationLedgerError(f"confidence must be a number, got {confidence!r}") from e
87
+ if not (0.5 <= c <= 1.0):
88
+ raise CalibrationLedgerError(
89
+ f"confidence {c} outside [0.5, 1.0] — state the claim in the "
90
+ f"direction believed (calibration D3), a claim you think false is a "
91
+ f"true-claim restated, not a confidence below 0.5"
92
+ )
93
+ return c
94
+
95
+
96
+ def _prediction_id(claim: str) -> str:
97
+ """A stable id for a claim with no explicit id — deterministic (no wall
98
+ clock, no randomness), so re-stating the same claim addresses the same
99
+ pending prediction rather than spawning a duplicate."""
100
+ h = hashlib.sha256(claim.encode("utf-8")).hexdigest()[:16]
101
+ return f"pred-{h}"
102
+
103
+
104
+ def record_prediction(
105
+ builder_id: str,
106
+ claim: str,
107
+ confidence: float,
108
+ *,
109
+ prediction_id: str | None = None,
110
+ kind: str = "",
111
+ root: Path | None = None,
112
+ ) -> dict:
113
+ """Record the model's stated confidence in a claim, pending ground truth.
114
+
115
+ `confidence` is P(claim true) in [0.5, 1.0]. Idempotent on the claim: with
116
+ no `prediction_id`, the id is derived from the claim text, so re-stating the
117
+ same claim UPDATES its (still-pending) confidence rather than duplicating it.
118
+ Refuses to re-record a claim that has already been RESOLVED — a settled data
119
+ point is history, not something a later re-prediction may overwrite."""
120
+ if not isinstance(claim, str) or not claim.strip():
121
+ raise CalibrationLedgerError("claim must be a non-empty string")
122
+ c = _check_confidence(confidence)
123
+ pid = prediction_id or _prediction_id(claim)
124
+ store = _store(builder_id, root)
125
+ existing = store.get(PREDICTIONS, pid)
126
+ if existing is not None and existing.get("resolved"):
127
+ raise CalibrationLedgerError(
128
+ f"prediction {pid!r} is already resolved (outcome="
129
+ f"{existing.get('outcome')!r}) — a settled prediction is not "
130
+ f"re-recordable; use a fresh prediction_id for a new claim"
131
+ )
132
+ record = {
133
+ "id": pid,
134
+ "claim": claim,
135
+ "confidence": c,
136
+ "kind": kind,
137
+ "outcome": None,
138
+ "resolved": False,
139
+ }
140
+ store.put(PREDICTIONS, record, record_id=pid)
141
+ return record
142
+
143
+
144
+ def resolve_prediction(
145
+ builder_id: str,
146
+ prediction_id: str,
147
+ outcome: bool,
148
+ *,
149
+ root: Path | None = None,
150
+ ) -> dict:
151
+ """Settle a pending prediction with ground truth (`outcome` True/False).
152
+
153
+ Refuses an unknown id, and refuses to re-resolve an already-settled one — a
154
+ double-resolve would silently rewrite history (the same single-use guard the
155
+ governance resume seam already learned to keep)."""
156
+ store = _store(builder_id, root)
157
+ record = store.get(PREDICTIONS, prediction_id)
158
+ if record is None:
159
+ raise CalibrationLedgerError(f"no prediction {prediction_id!r} to resolve")
160
+ if record.get("resolved"):
161
+ raise CalibrationLedgerError(
162
+ f"prediction {prediction_id!r} already resolved (outcome="
163
+ f"{record.get('outcome')!r}) — a settled prediction is single-use"
164
+ )
165
+ record["outcome"] = bool(outcome)
166
+ record["resolved"] = True
167
+ store.put(PREDICTIONS, record, record_id=prediction_id)
168
+ return record
169
+
170
+
171
+ def scorecard(builder_id: str, *, root: Path | None = None) -> dict:
172
+ """The model's calibration mirror: the `calibration.summary` header (brier,
173
+ log_score, hit_rate, overconfidence) plus the `calibration.bins` reliability
174
+ table, over the builder's RESOLVED predictions, with a pending count so a
175
+ thin scorecard reads as thin, not as confident."""
176
+ store = _store(builder_id, root)
177
+ all_recs = store.all(PREDICTIONS)
178
+ pairs = [(float(r["confidence"]), bool(r["outcome"])) for r in all_recs if r.get("resolved")]
179
+ return {
180
+ "builder_id": store.builder_id,
181
+ "resolved": len(pairs),
182
+ "pending": sum(1 for r in all_recs if not r.get("resolved")),
183
+ "summary": calibration.summary(pairs),
184
+ "bins": calibration.bins(pairs),
185
+ }
186
+
187
+
188
+ def overconfidence_signal(
189
+ builder_id: str,
190
+ *,
191
+ root: Path | None = None,
192
+ min_n: int = MIN_N,
193
+ floor: float = OVERCONFIDENCE_FLOOR,
194
+ ) -> dict | None:
195
+ """Route ONE `review` nudge when the model is measurably overconfident.
196
+
197
+ Fires only when there are at least `min_n` resolved predictions (a thin
198
+ record is not a calibration) AND `overconfidence` (mean stated confidence −
199
+ hit rate) is at least `floor`. Reuses `checkpoint_governance.route_nudge`,
200
+ so it is deduped by a per-builder `source_ref` (one standing open flag) and
201
+ persisted in the same `human_required` queue every other model-side signal
202
+ feeds. Returns the routed item, or None when the bar is not met OR the flag
203
+ is already open (route_nudge's dedupe). NEVER blocks — it surfaces."""
204
+ card = scorecard(builder_id, root=root)
205
+ n = card["resolved"]
206
+ over = card["summary"]["overconfidence"]
207
+ if n < min_n or over is None or over < floor:
208
+ return None
209
+ return governance.route_nudge(
210
+ builder_id,
211
+ kind="review",
212
+ title="model overconfidence",
213
+ summary=(
214
+ f"calibration: {n} resolved predictions, stated confidence "
215
+ f"{card['summary']['mean_confidence']:.2f} vs hit rate "
216
+ f"{card['summary']['hit_rate']:.2f} (overconfidence {over:+.2f}, "
217
+ f"brier {card['summary']['brier']:.3f}). The model promises more "
218
+ f"than it delivers — a human should recalibrate what it asserts."
219
+ ),
220
+ source_ref=f"{_SIGNAL_SOURCE_REF_PREFIX}{card['builder_id']}",
221
+ priority="normal",
222
+ root=soil_store.DEFAULT_CHECKPOINT_ROOT if root is None else Path(root),
223
+ )
224
+
225
+
226
+ # ── CLI ──────────────────────────────────────────────────────────────────────
227
+
228
+ def _cmd_record(args: argparse.Namespace) -> int:
229
+ rec = record_prediction(
230
+ args.builder_id, args.claim, args.confidence,
231
+ prediction_id=args.id, kind=args.kind or "", root=Path(args.root) if args.root else None,
232
+ )
233
+ print(json.dumps(rec, indent=2))
234
+ return 0
235
+
236
+
237
+ def _cmd_resolve(args: argparse.Namespace) -> int:
238
+ rec = resolve_prediction(
239
+ args.builder_id, args.id, args.outcome == "true",
240
+ root=Path(args.root) if args.root else None,
241
+ )
242
+ print(json.dumps(rec, indent=2))
243
+ return 0
244
+
245
+
246
+ def _cmd_scorecard(args: argparse.Namespace) -> int:
247
+ card = scorecard(args.builder_id, root=Path(args.root) if args.root else None)
248
+ print(json.dumps(card, indent=2))
249
+ # a routed overconfidence flag is a CI-visible signal (exit 1), never a block
250
+ routed = overconfidence_signal(args.builder_id, root=Path(args.root) if args.root else None)
251
+ return 1 if routed else 0
252
+
253
+
254
+ def build_parser() -> argparse.ArgumentParser:
255
+ p = argparse.ArgumentParser(prog="calibration_ledger.py")
256
+ sub = p.add_subparsers(dest="command", required=True)
257
+
258
+ r = sub.add_parser("record", help="record a pending prediction (confidence in [0.5, 1.0])")
259
+ r.add_argument("builder_id")
260
+ r.add_argument("claim")
261
+ r.add_argument("confidence", type=float)
262
+ r.add_argument("--id", default=None)
263
+ r.add_argument("--kind", default="")
264
+ r.add_argument("--root", default=None)
265
+ r.set_defaults(func=_cmd_record)
266
+
267
+ v = sub.add_parser("resolve", help="settle a prediction with ground truth")
268
+ v.add_argument("builder_id")
269
+ v.add_argument("id")
270
+ v.add_argument("outcome", choices=["true", "false"])
271
+ v.add_argument("--root", default=None)
272
+ v.set_defaults(func=_cmd_resolve)
273
+
274
+ s = sub.add_parser("scorecard", help="the model's calibration mirror (+ routes an overconfidence nudge)")
275
+ s.add_argument("builder_id")
276
+ s.add_argument("--root", default=None)
277
+ s.set_defaults(func=_cmd_scorecard)
278
+
279
+ return p
280
+
281
+
282
+ def main(argv: list[str] | None = None) -> int:
283
+ args = build_parser().parse_args(argv)
284
+ return args.func(args)
285
+
286
+
287
+ if __name__ == "__main__":
288
+ raise SystemExit(main())