forge-play 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- forge/__init__.py +1 -0
- forge/_ids.py +32 -0
- forge/build_loop.py +206 -0
- forge/calibration.py +94 -0
- forge/calibration_ledger.py +288 -0
- forge/checkpoint.py +742 -0
- forge/checkpoint_calibration.py +536 -0
- forge/checkpoint_engagement.py +169 -0
- forge/checkpoint_governance.py +265 -0
- forge/checkpoint_memory.py +789 -0
- forge/checkpoint_nudge.py +268 -0
- forge/checkpoint_schedule.py +406 -0
- forge/decision_extract.py +173 -0
- forge/entry.py +254 -0
- forge/friction_floor.py +341 -0
- forge/human_loop.py +245 -0
- forge/instrument_callgraph.py +241 -0
- forge/instrument_execution.py +185 -0
- forge/keywords.toml +96 -0
- forge/majors.py +155 -0
- forge/measure_panel.py +468 -0
- forge/model_egress.py +85 -0
- forge/model_route.py +174 -0
- forge/paths.py +51 -0
- forge/plan_shape.py +217 -0
- forge/soil_store.py +119 -0
- forge/trust.py +107 -0
- forge_play-0.1.0.dist-info/METADATA +141 -0
- forge_play-0.1.0.dist-info/RECORD +32 -0
- forge_play-0.1.0.dist-info/WHEEL +4 -0
- forge_play-0.1.0.dist-info/licenses/LICENSE +202 -0
- forge_play-0.1.0.dist-info/licenses/NOTICE +14 -0
forge/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The Forge — a refuse-a-confident-wrong-answer harness (the engine)."""
|
forge/_ids.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""forge/core/_ids.py — builder-id charset (vendored from safe-app-store principal.py).
|
|
2
|
+
|
|
3
|
+
The model side keys every per-builder record on a builder_id; principal.py (D2/D11)
|
|
4
|
+
is the one place that charset is defined, and checkpoint_memory imported
|
|
5
|
+
`principal._check_builder_id` from it. Rather than drag the 900-line SAFE identity
|
|
6
|
+
core into a dependency-light library, this vendors the ~10 lines actually used —
|
|
7
|
+
the same "vendor the primitive, not the package" discipline calibration/human_loop/
|
|
8
|
+
friction_floor follow. Bound as `principal` at the import site so the reference is
|
|
9
|
+
unchanged. If principal.py's charset moves, reconcile this copy.
|
|
10
|
+
"""
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import re
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
17
|
+
_MAX_BUILDER_ID_LEN = 128
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class PrincipalError(Exception):
|
|
21
|
+
"""Fail-closed refusal — a builder_id that is not a str or fails the
|
|
22
|
+
path-safety charset (D11)."""
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _check_builder_id(builder_id: Any) -> str:
|
|
26
|
+
if not isinstance(builder_id, str):
|
|
27
|
+
raise PrincipalError(f"builder_id must be a str, got {type(builder_id).__name__}")
|
|
28
|
+
if not builder_id or not _ID_PATTERN.match(builder_id):
|
|
29
|
+
raise PrincipalError(f"builder_id {builder_id!r} fails the path-safety charset (D11)")
|
|
30
|
+
if len(builder_id) > _MAX_BUILDER_ID_LEN:
|
|
31
|
+
raise PrincipalError(f"builder_id is longer than {_MAX_BUILDER_ID_LEN} characters")
|
|
32
|
+
return builder_id
|
forge/build_loop.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""forge/build_loop.py — resolve a plan's decisions through memory, and find
|
|
3
|
+
out whether the proposer's confidence was earned.
|
|
4
|
+
|
|
5
|
+
decision_extract → run_checkpoint (per decision, in plan order) → the
|
|
6
|
+
answer substituted back into the plan → a plan with no fork left in it.
|
|
7
|
+
|
|
8
|
+
The model never touches the router: extraction is rule-based, the band comes
|
|
9
|
+
from the maker's own memory, and a fork's `recommended` is only ever a
|
|
10
|
+
suggestion the deferral path may fall back to. What the model DOES get is a
|
|
11
|
+
scorecard. For every fork that carried a `recommended` and a `confidence`:
|
|
12
|
+
|
|
13
|
+
before the ask calibration_ledger.record_prediction(
|
|
14
|
+
claim="<decision_type>: maker picks <recommended>",
|
|
15
|
+
confidence)
|
|
16
|
+
after the ask calibration_ledger.resolve_prediction(
|
|
17
|
+
outcome = (chosen == recommended))
|
|
18
|
+
|
|
19
|
+
Ground truth that arrives on its own — the maker's pick IS the outcome, no
|
|
20
|
+
opinion enters — which is the loop the Forge exists to close and which, until
|
|
21
|
+
this module, nothing outside the ledger's own tests had ever called. A fork
|
|
22
|
+
without a confidence records nothing: no fabricated 0.5.
|
|
23
|
+
|
|
24
|
+
Refusals: a plan whose extraction refused a fork cannot be resolved (the
|
|
25
|
+
reasons are in the error); a plan that still holds a fork after resolution is
|
|
26
|
+
a bug and is refused rather than returned; a memory answer that names no
|
|
27
|
+
option on offer is refused rather than guessed at.
|
|
28
|
+
"""
|
|
29
|
+
from __future__ import annotations
|
|
30
|
+
|
|
31
|
+
import argparse
|
|
32
|
+
import json
|
|
33
|
+
import sys
|
|
34
|
+
from dataclasses import dataclass, field
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
|
|
37
|
+
from . import calibration_ledger, checkpoint, checkpoint_memory, decision_extract, plan_shape
|
|
38
|
+
from .decision_extract import ORIGIN_CONFLICT, ORIGIN_ENTRY, ORIGIN_FORK, Extraction
|
|
39
|
+
from .plan_shape import FileWrite, Fork, PlanDoc
|
|
40
|
+
|
|
41
|
+
__all__ = ["BuildLoopError", "Resolved", "resolve", "label_from_chosen"]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class BuildLoopError(Exception):
|
|
45
|
+
"""A plan that cannot be resolved, or an answer that cannot be placed."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass
|
|
49
|
+
class Resolved:
|
|
50
|
+
plan: PlanDoc
|
|
51
|
+
outcomes: list[checkpoint.CheckpointOutcome] = field(default_factory=list)
|
|
52
|
+
predictions: list[dict] = field(default_factory=list) # {claim, confidence, outcome | already_settled}
|
|
53
|
+
chosen: dict[str, str] = field(default_factory=dict) # decision_type -> chosen label
|
|
54
|
+
|
|
55
|
+
def to_dict(self) -> dict:
|
|
56
|
+
from dataclasses import asdict
|
|
57
|
+
return {"plan": self.plan.to_dict(), "outcomes": [asdict(o) for o in self.outcomes],
|
|
58
|
+
"predictions": self.predictions, "chosen": self.chosen}
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def label_from_chosen(chosen: str, labels: tuple[str, ...] | list[str]) -> str | None:
|
|
62
|
+
"""A memory answer is `label` (fresh) or `label: rationale` (a prior seal's
|
|
63
|
+
one canonical string — `CheckpointOutcome.chosen` says why it is not split).
|
|
64
|
+
Longest label that the answer starts with wins; nothing matching is None,
|
|
65
|
+
and the caller refuses rather than guesses."""
|
|
66
|
+
c = chosen.strip()
|
|
67
|
+
for lab in sorted(labels, key=len, reverse=True):
|
|
68
|
+
if c == lab or c.startswith(lab + ":") or c.startswith(lab + " "):
|
|
69
|
+
return lab
|
|
70
|
+
return None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _predict(builder_id: str, x: decision_extract.Extracted, root: Path) -> dict | None:
|
|
74
|
+
if x.recommended is None or x.confidence is None:
|
|
75
|
+
return None
|
|
76
|
+
claim = f"{x.decision.decision_type}: maker picks {x.recommended}"
|
|
77
|
+
try:
|
|
78
|
+
rec = calibration_ledger.record_prediction(builder_id, claim, x.confidence, kind="fork", root=root)
|
|
79
|
+
except calibration_ledger.CalibrationLedgerError as e:
|
|
80
|
+
# The same claim already settled (a re-run of a resolved plan). A
|
|
81
|
+
# settled prediction is history; do not re-open it, do not fabricate.
|
|
82
|
+
return {"claim": claim, "confidence": x.confidence, "already_settled": True, "note": str(e)}
|
|
83
|
+
return {"claim": claim, "confidence": x.confidence, "prediction_id": rec["id"]}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _settle(builder_id: str, pred: dict | None, hit: bool, root: Path) -> None:
|
|
87
|
+
if pred is None or pred.get("already_settled"):
|
|
88
|
+
return
|
|
89
|
+
rec = calibration_ledger.resolve_prediction(builder_id, pred["prediction_id"], hit, root=root)
|
|
90
|
+
pred["outcome"] = rec["outcome"]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def resolve(
|
|
94
|
+
plan: PlanDoc,
|
|
95
|
+
*,
|
|
96
|
+
builder_id: str,
|
|
97
|
+
responder: checkpoint.Responder,
|
|
98
|
+
root: Path = checkpoint_memory.DEFAULT_CHECKPOINT_ROOT,
|
|
99
|
+
entry=None,
|
|
100
|
+
extraction: Extraction | None = None,
|
|
101
|
+
recognize_threshold: float = checkpoint.DEFAULT_RECOGNIZE_THRESHOLD,
|
|
102
|
+
) -> Resolved:
|
|
103
|
+
ex = extraction if extraction is not None else decision_extract.extract(plan, entry=entry)
|
|
104
|
+
if ex.refused:
|
|
105
|
+
raise BuildLoopError("plan holds a fork that cannot be asked: " +
|
|
106
|
+
"; ".join(f"entries[{r.index}]: {r.reason}" for r in ex.refused))
|
|
107
|
+
|
|
108
|
+
out = Resolved(plan=plan)
|
|
109
|
+
replacements: dict[int, tuple[FileWrite, ...] | None] = {} # entry index -> what replaces it
|
|
110
|
+
drop: set[int] = set()
|
|
111
|
+
|
|
112
|
+
for x in ex.items:
|
|
113
|
+
pred = _predict(builder_id, x, root)
|
|
114
|
+
outcome = checkpoint.run_checkpoint(
|
|
115
|
+
x.decision, builder_id=builder_id, responder=responder, root=root,
|
|
116
|
+
recognize_threshold=recognize_threshold,
|
|
117
|
+
)
|
|
118
|
+
out.outcomes.append(outcome)
|
|
119
|
+
label = label_from_chosen(outcome.chosen, x.decision.options and [o.label for o in x.decision.options])
|
|
120
|
+
if label is None:
|
|
121
|
+
raise BuildLoopError(
|
|
122
|
+
f"memory answered {outcome.chosen!r} for {x.decision.decision_type!r}, which names none of "
|
|
123
|
+
f"{[o.label for o in x.decision.options]} — refusing to guess")
|
|
124
|
+
out.chosen[x.decision.decision_type] = label
|
|
125
|
+
_settle(builder_id, pred, label == x.recommended, root)
|
|
126
|
+
if pred is not None:
|
|
127
|
+
out.predictions.append(pred)
|
|
128
|
+
|
|
129
|
+
if x.origin == ORIGIN_FORK and x.index is not None:
|
|
130
|
+
fork = plan.entries[x.index]
|
|
131
|
+
assert isinstance(fork, Fork)
|
|
132
|
+
replacements[x.index] = fork.resolves.get(label, ())
|
|
133
|
+
elif x.origin == ORIGIN_CONFLICT:
|
|
134
|
+
keep = int(label.split()[-1]) # "entry N"
|
|
135
|
+
drop.update(i for i in x.indices if i != keep)
|
|
136
|
+
elif x.origin == ORIGIN_ENTRY:
|
|
137
|
+
pass # the entry's major; nothing in the plan changes
|
|
138
|
+
|
|
139
|
+
entries: list = []
|
|
140
|
+
for i, e in enumerate(plan.entries):
|
|
141
|
+
if i in drop:
|
|
142
|
+
continue
|
|
143
|
+
if i in replacements:
|
|
144
|
+
entries.extend(replacements[i] or ())
|
|
145
|
+
continue
|
|
146
|
+
entries.append(e)
|
|
147
|
+
resolved = PlanDoc(app_name=plan.app_name, entries=tuple(entries))
|
|
148
|
+
if resolved.has_forks:
|
|
149
|
+
raise BuildLoopError("a fork survived resolution — refusing to return an unresolved plan")
|
|
150
|
+
out.plan = resolved
|
|
151
|
+
return out
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
# ── CLI (dev shape) ─────────────────────────────────────────────────────────
|
|
155
|
+
|
|
156
|
+
class _PickResponder:
|
|
157
|
+
def __init__(self, picks: dict[str, str], why: str):
|
|
158
|
+
self._picks, self._why = picks, why
|
|
159
|
+
|
|
160
|
+
def confirm(self, prompt: str) -> bool:
|
|
161
|
+
print(f"[confirm] {prompt}\n[confirm] -> yes", file=sys.stderr)
|
|
162
|
+
return True
|
|
163
|
+
|
|
164
|
+
def choose(self, d: checkpoint.Decision) -> checkpoint.ChoiceResult:
|
|
165
|
+
label = self._picks.get(d.decision_type) or d.options[0].label
|
|
166
|
+
print(f"[choose] {d.surface}\n[choose] -> {label}", file=sys.stderr)
|
|
167
|
+
return checkpoint.ChoiceResult(chosen_label=label, rationale=self._why)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
171
|
+
p = argparse.ArgumentParser(prog="build_loop.py", description="resolve a plan's forks through memory")
|
|
172
|
+
p.add_argument("plan")
|
|
173
|
+
p.add_argument("--builder", required=True, dest="builder_id")
|
|
174
|
+
p.add_argument("--root", default=str(checkpoint_memory.DEFAULT_CHECKPOINT_ROOT))
|
|
175
|
+
p.add_argument("--choose", action="append", default=[], metavar="TYPE=LABEL",
|
|
176
|
+
help="what to pick if asked, per decision_type (default: first option)")
|
|
177
|
+
p.add_argument("--why", default="picked at the command line")
|
|
178
|
+
p.add_argument("--json", action="store_true")
|
|
179
|
+
return p
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def main(argv: list[str] | None = None) -> int:
|
|
183
|
+
a = build_parser().parse_args(argv)
|
|
184
|
+
picks = dict(s.split("=", 1) for s in a.choose)
|
|
185
|
+
try:
|
|
186
|
+
plan = plan_shape.load(a.plan)
|
|
187
|
+
res = resolve(plan, builder_id=a.builder_id, responder=_PickResponder(picks, a.why), root=Path(a.root))
|
|
188
|
+
except (plan_shape.PlanShapeError, BuildLoopError, checkpoint_memory.CheckpointMemoryError) as e:
|
|
189
|
+
print(f"REFUSED: {e}", file=sys.stderr)
|
|
190
|
+
return 2
|
|
191
|
+
card = calibration_ledger.scorecard(a.builder_id, root=Path(a.root))
|
|
192
|
+
if a.json:
|
|
193
|
+
print(json.dumps({**res.to_dict(), "scorecard": card}, indent=2, default=str))
|
|
194
|
+
else:
|
|
195
|
+
for o in res.outcomes:
|
|
196
|
+
print(f"{o.decision_type:24} band {o.band:9} → {res.chosen[o.decision_type]}")
|
|
197
|
+
for pr in res.predictions:
|
|
198
|
+
print(f" prediction {pr['claim']!r} @ {pr['confidence']:.2f} → "
|
|
199
|
+
f"{'already settled' if pr.get('already_settled') else pr.get('outcome')}")
|
|
200
|
+
print(f"resolved plan: {len(res.plan.entries)} entries, no forks")
|
|
201
|
+
print(f"scorecard: {card['resolved']} resolved, {card['pending']} pending")
|
|
202
|
+
return 0
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
if __name__ == "__main__":
|
|
206
|
+
sys.exit(main())
|
forge/calibration.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# ── SAFE App Store vendor note (The Forge, calibration ledger, 2026-08-11) ────
|
|
3
|
+
# Vendored byte-for-byte (except this header block) from the oakenscrolls-office
|
|
4
|
+
# playground build in THIS repo: `apps/oakenscrolls-office/calibration.py`
|
|
5
|
+
# (b17: OAKOF), Apache-2.0 — same license as this store, zero friction.
|
|
6
|
+
#
|
|
7
|
+
# Why vendor rather than import the playground copy: `apps/` is the contested
|
|
8
|
+
# commons (rule 5) — a store-side organ (D1) must not import a build out of the
|
|
9
|
+
# untrusted playground, and `apps/oakenscrolls-office/` is not on the store's
|
|
10
|
+
# import path anyway. This is pure stdlib (`math`), ~40 lines, small enough to
|
|
11
|
+
# audit in one sitting — the same "vendor the core, not the package" discipline
|
|
12
|
+
# `friction_floor`/`human_loop` already follow. Rule 11: the house already grew
|
|
13
|
+
# this (oakenscrolls-office's confidence mirror, the box's calibration tool);
|
|
14
|
+
# the Forge re-points it at the MODEL's own confidence instead of rebuilding the
|
|
15
|
+
# Brier/log-score/reliability math. Kept diffable against the playground copy on
|
|
16
|
+
# purpose — if oakenscrolls' copy moves, `diff` this against it and reconcile.
|
|
17
|
+
#
|
|
18
|
+
# A "pair" everywhere below is (confidence, outcome): the final P(true) the
|
|
19
|
+
# predictor stated, and whether the claim turned out true.
|
|
20
|
+
"""
|
|
21
|
+
calibration.py — pure calibration math. b17: OAKOF
|
|
22
|
+
|
|
23
|
+
Dependency-free, stdlib-only, in the no-egress zone (tests/test_no_egress.py).
|
|
24
|
+
The vendoring pattern is utety/core/mastery.py: the ~40 lines of inference a
|
|
25
|
+
device actually needs, small enough to audit in one sitting.
|
|
26
|
+
|
|
27
|
+
A "pair" everywhere below is (confidence, outcome): the final P(true) the
|
|
28
|
+
person stated, and whether the claim turned out true.
|
|
29
|
+
"""
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import math
|
|
33
|
+
|
|
34
|
+
_EPS = 1e-9
|
|
35
|
+
|
|
36
|
+
# House convention (design D3): claims are stated in the direction believed,
|
|
37
|
+
# so confidence lives in [0.5, 0.99]. Bins cover that range in five steps.
|
|
38
|
+
BIN_EDGES = (0.5, 0.6, 0.7, 0.8, 0.9, 1.0)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def brier(confidence: float, outcome: bool) -> float:
|
|
42
|
+
"""Squared distance between what you said and what happened. 0 is prophecy,
|
|
43
|
+
0.25 is coin-flipping at 50%, 1.0 is confidently wrong."""
|
|
44
|
+
return (confidence - (1.0 if outcome else 0.0)) ** 2
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def log_score(confidence: float, outcome: bool) -> float:
|
|
48
|
+
"""Negative log likelihood of the outcome under your stated belief.
|
|
49
|
+
Punishes confident wrongness much harder than Brier does."""
|
|
50
|
+
p = confidence if outcome else 1.0 - confidence
|
|
51
|
+
return -math.log(max(p, _EPS))
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def bins(pairs: list[tuple[float, bool]]) -> list[dict]:
|
|
55
|
+
"""Reliability table: for each stated-confidence band, how often were you
|
|
56
|
+
actually right? Empty bands are returned with n=0 so the mirror shows the
|
|
57
|
+
whole range, not just where you've dared to speak."""
|
|
58
|
+
out = []
|
|
59
|
+
for lo, hi in zip(BIN_EDGES, BIN_EDGES[1:]):
|
|
60
|
+
members = [(c, o) for c, o in pairs if lo <= c < hi or (hi == 1.0 and c == 1.0)]
|
|
61
|
+
n = len(members)
|
|
62
|
+
out.append({
|
|
63
|
+
"lo": lo,
|
|
64
|
+
"hi": hi,
|
|
65
|
+
"n": n,
|
|
66
|
+
"mean_confidence": sum(c for c, _ in members) / n if n else None,
|
|
67
|
+
"hit_rate": sum(1 for _, o in members if o) / n if n else None,
|
|
68
|
+
})
|
|
69
|
+
return out
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def summary(pairs: list[tuple[float, bool]]) -> dict:
|
|
73
|
+
"""The scorecard header. `overconfidence` is mean stated confidence minus
|
|
74
|
+
actual hit rate: positive means you promise more than you deliver."""
|
|
75
|
+
n = len(pairs)
|
|
76
|
+
if not n:
|
|
77
|
+
return {
|
|
78
|
+
"n": 0,
|
|
79
|
+
"brier": None,
|
|
80
|
+
"log_score": None,
|
|
81
|
+
"mean_confidence": None,
|
|
82
|
+
"hit_rate": None,
|
|
83
|
+
"overconfidence": None,
|
|
84
|
+
}
|
|
85
|
+
mean_conf = sum(c for c, _ in pairs) / n
|
|
86
|
+
hit_rate = sum(1 for _, o in pairs if o) / n
|
|
87
|
+
return {
|
|
88
|
+
"n": n,
|
|
89
|
+
"brier": sum(brier(c, o) for c, o in pairs) / n,
|
|
90
|
+
"log_score": sum(log_score(c, o) for c, o in pairs) / n,
|
|
91
|
+
"mean_confidence": mean_conf,
|
|
92
|
+
"hit_rate": hit_rate,
|
|
93
|
+
"overconfidence": mean_conf - hit_rate,
|
|
94
|
+
}
|
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""forge/calibration_ledger.py — the model's confidence mirror
|
|
3
|
+
(docs/design/the-forge-measure.md, the `calibration` class).
|
|
4
|
+
|
|
5
|
+
The measuring panel refuses a confident wrong ARTIFACT; the checkpoint loop
|
|
6
|
+
refuses a confident wrong DECISION. This refuses a confident wrong PREDICTOR —
|
|
7
|
+
the model itself. It is the panel's `calibration` class, but it is NOT a per-build
|
|
8
|
+
instrument like `census`/`execution`: calibration is a claim about the model
|
|
9
|
+
ACROSS builds, so it is a longitudinal ledger, not a directory measurement.
|
|
10
|
+
|
|
11
|
+
The mechanism the box taught (oakenscrolls-office, the confidence mirror): a
|
|
12
|
+
prediction is a `(confidence, outcome)` pair — the model states P(true) for a
|
|
13
|
+
claim it makes while building ("this file parses", "this design holds", "this
|
|
14
|
+
dependency exists"), and later ground truth arrives (a test runs, the panel
|
|
15
|
+
measures, a human rules). Grade the model's stated confidence against what
|
|
16
|
+
actually happened: `brier`, `log_score`, and the reliability table (`bins`) —
|
|
17
|
+
all from the vendored `forge/calibration.py` (rule 11: the math is
|
|
18
|
+
oakenscrolls', not rebuilt here). The one signal that matters is
|
|
19
|
+
`overconfidence` — mean stated confidence minus actual hit rate; positive means
|
|
20
|
+
the model promises more than it delivers.
|
|
21
|
+
|
|
22
|
+
Persistence is the same per-builder SOIL file the governance layer already uses
|
|
23
|
+
(`forge/soil_store.py`, D6: a builder's records live in that builder's own file
|
|
24
|
+
by construction), under this module's own collection. The overconfidence signal
|
|
25
|
+
routes through `checkpoint_governance.route_nudge` (rule 11: reuse the deduped
|
|
26
|
+
outbox bite 3 / #67 and the panel already feed) — it surfaces a `review` item a
|
|
27
|
+
human should see; it NEVER blocks a build, exactly like every other model-side
|
|
28
|
+
signal.
|
|
29
|
+
|
|
30
|
+
Store-side (D1): `apps/the-forge/` never imports this — a sandboxed build does
|
|
31
|
+
not grade its own model's confidence, any more than it marks its own homework.
|
|
32
|
+
"""
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
import argparse
|
|
36
|
+
import hashlib
|
|
37
|
+
import json
|
|
38
|
+
from pathlib import Path
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
from . import calibration
|
|
43
|
+
from . import soil_store
|
|
44
|
+
from . import checkpoint_governance as governance
|
|
45
|
+
|
|
46
|
+
# Predictions live in this module's own collection inside the builder's SOIL
|
|
47
|
+
# file — alongside (never colliding with) the governance queue collection.
|
|
48
|
+
PREDICTIONS = "calibration_predictions"
|
|
49
|
+
|
|
50
|
+
# The overconfidence signal's defaults. `MIN_N` is the floor below which a
|
|
51
|
+
# scorecard is too thin to route on (two lucky calls are not a calibration);
|
|
52
|
+
# `OVERCONFIDENCE_FLOOR` is how far stated confidence may exceed hit rate before
|
|
53
|
+
# it is worth a human's eyes. Both are overridable per call.
|
|
54
|
+
MIN_N = 5
|
|
55
|
+
OVERCONFIDENCE_FLOOR = 0.10
|
|
56
|
+
|
|
57
|
+
# One standing open flag per builder: route_nudge dedupes by source_ref, so a
|
|
58
|
+
# persistently overconfident model surfaces ONE open `review` item, not a new
|
|
59
|
+
# one every resolve. When a human resolves that item and the model is still
|
|
60
|
+
# overconfident later, a fresh open item can form (route_nudge only dedupes
|
|
61
|
+
# against OPEN rows) — the correct "one live flag at a time" behavior.
|
|
62
|
+
_SIGNAL_SOURCE_REF_PREFIX = "calibration-overconfidence:"
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class CalibrationLedgerError(Exception):
|
|
66
|
+
"""This module's own refusal — a confidence outside the believed-direction
|
|
67
|
+
range, or a prediction lifecycle violation (re-predicting a settled claim,
|
|
68
|
+
resolving an unknown or already-resolved one). `soil_store`'s
|
|
69
|
+
`SoilStoreError` and `governance`'s errors propagate unwrapped."""
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _store(builder_id: str, root: Path | None):
|
|
73
|
+
kw = {} if root is None else {"root": Path(root)}
|
|
74
|
+
return soil_store.FilesystemSoilStore(builder_id, **kw)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _check_confidence(confidence: float) -> float:
|
|
78
|
+
"""House convention (calibration D3): a claim is stated in the direction
|
|
79
|
+
believed, so confidence is P(the claim is true) in [0.5, 1.0]. A value below
|
|
80
|
+
0.5 means the model actually believes the claim FALSE — it must be restated
|
|
81
|
+
in the believed direction, not recorded as a weak 'true'. Reject rather than
|
|
82
|
+
silently clamp: a clamp would fabricate a data point the model never made."""
|
|
83
|
+
try:
|
|
84
|
+
c = float(confidence)
|
|
85
|
+
except (TypeError, ValueError) as e:
|
|
86
|
+
raise CalibrationLedgerError(f"confidence must be a number, got {confidence!r}") from e
|
|
87
|
+
if not (0.5 <= c <= 1.0):
|
|
88
|
+
raise CalibrationLedgerError(
|
|
89
|
+
f"confidence {c} outside [0.5, 1.0] — state the claim in the "
|
|
90
|
+
f"direction believed (calibration D3), a claim you think false is a "
|
|
91
|
+
f"true-claim restated, not a confidence below 0.5"
|
|
92
|
+
)
|
|
93
|
+
return c
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _prediction_id(claim: str) -> str:
|
|
97
|
+
"""A stable id for a claim with no explicit id — deterministic (no wall
|
|
98
|
+
clock, no randomness), so re-stating the same claim addresses the same
|
|
99
|
+
pending prediction rather than spawning a duplicate."""
|
|
100
|
+
h = hashlib.sha256(claim.encode("utf-8")).hexdigest()[:16]
|
|
101
|
+
return f"pred-{h}"
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def record_prediction(
|
|
105
|
+
builder_id: str,
|
|
106
|
+
claim: str,
|
|
107
|
+
confidence: float,
|
|
108
|
+
*,
|
|
109
|
+
prediction_id: str | None = None,
|
|
110
|
+
kind: str = "",
|
|
111
|
+
root: Path | None = None,
|
|
112
|
+
) -> dict:
|
|
113
|
+
"""Record the model's stated confidence in a claim, pending ground truth.
|
|
114
|
+
|
|
115
|
+
`confidence` is P(claim true) in [0.5, 1.0]. Idempotent on the claim: with
|
|
116
|
+
no `prediction_id`, the id is derived from the claim text, so re-stating the
|
|
117
|
+
same claim UPDATES its (still-pending) confidence rather than duplicating it.
|
|
118
|
+
Refuses to re-record a claim that has already been RESOLVED — a settled data
|
|
119
|
+
point is history, not something a later re-prediction may overwrite."""
|
|
120
|
+
if not isinstance(claim, str) or not claim.strip():
|
|
121
|
+
raise CalibrationLedgerError("claim must be a non-empty string")
|
|
122
|
+
c = _check_confidence(confidence)
|
|
123
|
+
pid = prediction_id or _prediction_id(claim)
|
|
124
|
+
store = _store(builder_id, root)
|
|
125
|
+
existing = store.get(PREDICTIONS, pid)
|
|
126
|
+
if existing is not None and existing.get("resolved"):
|
|
127
|
+
raise CalibrationLedgerError(
|
|
128
|
+
f"prediction {pid!r} is already resolved (outcome="
|
|
129
|
+
f"{existing.get('outcome')!r}) — a settled prediction is not "
|
|
130
|
+
f"re-recordable; use a fresh prediction_id for a new claim"
|
|
131
|
+
)
|
|
132
|
+
record = {
|
|
133
|
+
"id": pid,
|
|
134
|
+
"claim": claim,
|
|
135
|
+
"confidence": c,
|
|
136
|
+
"kind": kind,
|
|
137
|
+
"outcome": None,
|
|
138
|
+
"resolved": False,
|
|
139
|
+
}
|
|
140
|
+
store.put(PREDICTIONS, record, record_id=pid)
|
|
141
|
+
return record
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def resolve_prediction(
|
|
145
|
+
builder_id: str,
|
|
146
|
+
prediction_id: str,
|
|
147
|
+
outcome: bool,
|
|
148
|
+
*,
|
|
149
|
+
root: Path | None = None,
|
|
150
|
+
) -> dict:
|
|
151
|
+
"""Settle a pending prediction with ground truth (`outcome` True/False).
|
|
152
|
+
|
|
153
|
+
Refuses an unknown id, and refuses to re-resolve an already-settled one — a
|
|
154
|
+
double-resolve would silently rewrite history (the same single-use guard the
|
|
155
|
+
governance resume seam already learned to keep)."""
|
|
156
|
+
store = _store(builder_id, root)
|
|
157
|
+
record = store.get(PREDICTIONS, prediction_id)
|
|
158
|
+
if record is None:
|
|
159
|
+
raise CalibrationLedgerError(f"no prediction {prediction_id!r} to resolve")
|
|
160
|
+
if record.get("resolved"):
|
|
161
|
+
raise CalibrationLedgerError(
|
|
162
|
+
f"prediction {prediction_id!r} already resolved (outcome="
|
|
163
|
+
f"{record.get('outcome')!r}) — a settled prediction is single-use"
|
|
164
|
+
)
|
|
165
|
+
record["outcome"] = bool(outcome)
|
|
166
|
+
record["resolved"] = True
|
|
167
|
+
store.put(PREDICTIONS, record, record_id=prediction_id)
|
|
168
|
+
return record
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def scorecard(builder_id: str, *, root: Path | None = None) -> dict:
|
|
172
|
+
"""The model's calibration mirror: the `calibration.summary` header (brier,
|
|
173
|
+
log_score, hit_rate, overconfidence) plus the `calibration.bins` reliability
|
|
174
|
+
table, over the builder's RESOLVED predictions, with a pending count so a
|
|
175
|
+
thin scorecard reads as thin, not as confident."""
|
|
176
|
+
store = _store(builder_id, root)
|
|
177
|
+
all_recs = store.all(PREDICTIONS)
|
|
178
|
+
pairs = [(float(r["confidence"]), bool(r["outcome"])) for r in all_recs if r.get("resolved")]
|
|
179
|
+
return {
|
|
180
|
+
"builder_id": store.builder_id,
|
|
181
|
+
"resolved": len(pairs),
|
|
182
|
+
"pending": sum(1 for r in all_recs if not r.get("resolved")),
|
|
183
|
+
"summary": calibration.summary(pairs),
|
|
184
|
+
"bins": calibration.bins(pairs),
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def overconfidence_signal(
|
|
189
|
+
builder_id: str,
|
|
190
|
+
*,
|
|
191
|
+
root: Path | None = None,
|
|
192
|
+
min_n: int = MIN_N,
|
|
193
|
+
floor: float = OVERCONFIDENCE_FLOOR,
|
|
194
|
+
) -> dict | None:
|
|
195
|
+
"""Route ONE `review` nudge when the model is measurably overconfident.
|
|
196
|
+
|
|
197
|
+
Fires only when there are at least `min_n` resolved predictions (a thin
|
|
198
|
+
record is not a calibration) AND `overconfidence` (mean stated confidence −
|
|
199
|
+
hit rate) is at least `floor`. Reuses `checkpoint_governance.route_nudge`,
|
|
200
|
+
so it is deduped by a per-builder `source_ref` (one standing open flag) and
|
|
201
|
+
persisted in the same `human_required` queue every other model-side signal
|
|
202
|
+
feeds. Returns the routed item, or None when the bar is not met OR the flag
|
|
203
|
+
is already open (route_nudge's dedupe). NEVER blocks — it surfaces."""
|
|
204
|
+
card = scorecard(builder_id, root=root)
|
|
205
|
+
n = card["resolved"]
|
|
206
|
+
over = card["summary"]["overconfidence"]
|
|
207
|
+
if n < min_n or over is None or over < floor:
|
|
208
|
+
return None
|
|
209
|
+
return governance.route_nudge(
|
|
210
|
+
builder_id,
|
|
211
|
+
kind="review",
|
|
212
|
+
title="model overconfidence",
|
|
213
|
+
summary=(
|
|
214
|
+
f"calibration: {n} resolved predictions, stated confidence "
|
|
215
|
+
f"{card['summary']['mean_confidence']:.2f} vs hit rate "
|
|
216
|
+
f"{card['summary']['hit_rate']:.2f} (overconfidence {over:+.2f}, "
|
|
217
|
+
f"brier {card['summary']['brier']:.3f}). The model promises more "
|
|
218
|
+
f"than it delivers — a human should recalibrate what it asserts."
|
|
219
|
+
),
|
|
220
|
+
source_ref=f"{_SIGNAL_SOURCE_REF_PREFIX}{card['builder_id']}",
|
|
221
|
+
priority="normal",
|
|
222
|
+
root=soil_store.DEFAULT_CHECKPOINT_ROOT if root is None else Path(root),
|
|
223
|
+
)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
# ── CLI ──────────────────────────────────────────────────────────────────────
|
|
227
|
+
|
|
228
|
+
def _cmd_record(args: argparse.Namespace) -> int:
|
|
229
|
+
rec = record_prediction(
|
|
230
|
+
args.builder_id, args.claim, args.confidence,
|
|
231
|
+
prediction_id=args.id, kind=args.kind or "", root=Path(args.root) if args.root else None,
|
|
232
|
+
)
|
|
233
|
+
print(json.dumps(rec, indent=2))
|
|
234
|
+
return 0
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _cmd_resolve(args: argparse.Namespace) -> int:
|
|
238
|
+
rec = resolve_prediction(
|
|
239
|
+
args.builder_id, args.id, args.outcome == "true",
|
|
240
|
+
root=Path(args.root) if args.root else None,
|
|
241
|
+
)
|
|
242
|
+
print(json.dumps(rec, indent=2))
|
|
243
|
+
return 0
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _cmd_scorecard(args: argparse.Namespace) -> int:
|
|
247
|
+
card = scorecard(args.builder_id, root=Path(args.root) if args.root else None)
|
|
248
|
+
print(json.dumps(card, indent=2))
|
|
249
|
+
# a routed overconfidence flag is a CI-visible signal (exit 1), never a block
|
|
250
|
+
routed = overconfidence_signal(args.builder_id, root=Path(args.root) if args.root else None)
|
|
251
|
+
return 1 if routed else 0
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
255
|
+
p = argparse.ArgumentParser(prog="calibration_ledger.py")
|
|
256
|
+
sub = p.add_subparsers(dest="command", required=True)
|
|
257
|
+
|
|
258
|
+
r = sub.add_parser("record", help="record a pending prediction (confidence in [0.5, 1.0])")
|
|
259
|
+
r.add_argument("builder_id")
|
|
260
|
+
r.add_argument("claim")
|
|
261
|
+
r.add_argument("confidence", type=float)
|
|
262
|
+
r.add_argument("--id", default=None)
|
|
263
|
+
r.add_argument("--kind", default="")
|
|
264
|
+
r.add_argument("--root", default=None)
|
|
265
|
+
r.set_defaults(func=_cmd_record)
|
|
266
|
+
|
|
267
|
+
v = sub.add_parser("resolve", help="settle a prediction with ground truth")
|
|
268
|
+
v.add_argument("builder_id")
|
|
269
|
+
v.add_argument("id")
|
|
270
|
+
v.add_argument("outcome", choices=["true", "false"])
|
|
271
|
+
v.add_argument("--root", default=None)
|
|
272
|
+
v.set_defaults(func=_cmd_resolve)
|
|
273
|
+
|
|
274
|
+
s = sub.add_parser("scorecard", help="the model's calibration mirror (+ routes an overconfidence nudge)")
|
|
275
|
+
s.add_argument("builder_id")
|
|
276
|
+
s.add_argument("--root", default=None)
|
|
277
|
+
s.set_defaults(func=_cmd_scorecard)
|
|
278
|
+
|
|
279
|
+
return p
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def main(argv: list[str] | None = None) -> int:
|
|
283
|
+
args = build_parser().parse_args(argv)
|
|
284
|
+
return args.func(args)
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
if __name__ == "__main__":
|
|
288
|
+
raise SystemExit(main())
|