mjolnir-ml 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mjolnir_ml/__init__.py +102 -0
- mjolnir_ml/_digest.py +41 -0
- mjolnir_ml/calibration.py +516 -0
- mjolnir_ml/causal.py +1251 -0
- mjolnir_ml/certificate.py +241 -0
- mjolnir_ml/costs.py +152 -0
- mjolnir_ml/discovery.py +966 -0
- mjolnir_ml/dsl.py +458 -0
- mjolnir_ml/feasibility.py +264 -0
- mjolnir_ml/hazards.py +181 -0
- mjolnir_ml/live.py +786 -0
- mjolnir_ml/model.py +794 -0
- mjolnir_ml/pair.py +131 -0
- mjolnir_ml/policy.py +176 -0
- mjolnir_ml/readiness.py +233 -0
- mjolnir_ml/regime.py +187 -0
- mjolnir_ml/report.py +394 -0
- mjolnir_ml/risk.py +345 -0
- mjolnir_ml/vendors/__init__.py +10 -0
- mjolnir_ml/vendors/core/__init__.py +5 -0
- mjolnir_ml/vendors/core/verdict.py +194 -0
- mjolnir_ml/vendors/wire.py +47 -0
- mjolnir_ml-0.1.0.dist-info/METADATA +864 -0
- mjolnir_ml-0.1.0.dist-info/RECORD +27 -0
- mjolnir_ml-0.1.0.dist-info/WHEEL +5 -0
- mjolnir_ml-0.1.0.dist-info/licenses/LICENSE +133 -0
- mjolnir_ml-0.1.0.dist-info/top_level.txt +1 -0
mjolnir_ml/__init__.py
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""mjolnir_ml — turn a fitted predictor into a fail-closed decision model.
|
|
2
|
+
|
|
3
|
+
from mjolnir_ml import dsl, rule, DecisionModel, DecisionPolicyBase, \
|
|
4
|
+
RELEASE, ASK, DENY, omega_pair
|
|
5
|
+
|
|
6
|
+
@dsl
|
|
7
|
+
class Policy(DecisionPolicyBase):
|
|
8
|
+
assert lambda amount: amount > 0, "amount_positive"
|
|
9
|
+
assert rule("auto_release",
|
|
10
|
+
when=lambda p_positive, conf: p_positive < 0.05
|
|
11
|
+
and conf >= 0.95,
|
|
12
|
+
verdict=RELEASE)
|
|
13
|
+
|
|
14
|
+
dm = DecisionModel(fitted_sklearn_estimator, Policy, features=(...))
|
|
15
|
+
dm.fit_regime(train_rows) # fail-closed regime floor
|
|
16
|
+
disp = dm.dispose(row) # release | ask | deny | abstain
|
|
17
|
+
|
|
18
|
+
STANDALONE: imports nothing outside this package at runtime; the frozen rule
|
|
19
|
+
engine and the wire projection are vendored under mjolnir_ml/vendors.
|
|
20
|
+
"""
|
|
21
|
+
from mjolnir_ml.causal import (CausalClaim, CausalError, CausalOracle,
|
|
22
|
+
CausalSession, Estimate, Series, Strategy,
|
|
23
|
+
Table, do, event_study, series_from_journal)
|
|
24
|
+
from mjolnir_ml.calibration import (AdaptiveKappa, CalibrationError, Kappa,
|
|
25
|
+
REGRESSOR_METHODS, fit_kappa)
|
|
26
|
+
from mjolnir_ml.certificate import (CERT_SCHEMA, CertificateError, certify,
|
|
27
|
+
verify_certificate)
|
|
28
|
+
from mjolnir_ml.costs import Costs, cost_policy
|
|
29
|
+
from mjolnir_ml.feasibility import (FeasibilityError, FeasibilityReport,
|
|
30
|
+
RuleCheck, feasibility)
|
|
31
|
+
from mjolnir_ml.regime import MultivariateRegimeFloor, RegimeError
|
|
32
|
+
from mjolnir_ml.risk import (RiskError, RiskIndex, adverse_event,
|
|
33
|
+
crc_threshold, crc_thresholds, fit_risk_index)
|
|
34
|
+
from mjolnir_ml.dsl import (DecisionPolicyBase, DslError, compile_class, dsl,
|
|
35
|
+
rule)
|
|
36
|
+
from mjolnir_ml.model import (BindError, DecisionModel, Disposition)
|
|
37
|
+
from mjolnir_ml.hazards import (Hazard, HazardLedger,
|
|
38
|
+
measure_completeness)
|
|
39
|
+
from mjolnir_ml.discovery import (CERTIFIED, CENSUS_VERDICTS, DECLARED,
|
|
40
|
+
DECLINED, REFUTED, Admission, Candidate,
|
|
41
|
+
Census, Containment as DiscoveryContainment,
|
|
42
|
+
DiscoveryDetection, DiscoveryDetector,
|
|
43
|
+
DiscoveryError, DiscoveryEvaluation,
|
|
44
|
+
DiscoveryGate, DiscoverySession,
|
|
45
|
+
KnowledgeLedger, KnowledgeRecord,
|
|
46
|
+
certify_admission,
|
|
47
|
+
default_discovery_detectors,
|
|
48
|
+
verify_admission_certificate)
|
|
49
|
+
from mjolnir_ml.live import (Approval, ApprovalAuthority, ChangeGate,
|
|
50
|
+
ChangeSet, Containment, Detection, DetectorSpec,
|
|
51
|
+
Evaluation, Journal, LiveSession, Promotion,
|
|
52
|
+
Retraction, default_detectors)
|
|
53
|
+
from mjolnir_ml.readiness import (Gate, ReadinessError, ReadinessReport,
|
|
54
|
+
ReadinessThresholds, readiness)
|
|
55
|
+
from mjolnir_ml.pair import (Pair, audit, audit_model, degenerate_controls,
|
|
56
|
+
omega_pair, wilson)
|
|
57
|
+
from mjolnir_ml.policy import MODEL_FACTS, Policy, Statement
|
|
58
|
+
from mjolnir_ml.report import report, report_data
|
|
59
|
+
from mjolnir_ml.vendors.wire import (ABSTAIN_W, ASK, DENY_W, RANK, RELEASE,
|
|
60
|
+
WIRE, project, strictest)
|
|
61
|
+
|
|
62
|
+
ASK = ASK # wire vocabulary: release | ask | deny | abstain
|
|
63
|
+
DENY = DENY_W
|
|
64
|
+
ABSTAIN = ABSTAIN_W
|
|
65
|
+
|
|
66
|
+
__version__ = "0.1.0"
|
|
67
|
+
|
|
68
|
+
__all__ = [
|
|
69
|
+
"dsl", "rule", "compile_class", "DecisionPolicyBase", "DslError",
|
|
70
|
+
"DecisionModel", "Disposition", "BindError",
|
|
71
|
+
"Policy", "Statement", "MODEL_FACTS",
|
|
72
|
+
"Kappa", "AdaptiveKappa", "fit_kappa", "CalibrationError",
|
|
73
|
+
"REGRESSOR_METHODS",
|
|
74
|
+
"Costs", "cost_policy",
|
|
75
|
+
"RegimeError", "MultivariateRegimeFloor",
|
|
76
|
+
"RiskError", "RiskIndex", "adverse_event", "crc_threshold",
|
|
77
|
+
"crc_thresholds", "fit_risk_index",
|
|
78
|
+
"FeasibilityError", "FeasibilityReport", "RuleCheck", "feasibility",
|
|
79
|
+
"certify", "verify_certificate", "CertificateError", "CERT_SCHEMA",
|
|
80
|
+
"Hazard", "HazardLedger", "measure_completeness",
|
|
81
|
+
"readiness", "ReadinessReport", "ReadinessThresholds", "Gate",
|
|
82
|
+
"ReadinessError",
|
|
83
|
+
"LiveSession", "Journal", "DetectorSpec", "Detection",
|
|
84
|
+
"default_detectors", "Containment", "Retraction",
|
|
85
|
+
"ChangeSet", "ChangeGate", "Evaluation", "Promotion",
|
|
86
|
+
"Approval", "ApprovalAuthority",
|
|
87
|
+
"DiscoverySession", "Census", "Candidate", "CENSUS_VERDICTS",
|
|
88
|
+
"CERTIFIED", "DECLARED", "DECLINED", "REFUTED",
|
|
89
|
+
"DiscoveryGate", "DiscoveryEvaluation", "Admission",
|
|
90
|
+
"KnowledgeLedger", "KnowledgeRecord", "DiscoveryError",
|
|
91
|
+
"DiscoveryDetector", "DiscoveryDetection",
|
|
92
|
+
"default_discovery_detectors", "DiscoveryContainment",
|
|
93
|
+
"certify_admission", "verify_admission_certificate",
|
|
94
|
+
"CausalSession", "CausalClaim", "CausalOracle", "CausalError",
|
|
95
|
+
"Strategy", "do", "Table", "Series", "Estimate",
|
|
96
|
+
"event_study", "series_from_journal",
|
|
97
|
+
"Pair", "omega_pair", "wilson", "audit", "audit_model",
|
|
98
|
+
"report", "report_data",
|
|
99
|
+
"degenerate_controls",
|
|
100
|
+
"RELEASE", "ASK", "DENY", "ABSTAIN", "WIRE", "RANK", "project",
|
|
101
|
+
"strictest", "__version__",
|
|
102
|
+
]
|
mjolnir_ml/_digest.py
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""mjolnir_ml._digest — content addressing shared by every calibrated index.
|
|
2
|
+
|
|
3
|
+
Digests are over canonical JSON (sorted keys, no whitespace) so a table and
|
|
4
|
+
its digest cannot drift. Estimator digests include the fitted-state attributes
|
|
5
|
+
we can read generically; they are audit aids, not a substitute for pinning the
|
|
6
|
+
artifact itself.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def digest(obj) -> str:
|
|
15
|
+
blob = json.dumps(obj, sort_keys=True, separators=(",", ":"),
|
|
16
|
+
default=repr, ensure_ascii=False).encode("utf-8")
|
|
17
|
+
return "sha256:" + hashlib.sha256(blob).hexdigest()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def estimator_digest(estimator) -> str:
|
|
21
|
+
"""Deterministic class + params + readable fitted-state digest."""
|
|
22
|
+
parts = [type(estimator).__module__ + "." + type(estimator).__name__]
|
|
23
|
+
if hasattr(estimator, "get_params"):
|
|
24
|
+
try:
|
|
25
|
+
params = estimator.get_params(deep=False)
|
|
26
|
+
parts.append(json.dumps({k: repr(v)
|
|
27
|
+
for k, v in sorted(params.items())},
|
|
28
|
+
sort_keys=True))
|
|
29
|
+
except Exception: # noqa: BLE001
|
|
30
|
+
parts.append(repr(estimator))
|
|
31
|
+
else:
|
|
32
|
+
parts.append(repr(estimator))
|
|
33
|
+
for attr in ("classes_", "coef_", "intercept_", "n_features_in_",
|
|
34
|
+
"feature_importances_"):
|
|
35
|
+
val = getattr(estimator, attr, None)
|
|
36
|
+
if val is not None:
|
|
37
|
+
parts.append(f"{attr}={val!r}")
|
|
38
|
+
return digest(parts)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
__all__ = ["digest", "estimator_digest"]
|
|
@@ -0,0 +1,516 @@
|
|
|
1
|
+
"""mjolnir_ml.calibration — κ: the calibrated trust index over a learner.
|
|
2
|
+
|
|
3
|
+
THE LOAD-BEARING IDEA: what a learner is
|
|
4
|
+
allowed to *claim* is not its softmax. It is a finite-sample guarantee about
|
|
5
|
+
the learner's own error, estimated per region and pinned by digest. This module
|
|
6
|
+
implements split conformal calibration with optional Mondrian (group) splitting:
|
|
7
|
+
|
|
8
|
+
classifier nonconformity s(x, y) = 1 - p_y(x)
|
|
9
|
+
prediction set P(x) = { y : s(x, y) <= q(group(x)) }
|
|
10
|
+
certified |P(x)| == 1 (a guaranteed singleton)
|
|
11
|
+
|
|
12
|
+
regressor nonconformity s(x) = |y - f(x)|
|
|
13
|
+
interval [f(x) - q, f(x) + q] (coverage >= 1 - alpha)
|
|
14
|
+
|
|
15
|
+
Three regressor widths share that guarantee; the method only changes WHERE the
|
|
16
|
+
uncertainty is spent (all keep the same finite-sample coverage):
|
|
17
|
+
|
|
18
|
+
split s = |y - f(x)| width 2q (constant)
|
|
19
|
+
normalized s = |y - f(x)| / sigma(x) width 2q*sigma(x) (local)
|
|
20
|
+
cqr s = max(l(x)-y, y-u(x)) width (u-l)+2q (local)
|
|
21
|
+
|
|
22
|
+
`normalized` takes a fitted scale estimator (a residual model); `cqr` takes
|
|
23
|
+
fitted lower/upper quantile estimators. The auxiliary estimators are pinned in
|
|
24
|
+
the digest. A non-finite or non-positive scale, or an inverted CQR interval,
|
|
25
|
+
yields no interval at all (unknown is not zero) — the policy then abstains.
|
|
26
|
+
|
|
27
|
+
one_sided=True calibrates each tail at its own level instead of paying one
|
|
28
|
+
symmetric margin:
|
|
29
|
+
|
|
30
|
+
lo = bound_lo - q_lower P(Y >= lo) >= 1 - alpha_lower
|
|
31
|
+
hi = bound_hi + q_upper P(Y <= hi) >= 1 - alpha_upper
|
|
32
|
+
|
|
33
|
+
For quantile models bound_lo/bound_hi are l(x)/u(x); for split/normalized both
|
|
34
|
+
are f(x) (times sigma(x)). This is the statistically honest form of a two-track
|
|
35
|
+
policy (release reads the upper tail, deny reads the lower), but the pair is a
|
|
36
|
+
decision envelope: joint coverage is only >= 1 - alpha_lower - alpha_upper, so
|
|
37
|
+
the report labels its width accordingly.
|
|
38
|
+
|
|
39
|
+
q is the finite-sample conformal quantile ceil((n+1)(1-alpha)) / n. The floor
|
|
40
|
+
n >= ceil(1/alpha) - 1 is enforced per group: a region with too little data is
|
|
41
|
+
a calibration ERROR, not a silent uncalibrated claim ("unknown is not zero").
|
|
42
|
+
|
|
43
|
+
WHAT THIS BUYS A HIGH-RISK APPLICATION
|
|
44
|
+
--------------------------------------
|
|
45
|
+
The policy can say `certified` (a singleton at the declared alpha) instead of
|
|
46
|
+
`conf > 0.9` (an uncalibrated number). That is the difference between "the
|
|
47
|
+
softmax looked confident" and "at most alpha of these claims are wrong, over
|
|
48
|
+
exchangeable data, per region". For a regressor the policy gates on `lo`/`hi`:
|
|
49
|
+
an action is releasable only if its entire conformal interval clears the
|
|
50
|
+
declared threshold.
|
|
51
|
+
|
|
52
|
+
NON-CLAIMS: coverage is marginal-per-group and relative to exchangeability.
|
|
53
|
+
Under drift, coverage can degrade (the G4 result); the honest
|
|
54
|
+
responses are (a) regroup, (b) recalibrate, (c) adaptive conformal — not a
|
|
55
|
+
claim that the guarantee survives distribution shift.
|
|
56
|
+
"""
|
|
57
|
+
from __future__ import annotations
|
|
58
|
+
|
|
59
|
+
import math
|
|
60
|
+
|
|
61
|
+
from mjolnir_ml._digest import digest as _digest
|
|
62
|
+
from mjolnir_ml._digest import estimator_digest as _estimator_digest
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _scalar(v):
|
|
66
|
+
return v.item() if hasattr(v, "item") else v
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class CalibrationError(ValueError):
|
|
70
|
+
"""Calibration refused: too little data, wrong shapes, unfitted estimator."""
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def _quantile(sorted_scores: list, n: int, alpha: float) -> float:
|
|
74
|
+
"""Split-conformal quantile at level ceil((n+1)(1-alpha))/n."""
|
|
75
|
+
rank = math.ceil((n + 1) * (1.0 - alpha))
|
|
76
|
+
if rank > n:
|
|
77
|
+
# alpha too small for n -> the guarantee needs more data; refuse.
|
|
78
|
+
raise CalibrationError(
|
|
79
|
+
f"alpha={alpha} needs n >= ceil(1/alpha)-1; got n={n}")
|
|
80
|
+
return float(sorted_scores[rank - 1])
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _predict1(estimator, x) -> float:
|
|
86
|
+
return float(_scalar(estimator.predict([x])[0]))
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
REGRESSOR_METHODS = ("split", "normalized", "cqr")
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _reg_scores(method: str, aux: dict, x, pred, y) -> tuple:
|
|
95
|
+
"""Signed regressor scores: (underprediction, overprediction).
|
|
96
|
+
|
|
97
|
+
Both are positive when the realization is on the dangerous side of the
|
|
98
|
+
fitted bound: `lo_score = bound_lo - y` (Y below the lower bound) and
|
|
99
|
+
`hi_score = y - bound_hi` (Y above the upper bound). The two-sided score is
|
|
100
|
+
their max, which is exactly |y - f(x)| for split/normalized and the CQR
|
|
101
|
+
score for quantile models.
|
|
102
|
+
"""
|
|
103
|
+
pred = float(pred)
|
|
104
|
+
y = float(y)
|
|
105
|
+
if method == "normalized":
|
|
106
|
+
sigma = _predict1(aux["scale"], x)
|
|
107
|
+
if not math.isfinite(sigma) or sigma <= 0.0:
|
|
108
|
+
raise CalibrationError(
|
|
109
|
+
f"scale estimator emitted a non-positive/non-finite width "
|
|
110
|
+
f"({sigma!r}); normalized conformal cannot divide by it")
|
|
111
|
+
return ((pred - y) / sigma, (y - pred) / sigma)
|
|
112
|
+
if method == "cqr":
|
|
113
|
+
lo = _predict1(aux["lower"], x)
|
|
114
|
+
hi = _predict1(aux["upper"], x)
|
|
115
|
+
if not (math.isfinite(lo) and math.isfinite(hi)):
|
|
116
|
+
raise CalibrationError(
|
|
117
|
+
f"quantile estimators emitted non-finite bounds "
|
|
118
|
+
f"({lo!r}, {hi!r})")
|
|
119
|
+
return (lo - y, y - hi)
|
|
120
|
+
return (pred - y, y - pred)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _reg_score(method: str, aux: dict, x, pred, y) -> float:
|
|
124
|
+
"""Two-sided regressor nonconformity (max of the two signed scores)."""
|
|
125
|
+
lo_s, hi_s = _reg_scores(method, aux, x, pred, y)
|
|
126
|
+
return max(lo_s, hi_s)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class Kappa:
|
|
130
|
+
"""The trust index: per-group conformal quantiles + the data they came from."""
|
|
131
|
+
|
|
132
|
+
def __init__(self, kind: str, alpha: float, group: str | None,
|
|
133
|
+
classes: list | None, per_group: dict, n_total: int,
|
|
134
|
+
method: str = "split", aux: dict | None = None,
|
|
135
|
+
aux_digest: str | None = None, one_sided: bool = False,
|
|
136
|
+
alpha_lower: float | None = None,
|
|
137
|
+
alpha_upper: float | None = None):
|
|
138
|
+
self.kind = kind # "clf" | "reg"
|
|
139
|
+
self.alpha = float(alpha)
|
|
140
|
+
self.group = group # feature name or None (pooled)
|
|
141
|
+
self.classes = list(classes) if classes is not None else None
|
|
142
|
+
self.per_group = per_group # key -> {"q",...} or {"q_lo","q_hi",...}
|
|
143
|
+
self.n_total = n_total
|
|
144
|
+
self.method = method # "split" | "normalized" | "cqr"
|
|
145
|
+
self.aux = dict(aux or {}) # scale / lower / upper estimators
|
|
146
|
+
self.one_sided = bool(one_sided)
|
|
147
|
+
self.alpha_lower = float(alpha_lower if alpha_lower is not None
|
|
148
|
+
else alpha)
|
|
149
|
+
self.alpha_upper = float(alpha_upper if alpha_upper is not None
|
|
150
|
+
else alpha)
|
|
151
|
+
body = {
|
|
152
|
+
"kind": kind, "alpha": self.alpha, "group": group,
|
|
153
|
+
"classes": self.classes, "method": self.method,
|
|
154
|
+
"aux": aux_digest,
|
|
155
|
+
"groups": {k: {kk: vv for kk, vv in v.items()}
|
|
156
|
+
for k, v in per_group.items()}}
|
|
157
|
+
if self.one_sided:
|
|
158
|
+
body.update(one_sided=True, alpha_lower=self.alpha_lower,
|
|
159
|
+
alpha_upper=self.alpha_upper)
|
|
160
|
+
self._digest = _digest(body)
|
|
161
|
+
|
|
162
|
+
# ------------------------------------------------------------- access --
|
|
163
|
+
@property
|
|
164
|
+
def digest(self) -> str:
|
|
165
|
+
return self._digest
|
|
166
|
+
|
|
167
|
+
def group_key(self, row) -> str:
|
|
168
|
+
return str(row.get(self.group, "__unstated__")) if self.group \
|
|
169
|
+
else "__all__"
|
|
170
|
+
|
|
171
|
+
def calibrated(self, key: str) -> bool:
|
|
172
|
+
return key in self.per_group
|
|
173
|
+
|
|
174
|
+
def q(self, key: str) -> float | None:
|
|
175
|
+
g = self.per_group.get(key)
|
|
176
|
+
return None if g is None else g.get("q")
|
|
177
|
+
|
|
178
|
+
def q_lo(self, key: str) -> float | None:
|
|
179
|
+
g = self.per_group.get(key)
|
|
180
|
+
return None if g is None else g.get("q_lo")
|
|
181
|
+
|
|
182
|
+
def q_hi(self, key: str) -> float | None:
|
|
183
|
+
g = self.per_group.get(key)
|
|
184
|
+
return None if g is None else g.get("q_hi")
|
|
185
|
+
|
|
186
|
+
# ------------------------------------------------------------ predict --
|
|
187
|
+
def prediction_set(self, proba_by_class: dict, key: str) -> list:
|
|
188
|
+
"""Conformal prediction set. Unknown group -> ALL classes: the honest
|
|
189
|
+
statement is 'no calibrated set here', and a set of size |classes|
|
|
190
|
+
can never be certified as a singleton."""
|
|
191
|
+
q = self.q(key)
|
|
192
|
+
if q is None:
|
|
193
|
+
return list(self.classes or ())
|
|
194
|
+
return [c for c in self.classes if (1.0 - float(proba_by_class[c])) <= q]
|
|
195
|
+
|
|
196
|
+
def interval(self, pred: float, key: str, x=None) -> tuple | None:
|
|
197
|
+
"""Conformal bounds, or None when the group is uncalibrated (the
|
|
198
|
+
harness then omits lo/hi and the policy abstains on missing).
|
|
199
|
+
|
|
200
|
+
Two-sided: [pred - q, pred + q], a joint 1-alpha interval.
|
|
201
|
+
One-sided: [pred - q_lo, pred + q_hi], where each tail is separately
|
|
202
|
+
calibrated at 1-alpha_lower / 1-alpha_upper. The pair is a decision
|
|
203
|
+
envelope, not a joint 1-alpha interval: P(Y >= lo) >= 1-alpha_lower and
|
|
204
|
+
P(Y <= hi) >= 1-alpha_upper, so a joint conservative bound is
|
|
205
|
+
1-alpha_lower-alpha_upper (union bound).
|
|
206
|
+
|
|
207
|
+
Normalized and CQR widths need the input row `x`; when it is absent or
|
|
208
|
+
the auxiliary estimator fails, the honest answer is no bounds."""
|
|
209
|
+
g = self.per_group.get(key)
|
|
210
|
+
if g is None:
|
|
211
|
+
return None
|
|
212
|
+
pred = float(pred)
|
|
213
|
+
if self.method == "normalized":
|
|
214
|
+
if x is None:
|
|
215
|
+
return None
|
|
216
|
+
try:
|
|
217
|
+
sigma = _predict1(self.aux["scale"], x)
|
|
218
|
+
except Exception: # noqa: BLE001
|
|
219
|
+
return None
|
|
220
|
+
if not math.isfinite(sigma) or sigma <= 0.0:
|
|
221
|
+
return None
|
|
222
|
+
if self.one_sided:
|
|
223
|
+
return (pred - g["q_lo"] * sigma, pred + g["q_hi"] * sigma)
|
|
224
|
+
return (pred - g["q"] * sigma, pred + g["q"] * sigma)
|
|
225
|
+
if self.method == "cqr":
|
|
226
|
+
if x is None:
|
|
227
|
+
return None
|
|
228
|
+
try:
|
|
229
|
+
lo = _predict1(self.aux["lower"], x)
|
|
230
|
+
hi = _predict1(self.aux["upper"], x)
|
|
231
|
+
except Exception: # noqa: BLE001
|
|
232
|
+
return None
|
|
233
|
+
if not (math.isfinite(lo) and math.isfinite(hi)):
|
|
234
|
+
return None
|
|
235
|
+
if self.one_sided:
|
|
236
|
+
lo, hi = lo - g["q_lo"], hi + g["q_hi"]
|
|
237
|
+
else:
|
|
238
|
+
lo, hi = lo - g["q"], hi + g["q"]
|
|
239
|
+
return (lo, hi) if lo <= hi else None
|
|
240
|
+
if self.one_sided:
|
|
241
|
+
return (pred - g["q_lo"], pred + g["q_hi"])
|
|
242
|
+
return (pred - g["q"], pred + g["q"])
|
|
243
|
+
|
|
244
|
+
def reg_nonconformity(self, x, pred, y) -> float:
|
|
245
|
+
"""The realized score under this index's interval method (ACI feedback)."""
|
|
246
|
+
return _reg_score(self.method, self.aux, x, pred, y)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
class AdaptiveKappa(Kappa):
|
|
250
|
+
"""ACI: the per-group risk level moves from realized miscoverage.
|
|
251
|
+
|
|
252
|
+
Gibbs & Candes-style update after each observed nonconformity score s:
|
|
253
|
+
|
|
254
|
+
err_t = 1{ s_t > q_t }
|
|
255
|
+
alpha_{t+1} = clip( alpha_t + gamma * (alpha0 - err_t),
|
|
256
|
+
1/(n+2), 1 )
|
|
257
|
+
|
|
258
|
+
q_t is recomputed from the score window at level 1 - alpha_t, so the
|
|
259
|
+
prediction set / interval widens after errors and narrows when it is
|
|
260
|
+
over-covering. Static calibration under drift is the G4 failure: coverage
|
|
261
|
+
degrades and a window alone does not restore it. ACI restores coverage by
|
|
262
|
+
trading it for width — the price is reported, not hidden.
|
|
263
|
+
|
|
264
|
+
The DIGEST still pins the initial calibration artifact (the data), not the
|
|
265
|
+
moving runtime state; certificates issued before drift remain auditable.
|
|
266
|
+
"""
|
|
267
|
+
|
|
268
|
+
def __init__(self, kind, alpha, group, classes, per_group, n_total,
|
|
269
|
+
gamma=0.05, window=400, method="split", aux=None,
|
|
270
|
+
aux_digest=None, one_sided=False, alpha_lower=None,
|
|
271
|
+
alpha_upper=None):
|
|
272
|
+
super().__init__(kind, alpha, group, classes, per_group, n_total,
|
|
273
|
+
method=method, aux=aux, aux_digest=aux_digest,
|
|
274
|
+
one_sided=one_sided, alpha_lower=alpha_lower,
|
|
275
|
+
alpha_upper=alpha_upper)
|
|
276
|
+
if not (0.0 < float(gamma) <= 1.0):
|
|
277
|
+
raise CalibrationError(f"gamma must be in (0,1], got {gamma!r}")
|
|
278
|
+
if int(window) < 2:
|
|
279
|
+
raise CalibrationError(f"window must be >= 2, got {window!r}")
|
|
280
|
+
self.gamma = float(gamma)
|
|
281
|
+
self.window = int(window)
|
|
282
|
+
self._state = {k: {"scores": list(v["scores"]), "alpha": self.alpha}
|
|
283
|
+
for k, v in per_group.items()}
|
|
284
|
+
|
|
285
|
+
# ------------------------------------------------------------- dynamic --
|
|
286
|
+
def alpha_at(self, key: str) -> float | None:
|
|
287
|
+
st = self._state.get(key)
|
|
288
|
+
return None if st is None else st["alpha"]
|
|
289
|
+
|
|
290
|
+
def q(self, key: str) -> float | None:
|
|
291
|
+
st = self._state.get(key)
|
|
292
|
+
if st is None:
|
|
293
|
+
return None
|
|
294
|
+
scores = sorted(st["scores"])
|
|
295
|
+
n = len(scores)
|
|
296
|
+
rank = math.ceil((n + 1) * (1.0 - st["alpha"]))
|
|
297
|
+
rank = min(max(rank, 1), n)
|
|
298
|
+
return float(scores[rank - 1])
|
|
299
|
+
|
|
300
|
+
def observe(self, key: str, score: float) -> None:
|
|
301
|
+
"""One realized nonconformity score for the region `key`."""
|
|
302
|
+
st = self._state.get(key)
|
|
303
|
+
if st is None:
|
|
304
|
+
raise CalibrationError(
|
|
305
|
+
f"observe: region {key!r} was never calibrated — unknown is "
|
|
306
|
+
f"not zero; regroup or decline the feedback")
|
|
307
|
+
if not math.isfinite(float(score)):
|
|
308
|
+
raise CalibrationError("observe: non-finite score")
|
|
309
|
+
q = self.q(key)
|
|
310
|
+
err = 1.0 if (q is None or float(score) > q) else 0.0
|
|
311
|
+
st["alpha"] = min(1.0, max(1.0 / (len(st["scores"]) + 2),
|
|
312
|
+
st["alpha"] + self.gamma * (self.alpha - err)))
|
|
313
|
+
st["scores"].append(float(score))
|
|
314
|
+
if len(st["scores"]) > self.window:
|
|
315
|
+
del st["scores"][0]
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def fit_kappa(estimator, rows, y, alpha: float = 0.05,
|
|
319
|
+
group: str | None = None, features=None, adaptive: bool = False,
|
|
320
|
+
gamma: float = 0.05, window: int = 400, method: str = "split",
|
|
321
|
+
scale=None, lower=None, upper=None, one_sided: bool = False,
|
|
322
|
+
alpha_lower: float | None = None,
|
|
323
|
+
alpha_upper: float | None = None) -> Kappa:
|
|
324
|
+
"""Split-conformal calibration. rows: list[dict] in the model's feature
|
|
325
|
+
schema; y: ground truth per row; features: the estimator's input order
|
|
326
|
+
(defaults to the first row's keys). group: feature name for Mondrian.
|
|
327
|
+
|
|
328
|
+
method selects the regressor width: "split" (constant), "normalized"
|
|
329
|
+
(needs a fitted `scale` estimator), or "cqr" (needs fitted `lower`/`upper`
|
|
330
|
+
quantile estimators). All three keep the same finite-sample coverage; the
|
|
331
|
+
auxiliary estimators only spend the width where the error is larger.
|
|
332
|
+
|
|
333
|
+
one_sided=True calibrates the lower and upper tails separately:
|
|
334
|
+
`lo = bound_lo - q_lo` guarantees P(Y >= lo) >= 1 - alpha_lower and
|
|
335
|
+
`hi = bound_hi + q_hi` guarantees P(Y <= hi) >= 1 - alpha_upper. The two
|
|
336
|
+
tails may use different risk levels (e.g. a stricter deny tail), but the
|
|
337
|
+
pair is a decision envelope, not a joint 1-alpha interval. ACI and
|
|
338
|
+
one-sided tails are not combinable (adaptive needs one score per update).
|
|
339
|
+
|
|
340
|
+
Refuses (CalibrationError) rather than producing an uncalibrated claim:
|
|
341
|
+
wrong lengths, unfitted estimator, labels outside classes_, a group with
|
|
342
|
+
fewer than ceil(1/alpha)-1 samples, a group key not stated in a row, a
|
|
343
|
+
method that does not match the estimator kind, or a missing/invalid
|
|
344
|
+
auxiliary estimator.
|
|
345
|
+
"""
|
|
346
|
+
if not (0.0 < float(alpha) < 1.0):
|
|
347
|
+
raise CalibrationError(f"alpha must be in (0,1), got {alpha!r}")
|
|
348
|
+
alpha = float(alpha)
|
|
349
|
+
if method not in REGRESSOR_METHODS:
|
|
350
|
+
raise CalibrationError(
|
|
351
|
+
f"method must be one of {REGRESSOR_METHODS}, got {method!r}")
|
|
352
|
+
one_sided = bool(one_sided)
|
|
353
|
+
if one_sided:
|
|
354
|
+
a_lo = alpha if alpha_lower is None else float(alpha_lower)
|
|
355
|
+
a_hi = alpha if alpha_upper is None else float(alpha_upper)
|
|
356
|
+
for name, a in (("alpha_lower", a_lo), ("alpha_upper", a_hi)):
|
|
357
|
+
if not (0.0 < a < 1.0):
|
|
358
|
+
raise CalibrationError(f"{name} must be in (0,1), got {a!r}")
|
|
359
|
+
if adaptive:
|
|
360
|
+
raise CalibrationError(
|
|
361
|
+
"one-sided bounds and adaptive ACI are not combinable; use "
|
|
362
|
+
"static one-sided or adaptive two-sided")
|
|
363
|
+
elif alpha_lower is not None or alpha_upper is not None:
|
|
364
|
+
raise CalibrationError(
|
|
365
|
+
"alpha_lower/alpha_upper need one_sided=True")
|
|
366
|
+
else:
|
|
367
|
+
a_lo = a_hi = alpha
|
|
368
|
+
rows = list(rows)
|
|
369
|
+
y = list(y)
|
|
370
|
+
if len(rows) != len(y) or not rows:
|
|
371
|
+
raise CalibrationError(
|
|
372
|
+
f"need equal non-empty rows/y, got {len(rows)}/{len(y)}")
|
|
373
|
+
if group is not None and not isinstance(group, str):
|
|
374
|
+
raise CalibrationError("group must be a feature name or None")
|
|
375
|
+
if group is not None and any(group not in r for r in rows):
|
|
376
|
+
raise CalibrationError(f"group feature {group!r} absent from some rows")
|
|
377
|
+
|
|
378
|
+
has_proba = callable(getattr(estimator, "predict_proba", None))
|
|
379
|
+
kind = "clf" if has_proba else "reg"
|
|
380
|
+
classes = None
|
|
381
|
+
if kind == "clf":
|
|
382
|
+
classes = [_scalar(c) for c in getattr(estimator, "classes_", [])]
|
|
383
|
+
if not classes:
|
|
384
|
+
raise CalibrationError(
|
|
385
|
+
"classifier must be fitted and expose classes_ before "
|
|
386
|
+
"calibration")
|
|
387
|
+
if method != "split" or scale is not None or lower is not None \
|
|
388
|
+
or upper is not None:
|
|
389
|
+
raise CalibrationError(
|
|
390
|
+
f"method={method!r} is a regressor width; classifiers only "
|
|
391
|
+
f"support method='split' (conformal prediction sets)")
|
|
392
|
+
if one_sided:
|
|
393
|
+
raise CalibrationError(
|
|
394
|
+
"one-sided bounds are a regressor width; classifiers use "
|
|
395
|
+
"conformal prediction sets")
|
|
396
|
+
|
|
397
|
+
aux: dict = {}
|
|
398
|
+
if method == "normalized":
|
|
399
|
+
if scale is None:
|
|
400
|
+
raise CalibrationError(
|
|
401
|
+
"method='normalized' needs a fitted `scale` estimator")
|
|
402
|
+
if lower is not None or upper is not None:
|
|
403
|
+
raise CalibrationError(
|
|
404
|
+
"method='normalized' takes `scale`, not quantile estimators")
|
|
405
|
+
if not callable(getattr(scale, "predict", None)):
|
|
406
|
+
raise CalibrationError("scale estimator must expose predict()")
|
|
407
|
+
aux["scale"] = scale
|
|
408
|
+
elif method == "cqr":
|
|
409
|
+
if lower is None or upper is None:
|
|
410
|
+
raise CalibrationError(
|
|
411
|
+
"method='cqr' needs fitted `lower` and `upper` quantile "
|
|
412
|
+
"estimators")
|
|
413
|
+
if scale is not None:
|
|
414
|
+
raise CalibrationError(
|
|
415
|
+
"method='cqr' takes `lower`/`upper`, not a scale estimator")
|
|
416
|
+
for name, est in (("lower", lower), ("upper", upper)):
|
|
417
|
+
if not callable(getattr(est, "predict", None)):
|
|
418
|
+
raise CalibrationError(
|
|
419
|
+
f"{name} quantile estimator must expose predict()")
|
|
420
|
+
aux["lower"], aux["upper"] = lower, upper
|
|
421
|
+
elif scale is not None or lower is not None or upper is not None:
|
|
422
|
+
raise CalibrationError(
|
|
423
|
+
"method='split' takes no auxiliary estimators; declare "
|
|
424
|
+
"method='normalized' or method='cqr'")
|
|
425
|
+
aux_digest = _digest({k: _estimator_digest(v)
|
|
426
|
+
for k, v in sorted(aux.items())}) if aux else None
|
|
427
|
+
|
|
428
|
+
buckets: dict = {}
|
|
429
|
+
feats = tuple(features) if features else tuple(rows[0].keys())
|
|
430
|
+
X_all = [[r[f] for f in feats] for r in rows]
|
|
431
|
+
try:
|
|
432
|
+
if kind == "clf":
|
|
433
|
+
proba = estimator.predict_proba(X_all)
|
|
434
|
+
else:
|
|
435
|
+
pred = estimator.predict(X_all)
|
|
436
|
+
except Exception as ex: # noqa: BLE001
|
|
437
|
+
raise CalibrationError(
|
|
438
|
+
f"estimator failed during calibration: {type(ex).__name__}: {ex}")
|
|
439
|
+
|
|
440
|
+
for i, row in enumerate(rows):
|
|
441
|
+
key = str(row[group]) if group else "__all__"
|
|
442
|
+
if kind == "clf":
|
|
443
|
+
label = _scalar(y[i])
|
|
444
|
+
if label not in classes:
|
|
445
|
+
raise CalibrationError(
|
|
446
|
+
f"row {i}: label {label!r} not in estimator classes_ "
|
|
447
|
+
f"{classes}")
|
|
448
|
+
p = {c: float(_scalar(proba[i][j])) for j, c in enumerate(classes)}
|
|
449
|
+
s = 1.0 - p[label]
|
|
450
|
+
if not math.isfinite(s):
|
|
451
|
+
raise CalibrationError(
|
|
452
|
+
f"row {i}: non-finite nonconformity score")
|
|
453
|
+
buckets.setdefault(key, []).append(s)
|
|
454
|
+
elif one_sided:
|
|
455
|
+
try:
|
|
456
|
+
lo_s, hi_s = _reg_scores(method, aux, X_all[i],
|
|
457
|
+
_scalar(pred[i]), y[i])
|
|
458
|
+
except CalibrationError as ex:
|
|
459
|
+
raise CalibrationError(f"row {i}: {ex}")
|
|
460
|
+
if not (math.isfinite(lo_s) and math.isfinite(hi_s)):
|
|
461
|
+
raise CalibrationError(
|
|
462
|
+
f"row {i}: non-finite nonconformity score")
|
|
463
|
+
bucket = buckets.setdefault(key, {"lo": [], "hi": []})
|
|
464
|
+
bucket["lo"].append(lo_s)
|
|
465
|
+
bucket["hi"].append(hi_s)
|
|
466
|
+
else:
|
|
467
|
+
try:
|
|
468
|
+
s = _reg_score(method, aux, X_all[i],
|
|
469
|
+
_scalar(pred[i]), y[i])
|
|
470
|
+
except CalibrationError as ex:
|
|
471
|
+
raise CalibrationError(f"row {i}: {ex}")
|
|
472
|
+
if not math.isfinite(s):
|
|
473
|
+
raise CalibrationError(
|
|
474
|
+
f"row {i}: non-finite nonconformity score")
|
|
475
|
+
buckets.setdefault(key, []).append(s)
|
|
476
|
+
|
|
477
|
+
per_group = {}
|
|
478
|
+
if one_sided:
|
|
479
|
+
floor_lo = math.ceil(1.0 / a_lo) - 1
|
|
480
|
+
floor_hi = math.ceil(1.0 / a_hi) - 1
|
|
481
|
+
for key, bucket in buckets.items():
|
|
482
|
+
lo_scores = sorted(bucket["lo"])
|
|
483
|
+
hi_scores = sorted(bucket["hi"])
|
|
484
|
+
n = len(lo_scores)
|
|
485
|
+
floor = max(floor_lo, floor_hi)
|
|
486
|
+
if n < floor:
|
|
487
|
+
raise CalibrationError(
|
|
488
|
+
f"group {key!r}: n={n} is below the conformal floor "
|
|
489
|
+
f"{floor} for alpha_lower={a_lo}, alpha_upper={a_hi}; "
|
|
490
|
+
f"regroup or collect more calibration data")
|
|
491
|
+
per_group[key] = {
|
|
492
|
+
"q_lo": _quantile(lo_scores, n, a_lo),
|
|
493
|
+
"q_hi": _quantile(hi_scores, n, a_hi),
|
|
494
|
+
"n": n, "scores_lo": lo_scores, "scores_hi": hi_scores}
|
|
495
|
+
else:
|
|
496
|
+
for key, scores in buckets.items():
|
|
497
|
+
scores.sort()
|
|
498
|
+
n = len(scores)
|
|
499
|
+
if n < math.ceil(1.0 / alpha) - 1:
|
|
500
|
+
raise CalibrationError(
|
|
501
|
+
f"group {key!r}: n={n} is below the conformal floor "
|
|
502
|
+
f"ceil(1/alpha)-1={math.ceil(1.0 / alpha) - 1} for "
|
|
503
|
+
f"alpha={alpha}; regroup or collect more calibration data")
|
|
504
|
+
per_group[key] = {"q": _quantile(scores, n, alpha), "n": n,
|
|
505
|
+
"scores": scores}
|
|
506
|
+
|
|
507
|
+
cls = AdaptiveKappa if adaptive else Kappa
|
|
508
|
+
kwargs = {"gamma": gamma, "window": window} if adaptive else {}
|
|
509
|
+
return cls(kind=kind, alpha=alpha, group=group, classes=classes,
|
|
510
|
+
per_group=per_group, n_total=len(rows), method=method,
|
|
511
|
+
aux=aux, aux_digest=aux_digest, one_sided=one_sided,
|
|
512
|
+
alpha_lower=a_lo, alpha_upper=a_hi, **kwargs)
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
__all__ = ["Kappa", "AdaptiveKappa", "fit_kappa", "CalibrationError",
|
|
516
|
+
"REGRESSOR_METHODS"]
|