physmap 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- physmap/__init__.py +61 -0
- physmap/_paths.py +69 -0
- physmap/applicability/__init__.py +0 -0
- physmap/applicability/fixtures.py +83 -0
- physmap/applicability/screen.py +99 -0
- physmap/baselines/__init__.py +0 -0
- physmap/benchmarks/__init__.py +0 -0
- physmap/benchmarks/benchmark_report.py +405 -0
- physmap/benchmarks/benchmark_v0_4.py +424 -0
- physmap/benchmarks/compare.py +149 -0
- physmap/benchmarks/registry.py +217 -0
- physmap/benchmarks/report.py +224 -0
- physmap/cli.py +301 -0
- physmap/closures/__init__.py +48 -0
- physmap/closures/data/__init__.py +7 -0
- physmap/closures/data/closure_index.json +2997 -0
- physmap/closures/formulas.py +213 -0
- physmap/closures/geometry_classes.py +109 -0
- physmap/closures/index.py +393 -0
- physmap/closures/registry.py +313 -0
- physmap/compat/__init__.py +0 -0
- physmap/core/__init__.py +0 -0
- physmap/core/mechanism.py +69 -0
- physmap/core/signals.py +50 -0
- physmap/corpus/__init__.py +12 -0
- physmap/corpus/calibration.py +543 -0
- physmap/corpus/data/__init__.py +12 -0
- physmap/corpus/data/corpus_seed.jsonl +15 -0
- physmap/corpus/data/evidence_claims_seed.jsonl +21 -0
- physmap/corpus/data/evidence_sources_seed.jsonl +8 -0
- physmap/corpus/data/premium_coverage.json +60 -0
- physmap/corpus/evidence.py +871 -0
- physmap/explain/__init__.py +0 -0
- physmap/explain/benchmark.py +101 -0
- physmap/explain/causal.py +82 -0
- physmap/guardrail/__init__.py +38 -0
- physmap/guardrail/aggregator_observability.py +187 -0
- physmap/guardrail/classify.py +147 -0
- physmap/guardrail/configs.py +120 -0
- physmap/guardrail/corpus_regimes.py +208 -0
- physmap/guardrail/detector_conformal.py +129 -0
- physmap/guardrail/detector_density.py +74 -0
- physmap/guardrail/enums.py +69 -0
- physmap/guardrail/graph.py +73 -0
- physmap/guardrail/guardrail.py +606 -0
- physmap/guardrail/io.py +201 -0
- physmap/guardrail/regime_observability.py +519 -0
- physmap/guardrail/render.py +159 -0
- physmap/guardrail/weighting_heuristic.py +216 -0
- physmap/infra/__init__.py +23 -0
- physmap/infra/blindspot_oracle.py +356 -0
- physmap/infra/corpus_runtime.py +275 -0
- physmap/integrations/__init__.py +0 -0
- physmap/materiality/__init__.py +0 -0
- physmap/materiality/estimator.py +239 -0
- physmap/materiality/independence.py +92 -0
- physmap/materiality/surrogate_fit.py +293 -0
- physmap/observability/__init__.py +0 -0
- physmap/pipeline/__init__.py +58 -0
- physmap/pipeline/aggregators.py +199 -0
- physmap/pipeline/assessment_v06.py +509 -0
- physmap/pipeline/core.py +442 -0
- physmap/pipeline/defeasible_aggregator.py +324 -0
- physmap/pipeline/detectors.py +309 -0
- physmap/pipeline/observability.py +430 -0
- physmap/pipeline/surrogate.py +251 -0
- physmap/pipeline/validity_signal.py +273 -0
- physmap/pipeline/vehicle_spec.py +287 -0
- physmap/release.py +81 -0
- physmap/stress_tests/__init__.py +9 -0
- physmap/stress_tests/lewis_reuse.py +517 -0
- physmap/substrate/__init__.py +28 -0
- physmap/substrate/corpus_real.py +206 -0
- physmap/substrate/engine.py +209 -0
- physmap/substrate/forrest.py +249 -0
- physmap/substrate/loaders.py +2176 -0
- physmap/substrate/naca_tn1451.py +379 -0
- physmap/substrate/naca_wpd_loader.py +187 -0
- physmap/substrate/stage1_ingest.py +187 -0
- physmap/substrate/vehicle_config.py +407 -0
- physmap-0.2.0.dist-info/METADATA +270 -0
- physmap-0.2.0.dist-info/RECORD +88 -0
- physmap-0.2.0.dist-info/WHEEL +5 -0
- physmap-0.2.0.dist-info/entry_points.txt +2 -0
- physmap-0.2.0.dist-info/licenses/LICENSE +21 -0
- physmap-0.2.0.dist-info/licenses/LICENSE-CORPUS +469 -0
- physmap-0.2.0.dist-info/licenses/NOTICE +77 -0
- physmap-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
"""PhysMAP Corpus-Query Runtime Test — four-arm verdict comparison on held-out test points.
|
|
2
|
+
|
|
3
|
+
Adds a fourth verdict — a corpus-query consult of accumulated closure-vs-truth
|
|
4
|
+
observations near a prediction's physics coordinates — to Step 0's three-arm comparison
|
|
5
|
+
(guardrails, naive, causal), plus a PDE-residual baseline stand-in. Scored on test points
|
|
6
|
+
held out from the corpus build, with a sparse-vs-dense split as the honesty separator.
|
|
7
|
+
|
|
8
|
+
G-not-just-interpolation is the HEADLINE gate: the sparse region is the deployment surface
|
|
9
|
+
for any rare-but-catastrophic framing of the corpus mechanism, so the lift must hold there,
|
|
10
|
+
not just in dense regions where the mechanism approaches near-duplicate interpolation.
|
|
11
|
+
|
|
12
|
+
Run: python -m physmap.infra.corpus_runtime
|
|
13
|
+
Torch-free: numpy + scikit-learn + matplotlib. Step 0 artifacts / physics_causal.py /
|
|
14
|
+
gate_ledger.json untouched (purely additive).
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import csv
|
|
20
|
+
import inspect
|
|
21
|
+
import json
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
import numpy as np
|
|
26
|
+
from sklearn.neighbors import NearestNeighbors
|
|
27
|
+
|
|
28
|
+
from physmap.infra.blindspot_oracle import (
|
|
29
|
+
EMAX,
|
|
30
|
+
SMAX,
|
|
31
|
+
THETA_MAT,
|
|
32
|
+
TOL,
|
|
33
|
+
T_CAL_E,
|
|
34
|
+
TAU_CAL_S,
|
|
35
|
+
Dataset,
|
|
36
|
+
build_dataset,
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
# The experiment runner that used this is not ported; kept only so module-level
|
|
40
|
+
# references resolve.
|
|
41
|
+
OUT_DIR = Path(__file__).parent.parent / "results" / "physmap_corpus_runtime"
|
|
42
|
+
SCALE = SMAX + EMAX
|
|
43
|
+
BOUND = SMAX + EMAX # saturating ceiling — the only universal physics constraint here
|
|
44
|
+
|
|
45
|
+
# defaults (all documented in `_scope` and `params`)
|
|
46
|
+
K_DEFAULT = 5
|
|
47
|
+
THETA_ERR = 0.05 # same normalized scale as TOL — symmetric, not tuned
|
|
48
|
+
TRAIN_FRAC = 0.5
|
|
49
|
+
SPARSE_PCT = 50 # median split of test points by k-th NN corpus distance
|
|
50
|
+
SPLIT_SEED = 0
|
|
51
|
+
|
|
52
|
+
# robustness sweep bands
|
|
53
|
+
K_GRID = [3, 5, 8]
|
|
54
|
+
THETA_ERR_GRID = [0.03, 0.05, 0.08]
|
|
55
|
+
TOL_GRID = [0.03, 0.05, 0.08]
|
|
56
|
+
SPLIT_SEED_GRID = [0, 1]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# ── physics coordinates (transfer-premise stub) ───────────────────────────────
|
|
60
|
+
|
|
61
|
+
def physics_coords(X: np.ndarray) -> np.ndarray:
|
|
62
|
+
"""Degenerate here (return X). The slot the cross-geometry version fills:
|
|
63
|
+
map each point to a coordinate the corpus is indexed by."""
|
|
64
|
+
return np.asarray(X, float)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
# ── corpus build (train-point truth) ──────────────────────────────────────────
|
|
68
|
+
|
|
69
|
+
@dataclass
|
|
70
|
+
class Entry:
|
|
71
|
+
"""One closure-vs-truth observation at a training-point coordinate."""
|
|
72
|
+
coords: np.ndarray # (2,) [tau, t]
|
|
73
|
+
closure: float # HI_clo(p)
|
|
74
|
+
truth: float # HI_true(p) — independent truth at THIS point only
|
|
75
|
+
error: float # HI_true(p) − HI_clo(p)
|
|
76
|
+
qoi_contrib_s: float # closure-derived shear materiality fraction
|
|
77
|
+
qoi_contrib_e: float # closure-derived exposure materiality fraction
|
|
78
|
+
truth_source_weight: float = 1.0 # trivial here (algebraic = equal); slot for real-data
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def build_corpus(ds: Dataset, train_idx: np.ndarray) -> list[Entry]:
|
|
82
|
+
total = ds.c_s + ds.c_e
|
|
83
|
+
ms = ds.c_s / total
|
|
84
|
+
me = ds.c_e / total
|
|
85
|
+
return [
|
|
86
|
+
Entry(
|
|
87
|
+
coords=ds.X[i].copy(),
|
|
88
|
+
closure=float(ds.hi_clo[i]),
|
|
89
|
+
truth=float(ds.hi_true[i]),
|
|
90
|
+
error=float(ds.hi_true[i] - ds.hi_clo[i]),
|
|
91
|
+
qoi_contrib_s=float(ms[i]),
|
|
92
|
+
qoi_contrib_e=float(me[i]),
|
|
93
|
+
)
|
|
94
|
+
for i in train_idx
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
# ── corpus-query verdict (leak-safe; signature has no truth at test point) ────
|
|
99
|
+
|
|
100
|
+
def _dedup_corpus_against_test_X(test_X: np.ndarray,
|
|
101
|
+
corpus: list[Entry]) -> list[Entry]:
|
|
102
|
+
"""Return a copy of `corpus` with any entry whose coords exactly match a
|
|
103
|
+
coord in `test_X` removed.
|
|
104
|
+
|
|
105
|
+
This is the substrate-independent leak guard for k-NN error lookups: if a
|
|
106
|
+
test point and a corpus entry share the same coordinate (e.g. because the
|
|
107
|
+
underlying grid has duplicate coords across the train/test split, as
|
|
108
|
+
`build_dataset`'s query_grid does at 3 points on the synthetic substrate),
|
|
109
|
+
the k-NN at distance 0 would read that entry's truth value — effectively
|
|
110
|
+
the test point's own truth, since the truth function is deterministic in
|
|
111
|
+
coords. Excluding such entries before the lookup prevents the leak
|
|
112
|
+
regardless of substrate.
|
|
113
|
+
|
|
114
|
+
On the grounded Lance & Smith vehicle (Track D), continuous measurements
|
|
115
|
+
at distinct (X, time) points make this filter a no-op; on the synthetic
|
|
116
|
+
substrate it drops a small number of cross-fold duplicates. The invariant
|
|
117
|
+
is: no distance-0 cross-fold neighbor in the queried corpus.
|
|
118
|
+
"""
|
|
119
|
+
if len(corpus) == 0:
|
|
120
|
+
return corpus
|
|
121
|
+
test_coord_set = {tuple(x.tolist()) for x in np.asarray(test_X)}
|
|
122
|
+
return [e for e in corpus
|
|
123
|
+
if tuple(e.coords.tolist()) not in test_coord_set]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _query_weighted_err(test_X: np.ndarray, corpus: list[Entry],
|
|
127
|
+
k: int) -> np.ndarray:
|
|
128
|
+
"""k-NN weighted-mean |error| at each row of `test_X`, computed against a
|
|
129
|
+
leak-guarded corpus (cross-fold-coord duplicates filtered out via
|
|
130
|
+
`_dedup_corpus_against_test_X`).
|
|
131
|
+
|
|
132
|
+
Returns a zero vector when the corpus is empty after dedup (degenerate
|
|
133
|
+
safe-default). Caller is responsible for any downstream thresholding.
|
|
134
|
+
"""
|
|
135
|
+
safe_corpus = _dedup_corpus_against_test_X(test_X, corpus)
|
|
136
|
+
if len(safe_corpus) == 0:
|
|
137
|
+
return np.zeros(len(test_X), dtype=float)
|
|
138
|
+
coords = np.stack([e.coords for e in safe_corpus], axis=0)
|
|
139
|
+
weights = np.array([e.truth_source_weight for e in safe_corpus], float)
|
|
140
|
+
errors = np.abs(np.array([e.error for e in safe_corpus], float))
|
|
141
|
+
k_eff = min(k, len(safe_corpus))
|
|
142
|
+
nbrs = NearestNeighbors(n_neighbors=k_eff).fit(physics_coords(coords))
|
|
143
|
+
_, idx = nbrs.kneighbors(physics_coords(test_X))
|
|
144
|
+
w = weights[idx]
|
|
145
|
+
e = errors[idx]
|
|
146
|
+
return (w * e).sum(axis=1) / w.sum(axis=1)
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def corpus_verdict(test_X: np.ndarray, test_c_s: np.ndarray, test_c_e: np.ndarray,
|
|
150
|
+
corpus: list[Entry], k: int = K_DEFAULT, theta_err: float = THETA_ERR,
|
|
151
|
+
theta_mat: float = THETA_MAT) -> np.ndarray:
|
|
152
|
+
"""Untrustworthy iff k nearest corpus entries (by physics_coords) show weighted-mean
|
|
153
|
+
|error| ≥ theta_err AND closure-derived materiality at the test point clears theta_mat.
|
|
154
|
+
|
|
155
|
+
Signature deliberately excludes any test-point truth array (G-no-leak). Also
|
|
156
|
+
drops any corpus entry whose coords exactly match a test_X coord before the
|
|
157
|
+
k-NN lookup (substrate-independent leak guard — no distance-0 cross-fold
|
|
158
|
+
neighbor in the queried corpus, regardless of whether the substrate has
|
|
159
|
+
duplicate coords)."""
|
|
160
|
+
if len(corpus) == 0:
|
|
161
|
+
return np.zeros(len(test_X), dtype=bool)
|
|
162
|
+
weighted_err = _query_weighted_err(test_X, corpus, k=k)
|
|
163
|
+
|
|
164
|
+
total = np.asarray(test_c_s, float) + np.asarray(test_c_e, float)
|
|
165
|
+
ms_test = np.asarray(test_c_s, float) / total
|
|
166
|
+
me_test = np.asarray(test_c_e, float) / total
|
|
167
|
+
material = (ms_test >= theta_mat) | (me_test >= theta_mat)
|
|
168
|
+
|
|
169
|
+
return (weighted_err >= theta_err) & material
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def corpus_error_magnitude(test_X: np.ndarray, corpus: list[Entry],
|
|
173
|
+
k: int = K_DEFAULT) -> np.ndarray:
|
|
174
|
+
"""Per-test-row weighted-mean |error| over the k nearest corpus entries by
|
|
175
|
+
physics_coords. The continuous scalar `corpus_verdict` thresholds internally,
|
|
176
|
+
exposed here for use as a magnitude in the ranking formula (PR-3 attach point).
|
|
177
|
+
|
|
178
|
+
Signature deliberately excludes any test-point truth array (G-no-leak). Also
|
|
179
|
+
drops any corpus entry whose coords exactly match a test_X coord before the
|
|
180
|
+
k-NN lookup (substrate-independent leak guard). The materiality gate that
|
|
181
|
+
`corpus_verdict` ANDs in is a separate concern; the ranking formula needs
|
|
182
|
+
magnitudes for ALL rows, so this function does NOT short-circuit on it.
|
|
183
|
+
"""
|
|
184
|
+
if len(corpus) == 0:
|
|
185
|
+
return np.zeros(len(test_X), dtype=float)
|
|
186
|
+
return _query_weighted_err(test_X, corpus, k=k)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
# ── PDE-residual baseline (bound-only stand-in for NVIDIA's PDE guardrail) ────
|
|
190
|
+
|
|
191
|
+
def pde_residual_flag(hi_sur: np.ndarray, bound: float = BOUND) -> np.ndarray:
|
|
192
|
+
"""Flag iff prediction violates the saturating-truth bound [0, bound].
|
|
193
|
+
A real PDE-residual check requires real fields; this is the cheapest stand-in in
|
|
194
|
+
this algebraic world (the overshooting closures route bound violations through the
|
|
195
|
+
surrogate at extrapolation, so this baseline has actual teeth — and is documented
|
|
196
|
+
in `_scope` as the conceptual limit)."""
|
|
197
|
+
a = np.asarray(hi_sur, float)
|
|
198
|
+
return (a > bound) | (a < 0)
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
# ── train/test split + sparse/dense split + coverage stats ────────────────────
|
|
202
|
+
|
|
203
|
+
def _bootstrap_ci(diff: np.ndarray, b: int = 2000, seed: int = 0) -> tuple:
|
|
204
|
+
"""Verbatim from the monorepo's blindspot_pilot, moved here rather than imported.
|
|
205
|
+
|
|
206
|
+
That import was the last edge reaching blindspot_causal and therefore rdflib. The
|
|
207
|
+
body is copied unchanged -- same default b, same seed, same percentiles -- so
|
|
208
|
+
every bootstrap interval this repository produces is bit-identical to the banked
|
|
209
|
+
ones. Do not "clean it up".
|
|
210
|
+
"""
|
|
211
|
+
if len(diff) == 0:
|
|
212
|
+
return (None, None)
|
|
213
|
+
rng = np.random.default_rng(seed)
|
|
214
|
+
boots = [float(rng.choice(diff, len(diff), replace=True).mean()) for _ in range(b)]
|
|
215
|
+
return (round(float(np.percentile(boots, 2.5)), 3), round(float(np.percentile(boots, 97.5)), 3))
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def split_train_test(n: int, train_frac: float = TRAIN_FRAC,
|
|
219
|
+
seed: int = SPLIT_SEED) -> tuple[np.ndarray, np.ndarray]:
|
|
220
|
+
"""Deterministic shuffle-and-cut split of n indices."""
|
|
221
|
+
rng = np.random.default_rng(seed)
|
|
222
|
+
idx = np.arange(n)
|
|
223
|
+
rng.shuffle(idx)
|
|
224
|
+
n_train = int(round(train_frac * n))
|
|
225
|
+
return np.sort(idx[:n_train]), np.sort(idx[n_train:])
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def sparse_dense_split(test_X: np.ndarray, corpus: list[Entry], k: int = K_DEFAULT,
|
|
229
|
+
pct: float = SPARSE_PCT):
|
|
230
|
+
"""Per test point: distance to k-th nearest corpus entry. Above pct percentile = sparse.
|
|
231
|
+
Returns (sparse_mask, dense_mask, threshold, kth_distances)."""
|
|
232
|
+
coords = np.stack([e.coords for e in corpus], axis=0)
|
|
233
|
+
k_eff = min(k, len(corpus))
|
|
234
|
+
nbrs = NearestNeighbors(n_neighbors=k_eff).fit(physics_coords(coords))
|
|
235
|
+
dists, _ = nbrs.kneighbors(physics_coords(test_X))
|
|
236
|
+
kth = dists[:, -1]
|
|
237
|
+
threshold = float(np.percentile(kth, pct))
|
|
238
|
+
sparse = kth > threshold
|
|
239
|
+
dense = ~sparse
|
|
240
|
+
return sparse, dense, threshold, kth
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _summary(arr: np.ndarray) -> dict:
|
|
244
|
+
if len(arr) == 0:
|
|
245
|
+
return {k: None for k in ("min", "p10", "p25", "median", "p75", "p90", "max", "mean")}
|
|
246
|
+
return {
|
|
247
|
+
"min": round(float(arr.min()), 4),
|
|
248
|
+
"p10": round(float(np.percentile(arr, 10)), 4),
|
|
249
|
+
"p25": round(float(np.percentile(arr, 25)), 4),
|
|
250
|
+
"median": round(float(np.median(arr)), 4),
|
|
251
|
+
"p75": round(float(np.percentile(arr, 75)), 4),
|
|
252
|
+
"p90": round(float(np.percentile(arr, 90)), 4),
|
|
253
|
+
"max": round(float(arr.max()), 4),
|
|
254
|
+
"mean": round(float(arr.mean()), 4),
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def coverage_stats(dists: np.ndarray, sparse_mask: np.ndarray, dense_mask: np.ndarray) -> dict:
|
|
259
|
+
"""k-th NN distance distribution for the reader: dense median near zero = near-duplicate
|
|
260
|
+
interpolation (a dense-only win would be definitional); dense median well above the grid
|
|
261
|
+
step = legitimate generalization signal."""
|
|
262
|
+
return {
|
|
263
|
+
"overall": _summary(dists),
|
|
264
|
+
"sparse": _summary(dists[sparse_mask]),
|
|
265
|
+
"dense": _summary(dists[dense_mask]),
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
# ---------------------------------------------------------------------------
|
|
269
|
+
# The step-0 fixture experiment path -- compute_runtime, metrics_four_arm,
|
|
270
|
+
# gates, robustness_sweep, plotting and the __main__ runner -- is NOT ported.
|
|
271
|
+
# It reached blindspot_pilot, which reaches blindspot_causal, which imports
|
|
272
|
+
# rdflib, and none of it is used by the benchmark. What remains above is the
|
|
273
|
+
# corpus primitive set the substrate actually calls: Entry, build_corpus,
|
|
274
|
+
# corpus_verdict, corpus_error_magnitude, pde_residual_flag, split_train_test.
|
|
275
|
+
# ---------------------------------------------------------------------------
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""Causal materiality by ablation counterfactual.
|
|
2
|
+
|
|
3
|
+
materiality(m, x, q) = 1 - q_ablated / q_full
|
|
4
|
+
|
|
5
|
+
How much of the quantity of interest `q` at operating point `x` goes away when mechanism
|
|
6
|
+
`m` is removed. Both numbers come from outside this module, and both are persisted on the
|
|
7
|
+
result, because a materiality whose inputs you cannot see is a number you cannot check.
|
|
8
|
+
|
|
9
|
+
WHY THIS IS A REWRITE AND NOT A PORT
|
|
10
|
+
------------------------------------
|
|
11
|
+
The monorepo computes a contribution FRACTION: each mechanism's declared contribution
|
|
12
|
+
divided by the sum over all mechanisms. For exactly two mechanisms whose contributions
|
|
13
|
+
are (q_ablated, q_full - q_ablated), that fraction happens to equal 1 - q_ablated/q_full.
|
|
14
|
+
The arithmetic agrees; the meaning does not.
|
|
15
|
+
|
|
16
|
+
A fraction of declared contributions is a bookkeeping identity -- it sums to one by
|
|
17
|
+
construction, no matter what the mechanisms are or whether the decomposition is real. A
|
|
18
|
+
counterfactual is a claim about what the world does when you remove something, and it can
|
|
19
|
+
be wrong, checked, and refused. Only the second is evidence.
|
|
20
|
+
|
|
21
|
+
The practical consequence is the one that matters. Under the fraction, a mechanism with
|
|
22
|
+
no recorded contribution gets 0.0 and is silently judged immaterial. Under the
|
|
23
|
+
counterfactual, a missing input yields INSUFFICIENT_EVIDENCE and no verdict at all. An
|
|
24
|
+
inferred zero is the failure mode this whole project is about: an absence of evidence
|
|
25
|
+
rendered as evidence of absence, in a number that looks just like a real one.
|
|
26
|
+
|
|
27
|
+
REFUSED PROVENANCE
|
|
28
|
+
------------------
|
|
29
|
+
An ablation is only a counterfactual if the ablated run is otherwise the same run. A
|
|
30
|
+
flat-plate correlation evaluated for a pipe is not an ablation of anything; it is a
|
|
31
|
+
different formula for a different geometry. It is refused by name rather than merely
|
|
32
|
+
discouraged, because it is cheap, available, and gives plausible-looking numbers.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
from dataclasses import dataclass, replace
|
|
38
|
+
from enum import Enum
|
|
39
|
+
|
|
40
|
+
from physmap.core.mechanism import Mechanism
|
|
41
|
+
from physmap.core.signals import Signal, SignalKind
|
|
42
|
+
from physmap.release import EvidenceState
|
|
43
|
+
|
|
44
|
+
__all__ = [
|
|
45
|
+
"AblationProvenance",
|
|
46
|
+
"REFUSED_PROVENANCE",
|
|
47
|
+
"MaterialityStatus",
|
|
48
|
+
"AblationInputs",
|
|
49
|
+
"MaterialityResult",
|
|
50
|
+
"estimate_materiality",
|
|
51
|
+
"materiality_signal",
|
|
52
|
+
]
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class AblationProvenance(str, Enum):
|
|
56
|
+
"""Where the ablated value came from. Not a label -- a gate."""
|
|
57
|
+
|
|
58
|
+
#: Same mesh, same boundary conditions, mechanism switched off. The real thing.
|
|
59
|
+
MATCHED_ABLATION = "matched_ablation"
|
|
60
|
+
#: A correlation for THIS geometry with the mechanism absent. Weaker, admissible.
|
|
61
|
+
GEOMETRY_MATCHED_CORRELATION = "geometry_matched_correlation"
|
|
62
|
+
#: A correlation for a DIFFERENT geometry. Refused; see the module docstring.
|
|
63
|
+
FLAT_PLATE_CORRELATION = "flat_plate_correlation"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
REFUSED_PROVENANCE: frozenset[AblationProvenance] = frozenset(
|
|
67
|
+
{AblationProvenance.FLAT_PLATE_CORRELATION}
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
class MaterialityStatus(str, Enum):
|
|
72
|
+
ESTIMATED = "estimated"
|
|
73
|
+
INSUFFICIENT_EVIDENCE = "insufficient_evidence"
|
|
74
|
+
REFUSED_PROVENANCE = "refused_provenance"
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class AblationInputs:
|
|
79
|
+
"""The two numbers the estimate is made of, kept so a reader can redo the division."""
|
|
80
|
+
|
|
81
|
+
qoi_full: float | None = None
|
|
82
|
+
qoi_ablated: float | None = None
|
|
83
|
+
provenance: AblationProvenance | None = None
|
|
84
|
+
|
|
85
|
+
def complete(self) -> bool:
|
|
86
|
+
return (
|
|
87
|
+
self.qoi_full is not None
|
|
88
|
+
and self.qoi_ablated is not None
|
|
89
|
+
and self.provenance is not None
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
def missing(self) -> list[str]:
|
|
93
|
+
return [
|
|
94
|
+
name
|
|
95
|
+
for name, v in (
|
|
96
|
+
("qoi_full", self.qoi_full),
|
|
97
|
+
("qoi_ablated", self.qoi_ablated),
|
|
98
|
+
("provenance", self.provenance),
|
|
99
|
+
)
|
|
100
|
+
if v is None
|
|
101
|
+
]
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@dataclass(frozen=True)
|
|
105
|
+
class MaterialityResult:
|
|
106
|
+
"""An estimate, or a stated reason there is none.
|
|
107
|
+
|
|
108
|
+
`value` is None unless `status` is ESTIMATED. There is no zero-valued failure: a
|
|
109
|
+
materiality of 0.0 means the ablation genuinely changed nothing.
|
|
110
|
+
"""
|
|
111
|
+
|
|
112
|
+
mechanism_id: str
|
|
113
|
+
qoi: str
|
|
114
|
+
status: MaterialityStatus
|
|
115
|
+
inputs: AblationInputs
|
|
116
|
+
evidence_state: EvidenceState
|
|
117
|
+
reason: str
|
|
118
|
+
value: float | None = None
|
|
119
|
+
|
|
120
|
+
def __post_init__(self) -> None:
|
|
121
|
+
if self.status is MaterialityStatus.ESTIMATED and self.value is None:
|
|
122
|
+
raise ValueError("an ESTIMATED materiality must carry a value")
|
|
123
|
+
if self.status is not MaterialityStatus.ESTIMATED and self.value is not None:
|
|
124
|
+
raise ValueError(
|
|
125
|
+
f"materiality status {self.status.value} must not carry a value; "
|
|
126
|
+
f"got {self.value!r}. A failed estimate has no number, not a zero."
|
|
127
|
+
)
|
|
128
|
+
|
|
129
|
+
def is_usable(self) -> bool:
|
|
130
|
+
return self.status is MaterialityStatus.ESTIMATED
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def estimate_materiality(
|
|
134
|
+
mechanism: Mechanism,
|
|
135
|
+
qoi: str,
|
|
136
|
+
inputs: AblationInputs,
|
|
137
|
+
*,
|
|
138
|
+
evidence_state: EvidenceState,
|
|
139
|
+
) -> MaterialityResult:
|
|
140
|
+
"""Compute 1 - q_ablated/q_full, or say why not."""
|
|
141
|
+
|
|
142
|
+
def _fail(status: MaterialityStatus, reason: str) -> MaterialityResult:
|
|
143
|
+
return MaterialityResult(
|
|
144
|
+
mechanism_id=mechanism.mechanism_id, qoi=qoi, status=status,
|
|
145
|
+
inputs=inputs, evidence_state=evidence_state, reason=reason, value=None,
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
if inputs.provenance in REFUSED_PROVENANCE:
|
|
149
|
+
return _fail(
|
|
150
|
+
MaterialityStatus.REFUSED_PROVENANCE,
|
|
151
|
+
f"ablation provenance {inputs.provenance.value!r} is refused: it is a "
|
|
152
|
+
f"correlation for a different geometry, not an ablation of this case.",
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
if not inputs.complete():
|
|
156
|
+
return _fail(
|
|
157
|
+
MaterialityStatus.INSUFFICIENT_EVIDENCE,
|
|
158
|
+
f"missing ablation input(s): {', '.join(inputs.missing())}. No materiality "
|
|
159
|
+
f"is inferred; an absent input is not a zero.",
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
if inputs.qoi_full == 0.0:
|
|
163
|
+
return _fail(
|
|
164
|
+
MaterialityStatus.INSUFFICIENT_EVIDENCE,
|
|
165
|
+
"qoi_full is zero, so the materiality fraction is undefined.",
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
value = 1.0 - (inputs.qoi_ablated / inputs.qoi_full)
|
|
169
|
+
return MaterialityResult(
|
|
170
|
+
mechanism_id=mechanism.mechanism_id, qoi=qoi,
|
|
171
|
+
status=MaterialityStatus.ESTIMATED, inputs=inputs,
|
|
172
|
+
evidence_state=evidence_state,
|
|
173
|
+
reason=(
|
|
174
|
+
f"1 - {inputs.qoi_ablated:g}/{inputs.qoi_full:g} = {value:.4g} "
|
|
175
|
+
f"({inputs.provenance.value})"
|
|
176
|
+
),
|
|
177
|
+
value=value,
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def materiality_signal(
|
|
182
|
+
mechanism: Mechanism,
|
|
183
|
+
result: MaterialityResult,
|
|
184
|
+
*,
|
|
185
|
+
theta: float,
|
|
186
|
+
) -> Signal:
|
|
187
|
+
"""The flag rule: fire iff the mechanism is OUTSIDE its calibration window AND its
|
|
188
|
+
materiality reaches theta.
|
|
189
|
+
|
|
190
|
+
Both halves are required. Out-of-range alone is the naive box check, which fires on
|
|
191
|
+
every excursion however irrelevant. Material alone says nothing is wrong -- a
|
|
192
|
+
mechanism can dominate the QoI and be perfectly well calibrated.
|
|
193
|
+
|
|
194
|
+
An unusable materiality cannot fire. It also cannot clear: the honest output is a
|
|
195
|
+
quiet signal whose rationale states that the question was not answered.
|
|
196
|
+
"""
|
|
197
|
+
outside = mechanism.outside_calibration()
|
|
198
|
+
window = mechanism.window.describe()
|
|
199
|
+
|
|
200
|
+
if not result.is_usable():
|
|
201
|
+
return Signal(
|
|
202
|
+
kind=SignalKind.CAUSAL_MATERIALITY, fired=False, value=None, threshold=theta,
|
|
203
|
+
rationale=(
|
|
204
|
+
f"causal_materiality: NOT ASSESSED for {mechanism.name} "
|
|
205
|
+
f"({result.status.value}) -- {result.reason} "
|
|
206
|
+
f"The mechanism is {'outside' if outside else 'inside'} {window}, but "
|
|
207
|
+
f"without a materiality this is not a causal verdict either way."
|
|
208
|
+
),
|
|
209
|
+
)
|
|
210
|
+
|
|
211
|
+
material = result.value >= theta
|
|
212
|
+
if outside and material:
|
|
213
|
+
rationale = (
|
|
214
|
+
f"causal_materiality: FIRED. {mechanism.name} is outside {window} "
|
|
215
|
+
f"(operating value {mechanism.operating_value:g}) and carries materiality "
|
|
216
|
+
f"{result.value:.3g} >= theta={theta:g} for {result.qoi}. {result.reason}"
|
|
217
|
+
)
|
|
218
|
+
elif outside:
|
|
219
|
+
rationale = (
|
|
220
|
+
f"causal_materiality: quiet. {mechanism.name} is outside {window} but its "
|
|
221
|
+
f"materiality {result.value:.3g} < theta={theta:g} for {result.qoi}, so the "
|
|
222
|
+
f"excursion does not reach the quantity of interest. {result.reason}"
|
|
223
|
+
)
|
|
224
|
+
else:
|
|
225
|
+
rationale = (
|
|
226
|
+
f"causal_materiality: quiet. {mechanism.name} is within {window}; "
|
|
227
|
+
f"materiality {result.value:.3g} is not a defect on its own."
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
return Signal(
|
|
231
|
+
kind=SignalKind.CAUSAL_MATERIALITY,
|
|
232
|
+
fired=bool(outside and material),
|
|
233
|
+
value=result.value, threshold=theta, rationale=rationale,
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def with_inputs(result: MaterialityResult, **kwargs) -> MaterialityResult:
|
|
238
|
+
"""Return a copy with replaced ablation inputs. Used by fixtures and tests."""
|
|
239
|
+
return replace(result, inputs=replace(result.inputs, **kwargs))
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
"""The truth-independence guard. Refuses before any metric is computed.
|
|
2
|
+
|
|
3
|
+
The whole value of a performance claim here is that the truth label is independent of the
|
|
4
|
+
closure the causal method checks. If the "truth" a prediction is scored against was
|
|
5
|
+
produced by that same closure, then closure-versus-truth measures closure-versus-itself,
|
|
6
|
+
every disagreement is definitionally zero, and any reported lift is an artifact of the
|
|
7
|
+
setup rather than a finding about the world.
|
|
8
|
+
|
|
9
|
+
This guard raises. It does not warn, and it does not return a lower confidence, because a
|
|
10
|
+
circular comparison has no degraded-but-usable form -- there is nothing to salvage from
|
|
11
|
+
it. The refusal happens at load time, before precision, recall or F1 exist, so a circular
|
|
12
|
+
run cannot produce a number that someone later quotes.
|
|
13
|
+
|
|
14
|
+
The open question this guards is live, not hypothetical. The prior implementation marked
|
|
15
|
+
the mixed-convection pipe substrate as non-divergent on exactly these grounds: its truth
|
|
16
|
+
column was a closure-style correlation rather than a measurement. The abstract describes
|
|
17
|
+
that truth as experimental. Until that is resolved, the guard is what stands between the
|
|
18
|
+
two readings.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
from enum import Enum
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"TruthIndependence",
|
|
27
|
+
"INDEPENDENT_SOURCES",
|
|
28
|
+
"CircularTruthError",
|
|
29
|
+
"require_independent_truth",
|
|
30
|
+
"may_report_performance",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class TruthIndependence(str, Enum):
|
|
35
|
+
#: Measured. The only source the abstract's phrasing would justify.
|
|
36
|
+
EXPERIMENTAL = "experimental"
|
|
37
|
+
#: A high-fidelity model that does not share the closure under test.
|
|
38
|
+
INDEPENDENT_HIGH_FIDELITY = "independent_high_fidelity"
|
|
39
|
+
#: Declared by a fixture. Exercises the machinery; backs no performance claim.
|
|
40
|
+
FIXTURE = "fixture"
|
|
41
|
+
#: Produced by the closure under test. Circular. Always refused.
|
|
42
|
+
SAME_CLOSURE = "same_closure"
|
|
43
|
+
#: Provenance not established. Refused, because unknown is not innocent.
|
|
44
|
+
UNKNOWN = "unknown"
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
#: The only sources that may back a reported performance metric.
|
|
48
|
+
INDEPENDENT_SOURCES: frozenset[TruthIndependence] = frozenset(
|
|
49
|
+
{TruthIndependence.EXPERIMENTAL, TruthIndependence.INDEPENDENT_HIGH_FIDELITY}
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class CircularTruthError(ValueError):
|
|
54
|
+
"""Raised when truth is not independent of the closure being checked."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def require_independent_truth(source: TruthIndependence | str) -> TruthIndependence:
|
|
58
|
+
"""Raise unless `source` can back a performance claim. Returns it otherwise."""
|
|
59
|
+
try:
|
|
60
|
+
s = TruthIndependence(source)
|
|
61
|
+
except ValueError as e:
|
|
62
|
+
raise CircularTruthError(
|
|
63
|
+
f"unknown truth source {source!r}; allowed: "
|
|
64
|
+
f"{sorted(t.value for t in TruthIndependence)}"
|
|
65
|
+
) from e
|
|
66
|
+
|
|
67
|
+
if s is TruthIndependence.SAME_CLOSURE:
|
|
68
|
+
raise CircularTruthError(
|
|
69
|
+
"truth_source is 'same_closure': the truth was produced by the closure under "
|
|
70
|
+
"test, so any comparison measures the closure against itself. Refused before "
|
|
71
|
+
"any metric is computed."
|
|
72
|
+
)
|
|
73
|
+
if s is TruthIndependence.UNKNOWN:
|
|
74
|
+
raise CircularTruthError(
|
|
75
|
+
"truth_source is 'unknown': independence has not been established. Unknown "
|
|
76
|
+
"provenance is refused rather than assumed innocent."
|
|
77
|
+
)
|
|
78
|
+
if s is TruthIndependence.FIXTURE:
|
|
79
|
+
raise CircularTruthError(
|
|
80
|
+
"truth_source is 'fixture': a declared value exercises the machinery but "
|
|
81
|
+
"cannot back a performance claim. Use may_report_performance() to branch on "
|
|
82
|
+
"this instead of calling the guard."
|
|
83
|
+
)
|
|
84
|
+
return s
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def may_report_performance(source: TruthIndependence | str) -> bool:
|
|
88
|
+
"""Non-raising form, for deciding whether to compute metrics at all."""
|
|
89
|
+
try:
|
|
90
|
+
return TruthIndependence(source) in INDEPENDENT_SOURCES
|
|
91
|
+
except ValueError:
|
|
92
|
+
return False
|