physmap 0.2.1__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {physmap-0.2.1/src/physmap.egg-info → physmap-0.2.2}/PKG-INFO +157 -58
- {physmap-0.2.1 → physmap-0.2.2}/README.md +156 -57
- {physmap-0.2.1 → physmap-0.2.2}/pyproject.toml +1 -1
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/__init__.py +1 -1
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/benchmarks/report.py +31 -0
- {physmap-0.2.1 → physmap-0.2.2/src/physmap.egg-info}/PKG-INFO +157 -58
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_benchmark_boundaries.py +4 -1
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_talk_package.py +12 -0
- {physmap-0.2.1 → physmap-0.2.2}/LICENSE +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/LICENSE-CORPUS +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/NOTICE +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/setup.cfg +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/_paths.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/applicability/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/applicability/fixtures.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/applicability/screen.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/baselines/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/benchmarks/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/benchmarks/benchmark_report.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/benchmarks/benchmark_v0_4.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/benchmarks/compare.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/benchmarks/registry.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/cli.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/closures/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/closures/data/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/closures/data/closure_index.json +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/closures/formulas.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/closures/geometry_classes.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/closures/index.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/closures/registry.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/compat/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/core/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/core/mechanism.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/core/signals.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/corpus/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/corpus/calibration.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/corpus/data/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/corpus/data/corpus_seed.jsonl +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/corpus/data/evidence_claims_seed.jsonl +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/corpus/data/evidence_sources_seed.jsonl +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/corpus/data/premium_coverage.json +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/corpus/evidence.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/explain/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/explain/benchmark.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/explain/causal.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/aggregator_observability.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/classify.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/configs.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/corpus_regimes.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/detector_conformal.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/detector_density.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/enums.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/graph.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/guardrail.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/io.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/regime_observability.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/render.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/guardrail/weighting_heuristic.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/infra/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/infra/blindspot_oracle.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/infra/corpus_runtime.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/integrations/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/materiality/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/materiality/estimator.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/materiality/independence.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/materiality/surrogate_fit.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/observability/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/pipeline/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/pipeline/aggregators.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/pipeline/assessment_v06.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/pipeline/core.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/pipeline/defeasible_aggregator.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/pipeline/detectors.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/pipeline/observability.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/pipeline/surrogate.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/pipeline/validity_signal.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/pipeline/vehicle_spec.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/release.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/stress_tests/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/stress_tests/lewis_reuse.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/substrate/__init__.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/substrate/corpus_real.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/substrate/engine.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/substrate/forrest.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/substrate/loaders.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/substrate/naca_tn1451.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/substrate/naca_wpd_loader.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/substrate/stage1_ingest.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap/substrate/vehicle_config.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap.egg-info/SOURCES.txt +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap.egg-info/dependency_links.txt +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap.egg-info/entry_points.txt +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap.egg-info/requires.txt +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/src/physmap.egg-info/top_level.txt +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_applicability_screen.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_benchmark_reproduces.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_causal_materiality.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_cli_without_checkout.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_guardrail_public_api.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_no_conflation.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_no_heavy_imports.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_render_ood_wording.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_stress_test_lewis_reuse.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_surrogate_fit.py +0 -0
- {physmap-0.2.1 → physmap-0.2.2}/tests/test_version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: physmap
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Physics-aware credibility checks for AI surrogates in multiphysics simulation: closure validity, surrogate observability, and causal materiality.
|
|
5
5
|
Author: Vishnu Vettrivel
|
|
6
6
|
License-Expression: MIT
|
|
@@ -44,6 +44,9 @@ PhysMAP reads the bound variable from the test coordinates instead of from the
|
|
|
44
44
|
surrogate's inputs, checks it against the closure relation's validated range, and knows
|
|
45
45
|
at fit time whether that variable is structurally observable to the surrogate at all.
|
|
46
46
|
|
|
47
|
+
**That is the point of PhysMAP: it catches the failures an input-based OOD detector cannot
|
|
48
|
+
see, because what broke the surrogate is not one of its inputs.**
|
|
49
|
+
|
|
47
50
|
```
|
|
48
51
|
verdicts: {'REJECT': 45}
|
|
49
52
|
rationale: Prediction relies on gnielinski-1976 beyond its validated x_over_D bound
|
|
@@ -57,17 +60,108 @@ No LLM is in any path. Every explanation is a deterministic rendered template.
|
|
|
57
60
|
|
|
58
61
|
Run it yourself: [`examples/naca_entrance_region.py`](https://github.com/cloudronin/physmap/blob/main/examples/naca_entrance_region.py).
|
|
59
62
|
|
|
63
|
+
## What PhysMAP adds to OOD detection
|
|
64
|
+
|
|
65
|
+

|
|
66
|
+
|
|
67
|
+
An input-based OOD detector judges a prediction by where its inputs sit relative to the
|
|
68
|
+
training data. It asks that well — but it cannot see a change in something that is not an
|
|
69
|
+
input. PhysMAP adds the check it cannot make: it reads the variables the surrogate never saw,
|
|
70
|
+
tests the physics relation behind the surrogate against its validated range, and knows at
|
|
71
|
+
setup which of those variables the detectors can see. It keeps the OOD detectors, and
|
|
72
|
+
overrides them only where they are structurally blind.
|
|
73
|
+
|
|
74
|
+
The seven-vehicle benchmark measures exactly that, per dataset, at the default setting (the
|
|
75
|
+
99th percentile):
|
|
76
|
+
|
|
77
|
+
| Dataset | Can the OOD detectors see the cause? | Wrong predictions caught only by PhysMAP | Right predictions flagged anyway |
|
|
78
|
+
|---|---|---|---|
|
|
79
|
+
| `naca_tn1451` — heated pipe, entrance region | no | 20 of 20 | 24 |
|
|
80
|
+
| `casper_hypersonic_transition` — hypersonic transition | no | 4 of 8 | 0 |
|
|
81
|
+
| `jin_sco2_buoyancy` — supercritical CO2, vertical tube | no | 15 of 26 | 0 |
|
|
82
|
+
| `velazquez_sco2` — supercritical CO2, property variation | partly | 18 of 67 | 0 |
|
|
83
|
+
| `dirker_water` — water, horizontal tube | partly | 2 of 11 | 16 |
|
|
84
|
+
| `marineau_hypersonic_transition` — hypersonic transition | yes | 0 of 6 | 0 |
|
|
85
|
+
| `forrest` — rectangular channel | yes | not tested: one training row | — |
|
|
86
|
+
|
|
87
|
+
- **Where it helps:** when the cause of failure is hidden from the surrogate's inputs. There
|
|
88
|
+
PhysMAP catches wrong predictions the OOD detectors miss entirely — all 20 for the pipe
|
|
89
|
+
entrance.
|
|
90
|
+
- **Where it doesn't:** when the cause is an input, the OOD detectors already see it and
|
|
91
|
+
PhysMAP adds nothing. `marineau_hypersonic_transition` is the control that shows it.
|
|
92
|
+
- **The cost:** the closure check flags anything outside a relation's validated range, even
|
|
93
|
+
when the surrogate happens to be right — 24 false alarms for the pipe entrance, 16 for
|
|
94
|
+
`dirker_water`.
|
|
95
|
+
|
|
96
|
+
Counts are rows of each dataset, not independent cases, and are never pooled into a rate
|
|
97
|
+
across datasets; the source values are digitised from publications. The benchmark's
|
|
98
|
+
input-based detectors are distance-to-training and GP variance. `physmap benchmark report`
|
|
99
|
+
prints these counts, and `physmap benchmark run` recomputes them from a clone.
|
|
100
|
+
|
|
101
|
+
This is a different question from PhysMAP's causal-materiality check — whether a mechanism
|
|
102
|
+
the surrogate never saw is large enough to matter — which is demonstrated separately below.
|
|
103
|
+
Neither is evidence for the other.
|
|
104
|
+
|
|
105
|
+
## The benchmark: seven vehicles, all rerunnable
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
physmap benchmark report # all seven outcomes and the counts above, each marked recomputed or banked
|
|
109
|
+
physmap benchmark run # recomputes all seven from this checkout, diffs against the bank
|
|
110
|
+
physmap benchmark coverage # what that subset does and does not cover
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
All seven vehicles ran, all seven outcomes are reported, and **all seven now ship their
|
|
114
|
+
source data**.
|
|
115
|
+
|
|
116
|
+
**`physmap benchmark run` recomputes all seven from this checkout and diffs the result
|
|
117
|
+
against the committed matrix.** Every field of every cell, not just the headline outcome.
|
|
118
|
+
It exits non-zero if anything drifted, and never writes the bank it is checking itself
|
|
119
|
+
against.
|
|
120
|
+
|
|
121
|
+
Floats are compared within `1e-9` relative, everything else exactly — and the distinction
|
|
122
|
+
is load-bearing rather than a convenience. Every field that decides an outcome is an int
|
|
123
|
+
or a string (counts, verdicts, outcome labels, observability classes), so the tolerance
|
|
124
|
+
cannot absorb a real change. It exists because a clean-clone check on numpy 2.5 found
|
|
125
|
+
`dirker_water.observability_score` differing from the banked value by **one unit in the
|
|
126
|
+
last place**. A match that needed the tolerance is reported as such, not as "identical".
|
|
127
|
+
|
|
128
|
+
`physmap benchmark report` reads the bank without running anything, and says so — the
|
|
129
|
+
report distinguishes a recomputed row from a banked one, and the counts are computed
|
|
130
|
+
rather than written down, so they cannot quietly go stale.
|
|
131
|
+
|
|
132
|
+
**Shipping is also not licensing.** Two of the seven carry affirmative permission:
|
|
133
|
+
`naca_tn1451` (public domain) and `velazquez_sco2` (CC BY 4.0). **Five do not**, and the
|
|
134
|
+
registry and report label them rather than calling everything clear:
|
|
135
|
+
|
|
136
|
+
- `marineau_hypersonic_transition` — no licence and **no prohibition**. Values transcribed
|
|
137
|
+
from a published table in a publicly funded, public-release, government-hosted document.
|
|
138
|
+
- `forrest`, `casper_hypersonic_transition`, `dirker_water`, `jin_sco2_buoyancy` —
|
|
139
|
+
published **against** express publisher terms. AIAA prohibits using its content to
|
|
140
|
+
develop machine-learning models; ASME and Elsevier require permission to reproduce, and
|
|
141
|
+
Elsevier's licence forbids systematic redistribution. Risks accepted knowingly, not
|
|
142
|
+
findings that the terms do not apply.
|
|
143
|
+
|
|
144
|
+
All five redistribute **numbers only** — no paper, figure or PDF, enforced by two audits —
|
|
145
|
+
and all five are removed on objection. The full basis, including the arguments against, is
|
|
146
|
+
in [data/REDISTRIBUTION.md](https://github.com/cloudronin/physmap/blob/main/data/REDISTRIBUTION.md) and [NOTICE](https://github.com/cloudronin/physmap/blob/main/NOTICE).
|
|
147
|
+
|
|
148
|
+
**One dataset is published but not benchmark-grade.** `forrest`'s own header calls its
|
|
149
|
+
values visual estimates for triage only, and its cell is degenerate — one training row, no
|
|
150
|
+
detector fit — so its `DO_NO_HARM` outcome is short-circuited rather than earned. Being
|
|
151
|
+
legal to publish and being fit to benchmark on are different questions; the registry tracks
|
|
152
|
+
them on separate axes. `physmap benchmark coverage` prints it.
|
|
153
|
+
|
|
60
154
|
## Three checks, deliberately kept apart
|
|
61
155
|
|
|
62
156
|
These are different claims resting on different evidence. Conflating them is the
|
|
63
157
|
specific error this project is built to avoid, so nothing here attributes the results of
|
|
64
158
|
one to another.
|
|
65
159
|
|
|
66
|
-
| Check | Question | Status |
|
|
67
|
-
|
|
68
|
-
| **Closure validity** | Is this closure relation being applied outside the range it was calibrated on? | Shipping |
|
|
69
|
-
| **Surrogate observability** | Can the surrogate's inputs even represent the variable that governs the failure? | Shipping |
|
|
70
|
-
| **Causal materiality** | Is the out-of-range mechanism large enough to matter for the quantity of interest? | Preview — method, plus one controlled stress test |
|
|
160
|
+
| Check | Question | Status | Evidence |
|
|
161
|
+
|---|---|---|---|
|
|
162
|
+
| **Closure validity** | Is this closure relation being applied outside the range it was calibrated on? | Shipping | the seven-vehicle benchmark, above |
|
|
163
|
+
| **Surrogate observability** | Can the surrogate's inputs even represent the variable that governs the failure? | Shipping | the seven-vehicle benchmark, above |
|
|
164
|
+
| **Causal materiality** | Is the out-of-range mechanism large enough to matter for the quantity of interest? | Preview — method, plus one controlled stress test | Lewis 35A, below |
|
|
71
165
|
|
|
72
166
|
## What this release claims
|
|
73
167
|
|
|
@@ -82,8 +176,9 @@ screen, the independence guard, and deterministic explanations. Its fixtures are
|
|
|
82
176
|
description. Beyond them, **one controlled stress test runs the causal path on a real
|
|
83
177
|
experiment** (below) — a development demonstration on a single run, not an evaluation.
|
|
84
178
|
|
|
85
|
-
**
|
|
86
|
-
|
|
179
|
+
**No precision, recall or F1.** None is computed, reported or shipped anywhere in this
|
|
180
|
+
package, and a test parses the package and fails if one appears. The benchmark reports counts
|
|
181
|
+
per dataset, never a rate across datasets. The causal-materiality results presented in the NAFEMS
|
|
87
182
|
Multiphysics 2026 abstract are **not** reproduced here: the original study's inputs are
|
|
88
183
|
gone, and the basis for its experimental truth is unresolved. A reconstruction is under
|
|
89
184
|
way under a locked protocol that fixes its rules before any rebuilt number is examined.
|
|
@@ -94,7 +189,11 @@ not reproduction of the published numbers. The protocol defines `REPRODUCED` and
|
|
|
94
189
|
unreachable, so the word cannot drift onto a weaker result. See
|
|
95
190
|
[`protocols/`](https://github.com/cloudronin/physmap/tree/main/protocols).
|
|
96
191
|
|
|
97
|
-
## A controlled
|
|
192
|
+
## A separate question: causal materiality, in a controlled stress test (Lewis 35A)
|
|
193
|
+
|
|
194
|
+
The benchmark asks whether PhysMAP catches failures the OOD detectors cannot see. This asks
|
|
195
|
+
something else: when a mechanism the surrogate never saw becomes active, does it matter for the
|
|
196
|
+
answer — and where?
|
|
98
197
|
|
|
99
198
|
```bash
|
|
100
199
|
physmap stress-test lewis-reuse
|
|
@@ -145,55 +244,6 @@ OpenFOAM. Full record:
|
|
|
145
244
|
talk package — figures, facts sheet, claims ledger, and a clean-clone reproduction record — is
|
|
146
245
|
in [docs/talk/](https://github.com/cloudronin/physmap/blob/main/docs/talk/README.md).
|
|
147
246
|
|
|
148
|
-
## The benchmark: seven vehicles, all rerunnable
|
|
149
|
-
|
|
150
|
-
```bash
|
|
151
|
-
physmap benchmark report # all seven outcomes, each marked recomputed or banked
|
|
152
|
-
physmap benchmark run # recomputes all seven from this checkout, diffs against the bank
|
|
153
|
-
physmap benchmark coverage # what that subset does and does not cover
|
|
154
|
-
```
|
|
155
|
-
|
|
156
|
-
All seven vehicles ran, all seven outcomes are reported, and **all seven now ship their
|
|
157
|
-
source data**.
|
|
158
|
-
|
|
159
|
-
**`physmap benchmark run` recomputes all seven from this checkout and diffs the result
|
|
160
|
-
against the committed matrix.** Every field of every cell, not just the headline outcome.
|
|
161
|
-
It exits non-zero if anything drifted, and never writes the bank it is checking itself
|
|
162
|
-
against.
|
|
163
|
-
|
|
164
|
-
Floats are compared within `1e-9` relative, everything else exactly — and the distinction
|
|
165
|
-
is load-bearing rather than a convenience. Every field that decides an outcome is an int
|
|
166
|
-
or a string (counts, verdicts, outcome labels, observability classes), so the tolerance
|
|
167
|
-
cannot absorb a real change. It exists because a clean-clone check on numpy 2.5 found
|
|
168
|
-
`dirker_water.observability_score` differing from the banked value by **one unit in the
|
|
169
|
-
last place**. A match that needed the tolerance is reported as such, not as "identical".
|
|
170
|
-
|
|
171
|
-
`physmap benchmark report` reads the bank without running anything, and says so — the
|
|
172
|
-
report distinguishes a recomputed row from a banked one, and the counts are computed
|
|
173
|
-
rather than written down, so they cannot quietly go stale.
|
|
174
|
-
|
|
175
|
-
**Shipping is also not licensing.** Two of the seven carry affirmative permission:
|
|
176
|
-
`naca_tn1451` (public domain) and `velazquez_sco2` (CC BY 4.0). **Five do not**, and the
|
|
177
|
-
registry and report label them rather than calling everything clear:
|
|
178
|
-
|
|
179
|
-
- `marineau_hypersonic_transition` — no licence and **no prohibition**. Values transcribed
|
|
180
|
-
from a published table in a publicly funded, public-release, government-hosted document.
|
|
181
|
-
- `forrest`, `casper_hypersonic_transition`, `dirker_water`, `jin_sco2_buoyancy` —
|
|
182
|
-
published **against** express publisher terms. AIAA prohibits using its content to
|
|
183
|
-
develop machine-learning models; ASME and Elsevier require permission to reproduce, and
|
|
184
|
-
Elsevier's licence forbids systematic redistribution. Risks accepted knowingly, not
|
|
185
|
-
findings that the terms do not apply.
|
|
186
|
-
|
|
187
|
-
All five redistribute **numbers only** — no paper, figure or PDF, enforced by two audits —
|
|
188
|
-
and all five are removed on objection. The full basis, including the arguments against, is
|
|
189
|
-
in [data/REDISTRIBUTION.md](https://github.com/cloudronin/physmap/blob/main/data/REDISTRIBUTION.md) and [NOTICE](https://github.com/cloudronin/physmap/blob/main/NOTICE).
|
|
190
|
-
|
|
191
|
-
**One dataset is published but not benchmark-grade.** `forrest`'s own header calls its
|
|
192
|
-
values visual estimates for triage only, and its cell is degenerate — one training row, no
|
|
193
|
-
detector fit — so its `DO_NO_HARM` outcome is short-circuited rather than earned. Being
|
|
194
|
-
legal to publish and being fit to benchmark on are different questions; the registry tracks
|
|
195
|
-
them on separate axes. `physmap benchmark coverage` prints it.
|
|
196
|
-
|
|
197
247
|
## Install
|
|
198
248
|
|
|
199
249
|
```bash
|
|
@@ -217,6 +267,55 @@ redistribution terms and partly because it is not runtime data. From a plain
|
|
|
217
267
|
Optional extras: `[jsonld]` adds evidence export via `uofa`, `[experiment]` adds
|
|
218
268
|
`matplotlib` for benchmark figures, `[dev]` adds the test tooling.
|
|
219
269
|
|
|
270
|
+
## Using the guardrail
|
|
271
|
+
|
|
272
|
+
The guardrail does not guess which checks to combine. Its setup decides.
|
|
273
|
+
|
|
274
|
+
```python
|
|
275
|
+
from physmap import ColumnMap, CredibilityGuardrail, Regime
|
|
276
|
+
|
|
277
|
+
guard = CredibilityGuardrail(
|
|
278
|
+
surrogate_inputs=["Re", "Pr"], # exactly the inputs your surrogate takes
|
|
279
|
+
regime=Regime.ENTRANCE_REGION_PIPE, # the physics it is used in: picks the closures
|
|
280
|
+
)
|
|
281
|
+
cols = ColumnMap(inputs=["Re", "Pr", "x_over_D"]) # the surrogate inputs, plus every
|
|
282
|
+
# variable those closures put a bound on
|
|
283
|
+
guard.fit(train_X, train_y=train_y, columns=cols) # arrays whose columns follow `cols`
|
|
284
|
+
for a in guard.assess(test_X, columns=cols):
|
|
285
|
+
print(a.verdict.name, a.rationale)
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
**What each piece does**
|
|
289
|
+
|
|
290
|
+
- **`surrogate_inputs`** — the columns your surrogate actually uses. The input-based OOD
|
|
291
|
+
detectors are fitted on exactly these.
|
|
292
|
+
- **`regime`** — picks the closure relations whose validated ranges the closure check uses.
|
|
293
|
+
Choose the regime that matches your physics, or name one closure with `closure_id=`. If none
|
|
294
|
+
matches, use `Regime.UNLISTED`: the guardrail then runs the OOD detectors alone.
|
|
295
|
+
- **The data columns** — must include every variable those closures bound, even ones the
|
|
296
|
+
surrogate never sees (here `x_over_D`). If one is missing, `fit` stops and names it.
|
|
297
|
+
- **`train_y`** — the training targets. The GP-variance detector needs them.
|
|
298
|
+
- **`detectors=`** — optional. The default is novelty density and GP variance (the input-based
|
|
299
|
+
OOD detectors) plus closure validity. Distance-to-training and a conformal residual check are
|
|
300
|
+
also available.
|
|
301
|
+
|
|
302
|
+
**How the verdict is combined, for each prediction**
|
|
303
|
+
|
|
304
|
+
- At setup, each bounded variable is classed **observable** (it is a surrogate input),
|
|
305
|
+
**unobservable** (it is not), or **partial**. `guard.observability_classification` shows the
|
|
306
|
+
result.
|
|
307
|
+
- The closure check fires on an **unobservable** variable → `REJECT`. The OOD detectors
|
|
308
|
+
cannot see that variable, so the closure check is trusted.
|
|
309
|
+
- It fires on a **partial** variable → a calibrated blend where a calibration exists,
|
|
310
|
+
otherwise `UNCERTAIN`.
|
|
311
|
+
- Otherwise — the closure check is quiet, or the variable is one the detectors can see — the
|
|
312
|
+
OOD detectors decide: none fired → `TRUSTWORTHY`, a reject-level fire → `REJECT`, anything
|
|
313
|
+
else → `WARN`.
|
|
314
|
+
|
|
315
|
+
Every listed regime's closures ship in the published corpus. A complete, runnable version of
|
|
316
|
+
the example above is
|
|
317
|
+
[`examples/naca_entrance_region.py`](https://github.com/cloudronin/physmap/blob/main/examples/naca_entrance_region.py).
|
|
318
|
+
|
|
220
319
|
## The corpus
|
|
221
320
|
|
|
222
321
|
The code is MIT and the corpus data is CC BY 4.0 — see the licences below.
|
|
@@ -12,6 +12,9 @@ PhysMAP reads the bound variable from the test coordinates instead of from the
|
|
|
12
12
|
surrogate's inputs, checks it against the closure relation's validated range, and knows
|
|
13
13
|
at fit time whether that variable is structurally observable to the surrogate at all.
|
|
14
14
|
|
|
15
|
+
**That is the point of PhysMAP: it catches the failures an input-based OOD detector cannot
|
|
16
|
+
see, because what broke the surrogate is not one of its inputs.**
|
|
17
|
+
|
|
15
18
|
```
|
|
16
19
|
verdicts: {'REJECT': 45}
|
|
17
20
|
rationale: Prediction relies on gnielinski-1976 beyond its validated x_over_D bound
|
|
@@ -25,17 +28,108 @@ No LLM is in any path. Every explanation is a deterministic rendered template.
|
|
|
25
28
|
|
|
26
29
|
Run it yourself: [`examples/naca_entrance_region.py`](https://github.com/cloudronin/physmap/blob/main/examples/naca_entrance_region.py).
|
|
27
30
|
|
|
31
|
+
## What PhysMAP adds to OOD detection
|
|
32
|
+
|
|
33
|
+

|
|
34
|
+
|
|
35
|
+
An input-based OOD detector judges a prediction by where its inputs sit relative to the
|
|
36
|
+
training data. It asks that well — but it cannot see a change in something that is not an
|
|
37
|
+
input. PhysMAP adds the check it cannot make: it reads the variables the surrogate never saw,
|
|
38
|
+
tests the physics relation behind the surrogate against its validated range, and knows at
|
|
39
|
+
setup which of those variables the detectors can see. It keeps the OOD detectors, and
|
|
40
|
+
overrides them only where they are structurally blind.
|
|
41
|
+
|
|
42
|
+
The seven-vehicle benchmark measures exactly that, per dataset, at the default setting (the
|
|
43
|
+
99th percentile):
|
|
44
|
+
|
|
45
|
+
| Dataset | Can the OOD detectors see the cause? | Wrong predictions caught only by PhysMAP | Right predictions flagged anyway |
|
|
46
|
+
|---|---|---|---|
|
|
47
|
+
| `naca_tn1451` — heated pipe, entrance region | no | 20 of 20 | 24 |
|
|
48
|
+
| `casper_hypersonic_transition` — hypersonic transition | no | 4 of 8 | 0 |
|
|
49
|
+
| `jin_sco2_buoyancy` — supercritical CO2, vertical tube | no | 15 of 26 | 0 |
|
|
50
|
+
| `velazquez_sco2` — supercritical CO2, property variation | partly | 18 of 67 | 0 |
|
|
51
|
+
| `dirker_water` — water, horizontal tube | partly | 2 of 11 | 16 |
|
|
52
|
+
| `marineau_hypersonic_transition` — hypersonic transition | yes | 0 of 6 | 0 |
|
|
53
|
+
| `forrest` — rectangular channel | yes | not tested: one training row | — |
|
|
54
|
+
|
|
55
|
+
- **Where it helps:** when the cause of failure is hidden from the surrogate's inputs. There
|
|
56
|
+
PhysMAP catches wrong predictions the OOD detectors miss entirely — all 20 for the pipe
|
|
57
|
+
entrance.
|
|
58
|
+
- **Where it doesn't:** when the cause is an input, the OOD detectors already see it and
|
|
59
|
+
PhysMAP adds nothing. `marineau_hypersonic_transition` is the control that shows it.
|
|
60
|
+
- **The cost:** the closure check flags anything outside a relation's validated range, even
|
|
61
|
+
when the surrogate happens to be right — 24 false alarms for the pipe entrance, 16 for
|
|
62
|
+
`dirker_water`.
|
|
63
|
+
|
|
64
|
+
Counts are rows of each dataset, not independent cases, and are never pooled into a rate
|
|
65
|
+
across datasets; the source values are digitised from publications. The benchmark's
|
|
66
|
+
input-based detectors are distance-to-training and GP variance. `physmap benchmark report`
|
|
67
|
+
prints these counts, and `physmap benchmark run` recomputes them from a clone.
|
|
68
|
+
|
|
69
|
+
This is a different question from PhysMAP's causal-materiality check — whether a mechanism
|
|
70
|
+
the surrogate never saw is large enough to matter — which is demonstrated separately below.
|
|
71
|
+
Neither is evidence for the other.
|
|
72
|
+
|
|
73
|
+
## The benchmark: seven vehicles, all rerunnable
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
physmap benchmark report # all seven outcomes and the counts above, each marked recomputed or banked
|
|
77
|
+
physmap benchmark run # recomputes all seven from this checkout, diffs against the bank
|
|
78
|
+
physmap benchmark coverage # what that subset does and does not cover
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
All seven vehicles ran, all seven outcomes are reported, and **all seven now ship their
|
|
82
|
+
source data**.
|
|
83
|
+
|
|
84
|
+
**`physmap benchmark run` recomputes all seven from this checkout and diffs the result
|
|
85
|
+
against the committed matrix.** Every field of every cell, not just the headline outcome.
|
|
86
|
+
It exits non-zero if anything drifted, and never writes the bank it is checking itself
|
|
87
|
+
against.
|
|
88
|
+
|
|
89
|
+
Floats are compared within `1e-9` relative, everything else exactly — and the distinction
|
|
90
|
+
is load-bearing rather than a convenience. Every field that decides an outcome is an int
|
|
91
|
+
or a string (counts, verdicts, outcome labels, observability classes), so the tolerance
|
|
92
|
+
cannot absorb a real change. It exists because a clean-clone check on numpy 2.5 found
|
|
93
|
+
`dirker_water.observability_score` differing from the banked value by **one unit in the
|
|
94
|
+
last place**. A match that needed the tolerance is reported as such, not as "identical".
|
|
95
|
+
|
|
96
|
+
`physmap benchmark report` reads the bank without running anything, and says so — the
|
|
97
|
+
report distinguishes a recomputed row from a banked one, and the counts are computed
|
|
98
|
+
rather than written down, so they cannot quietly go stale.
|
|
99
|
+
|
|
100
|
+
**Shipping is also not licensing.** Two of the seven carry affirmative permission:
|
|
101
|
+
`naca_tn1451` (public domain) and `velazquez_sco2` (CC BY 4.0). **Five do not**, and the
|
|
102
|
+
registry and report label them rather than calling everything clear:
|
|
103
|
+
|
|
104
|
+
- `marineau_hypersonic_transition` — no licence and **no prohibition**. Values transcribed
|
|
105
|
+
from a published table in a publicly funded, public-release, government-hosted document.
|
|
106
|
+
- `forrest`, `casper_hypersonic_transition`, `dirker_water`, `jin_sco2_buoyancy` —
|
|
107
|
+
published **against** express publisher terms. AIAA prohibits using its content to
|
|
108
|
+
develop machine-learning models; ASME and Elsevier require permission to reproduce, and
|
|
109
|
+
Elsevier's licence forbids systematic redistribution. Risks accepted knowingly, not
|
|
110
|
+
findings that the terms do not apply.
|
|
111
|
+
|
|
112
|
+
All five redistribute **numbers only** — no paper, figure or PDF, enforced by two audits —
|
|
113
|
+
and all five are removed on objection. The full basis, including the arguments against, is
|
|
114
|
+
in [data/REDISTRIBUTION.md](https://github.com/cloudronin/physmap/blob/main/data/REDISTRIBUTION.md) and [NOTICE](https://github.com/cloudronin/physmap/blob/main/NOTICE).
|
|
115
|
+
|
|
116
|
+
**One dataset is published but not benchmark-grade.** `forrest`'s own header calls its
|
|
117
|
+
values visual estimates for triage only, and its cell is degenerate — one training row, no
|
|
118
|
+
detector fit — so its `DO_NO_HARM` outcome is short-circuited rather than earned. Being
|
|
119
|
+
legal to publish and being fit to benchmark on are different questions; the registry tracks
|
|
120
|
+
them on separate axes. `physmap benchmark coverage` prints it.
|
|
121
|
+
|
|
28
122
|
## Three checks, deliberately kept apart
|
|
29
123
|
|
|
30
124
|
These are different claims resting on different evidence. Conflating them is the
|
|
31
125
|
specific error this project is built to avoid, so nothing here attributes the results of
|
|
32
126
|
one to another.
|
|
33
127
|
|
|
34
|
-
| Check | Question | Status |
|
|
35
|
-
|
|
36
|
-
| **Closure validity** | Is this closure relation being applied outside the range it was calibrated on? | Shipping |
|
|
37
|
-
| **Surrogate observability** | Can the surrogate's inputs even represent the variable that governs the failure? | Shipping |
|
|
38
|
-
| **Causal materiality** | Is the out-of-range mechanism large enough to matter for the quantity of interest? | Preview — method, plus one controlled stress test |
|
|
128
|
+
| Check | Question | Status | Evidence |
|
|
129
|
+
|---|---|---|---|
|
|
130
|
+
| **Closure validity** | Is this closure relation being applied outside the range it was calibrated on? | Shipping | the seven-vehicle benchmark, above |
|
|
131
|
+
| **Surrogate observability** | Can the surrogate's inputs even represent the variable that governs the failure? | Shipping | the seven-vehicle benchmark, above |
|
|
132
|
+
| **Causal materiality** | Is the out-of-range mechanism large enough to matter for the quantity of interest? | Preview — method, plus one controlled stress test | Lewis 35A, below |
|
|
39
133
|
|
|
40
134
|
## What this release claims
|
|
41
135
|
|
|
@@ -50,8 +144,9 @@ screen, the independence guard, and deterministic explanations. Its fixtures are
|
|
|
50
144
|
description. Beyond them, **one controlled stress test runs the causal path on a real
|
|
51
145
|
experiment** (below) — a development demonstration on a single run, not an evaluation.
|
|
52
146
|
|
|
53
|
-
**
|
|
54
|
-
|
|
147
|
+
**No precision, recall or F1.** None is computed, reported or shipped anywhere in this
|
|
148
|
+
package, and a test parses the package and fails if one appears. The benchmark reports counts
|
|
149
|
+
per dataset, never a rate across datasets. The causal-materiality results presented in the NAFEMS
|
|
55
150
|
Multiphysics 2026 abstract are **not** reproduced here: the original study's inputs are
|
|
56
151
|
gone, and the basis for its experimental truth is unresolved. A reconstruction is under
|
|
57
152
|
way under a locked protocol that fixes its rules before any rebuilt number is examined.
|
|
@@ -62,7 +157,11 @@ not reproduction of the published numbers. The protocol defines `REPRODUCED` and
|
|
|
62
157
|
unreachable, so the word cannot drift onto a weaker result. See
|
|
63
158
|
[`protocols/`](https://github.com/cloudronin/physmap/tree/main/protocols).
|
|
64
159
|
|
|
65
|
-
## A controlled
|
|
160
|
+
## A separate question: causal materiality, in a controlled stress test (Lewis 35A)
|
|
161
|
+
|
|
162
|
+
The benchmark asks whether PhysMAP catches failures the OOD detectors cannot see. This asks
|
|
163
|
+
something else: when a mechanism the surrogate never saw becomes active, does it matter for the
|
|
164
|
+
answer — and where?
|
|
66
165
|
|
|
67
166
|
```bash
|
|
68
167
|
physmap stress-test lewis-reuse
|
|
@@ -113,55 +212,6 @@ OpenFOAM. Full record:
|
|
|
113
212
|
talk package — figures, facts sheet, claims ledger, and a clean-clone reproduction record — is
|
|
114
213
|
in [docs/talk/](https://github.com/cloudronin/physmap/blob/main/docs/talk/README.md).
|
|
115
214
|
|
|
116
|
-
## The benchmark: seven vehicles, all rerunnable
|
|
117
|
-
|
|
118
|
-
```bash
|
|
119
|
-
physmap benchmark report # all seven outcomes, each marked recomputed or banked
|
|
120
|
-
physmap benchmark run # recomputes all seven from this checkout, diffs against the bank
|
|
121
|
-
physmap benchmark coverage # what that subset does and does not cover
|
|
122
|
-
```
|
|
123
|
-
|
|
124
|
-
All seven vehicles ran, all seven outcomes are reported, and **all seven now ship their
|
|
125
|
-
source data**.
|
|
126
|
-
|
|
127
|
-
**`physmap benchmark run` recomputes all seven from this checkout and diffs the result
|
|
128
|
-
against the committed matrix.** Every field of every cell, not just the headline outcome.
|
|
129
|
-
It exits non-zero if anything drifted, and never writes the bank it is checking itself
|
|
130
|
-
against.
|
|
131
|
-
|
|
132
|
-
Floats are compared within `1e-9` relative, everything else exactly — and the distinction
|
|
133
|
-
is load-bearing rather than a convenience. Every field that decides an outcome is an int
|
|
134
|
-
or a string (counts, verdicts, outcome labels, observability classes), so the tolerance
|
|
135
|
-
cannot absorb a real change. It exists because a clean-clone check on numpy 2.5 found
|
|
136
|
-
`dirker_water.observability_score` differing from the banked value by **one unit in the
|
|
137
|
-
last place**. A match that needed the tolerance is reported as such, not as "identical".
|
|
138
|
-
|
|
139
|
-
`physmap benchmark report` reads the bank without running anything, and says so — the
|
|
140
|
-
report distinguishes a recomputed row from a banked one, and the counts are computed
|
|
141
|
-
rather than written down, so they cannot quietly go stale.
|
|
142
|
-
|
|
143
|
-
**Shipping is also not licensing.** Two of the seven carry affirmative permission:
|
|
144
|
-
`naca_tn1451` (public domain) and `velazquez_sco2` (CC BY 4.0). **Five do not**, and the
|
|
145
|
-
registry and report label them rather than calling everything clear:
|
|
146
|
-
|
|
147
|
-
- `marineau_hypersonic_transition` — no licence and **no prohibition**. Values transcribed
|
|
148
|
-
from a published table in a publicly funded, public-release, government-hosted document.
|
|
149
|
-
- `forrest`, `casper_hypersonic_transition`, `dirker_water`, `jin_sco2_buoyancy` —
|
|
150
|
-
published **against** express publisher terms. AIAA prohibits using its content to
|
|
151
|
-
develop machine-learning models; ASME and Elsevier require permission to reproduce, and
|
|
152
|
-
Elsevier's licence forbids systematic redistribution. Risks accepted knowingly, not
|
|
153
|
-
findings that the terms do not apply.
|
|
154
|
-
|
|
155
|
-
All five redistribute **numbers only** — no paper, figure or PDF, enforced by two audits —
|
|
156
|
-
and all five are removed on objection. The full basis, including the arguments against, is
|
|
157
|
-
in [data/REDISTRIBUTION.md](https://github.com/cloudronin/physmap/blob/main/data/REDISTRIBUTION.md) and [NOTICE](https://github.com/cloudronin/physmap/blob/main/NOTICE).
|
|
158
|
-
|
|
159
|
-
**One dataset is published but not benchmark-grade.** `forrest`'s own header calls its
|
|
160
|
-
values visual estimates for triage only, and its cell is degenerate — one training row, no
|
|
161
|
-
detector fit — so its `DO_NO_HARM` outcome is short-circuited rather than earned. Being
|
|
162
|
-
legal to publish and being fit to benchmark on are different questions; the registry tracks
|
|
163
|
-
them on separate axes. `physmap benchmark coverage` prints it.
|
|
164
|
-
|
|
165
215
|
## Install
|
|
166
216
|
|
|
167
217
|
```bash
|
|
@@ -185,6 +235,55 @@ redistribution terms and partly because it is not runtime data. From a plain
|
|
|
185
235
|
Optional extras: `[jsonld]` adds evidence export via `uofa`, `[experiment]` adds
|
|
186
236
|
`matplotlib` for benchmark figures, `[dev]` adds the test tooling.
|
|
187
237
|
|
|
238
|
+
## Using the guardrail
|
|
239
|
+
|
|
240
|
+
The guardrail does not guess which checks to combine. Its setup decides.
|
|
241
|
+
|
|
242
|
+
```python
|
|
243
|
+
from physmap import ColumnMap, CredibilityGuardrail, Regime
|
|
244
|
+
|
|
245
|
+
guard = CredibilityGuardrail(
|
|
246
|
+
surrogate_inputs=["Re", "Pr"], # exactly the inputs your surrogate takes
|
|
247
|
+
regime=Regime.ENTRANCE_REGION_PIPE, # the physics it is used in: picks the closures
|
|
248
|
+
)
|
|
249
|
+
cols = ColumnMap(inputs=["Re", "Pr", "x_over_D"]) # the surrogate inputs, plus every
|
|
250
|
+
# variable those closures put a bound on
|
|
251
|
+
guard.fit(train_X, train_y=train_y, columns=cols) # arrays whose columns follow `cols`
|
|
252
|
+
for a in guard.assess(test_X, columns=cols):
|
|
253
|
+
print(a.verdict.name, a.rationale)
|
|
254
|
+
```
|
|
255
|
+
|
|
256
|
+
**What each piece does**
|
|
257
|
+
|
|
258
|
+
- **`surrogate_inputs`** — the columns your surrogate actually uses. The input-based OOD
|
|
259
|
+
detectors are fitted on exactly these.
|
|
260
|
+
- **`regime`** — picks the closure relations whose validated ranges the closure check uses.
|
|
261
|
+
Choose the regime that matches your physics, or name one closure with `closure_id=`. If none
|
|
262
|
+
matches, use `Regime.UNLISTED`: the guardrail then runs the OOD detectors alone.
|
|
263
|
+
- **The data columns** — must include every variable those closures bound, even ones the
|
|
264
|
+
surrogate never sees (here `x_over_D`). If one is missing, `fit` stops and names it.
|
|
265
|
+
- **`train_y`** — the training targets. The GP-variance detector needs them.
|
|
266
|
+
- **`detectors=`** — optional. The default is novelty density and GP variance (the input-based
|
|
267
|
+
OOD detectors) plus closure validity. Distance-to-training and a conformal residual check are
|
|
268
|
+
also available.
|
|
269
|
+
|
|
270
|
+
**How the verdict is combined, for each prediction**
|
|
271
|
+
|
|
272
|
+
- At setup, each bounded variable is classed **observable** (it is a surrogate input),
|
|
273
|
+
**unobservable** (it is not), or **partial**. `guard.observability_classification` shows the
|
|
274
|
+
result.
|
|
275
|
+
- The closure check fires on an **unobservable** variable → `REJECT`. The OOD detectors
|
|
276
|
+
cannot see that variable, so the closure check is trusted.
|
|
277
|
+
- It fires on a **partial** variable → a calibrated blend where a calibration exists,
|
|
278
|
+
otherwise `UNCERTAIN`.
|
|
279
|
+
- Otherwise — the closure check is quiet, or the variable is one the detectors can see — the
|
|
280
|
+
OOD detectors decide: none fired → `TRUSTWORTHY`, a reject-level fire → `REJECT`, anything
|
|
281
|
+
else → `WARN`.
|
|
282
|
+
|
|
283
|
+
Every listed regime's closures ship in the published corpus. A complete, runnable version of
|
|
284
|
+
the example above is
|
|
285
|
+
[`examples/naca_entrance_region.py`](https://github.com/cloudronin/physmap/blob/main/examples/naca_entrance_region.py).
|
|
286
|
+
|
|
188
287
|
## The corpus
|
|
189
288
|
|
|
190
289
|
The code is MIT and the corpus data is CC BY 4.0 — see the licences below.
|
|
@@ -6,7 +6,7 @@ build-backend = "setuptools.build_meta"
|
|
|
6
6
|
|
|
7
7
|
[project]
|
|
8
8
|
name = "physmap"
|
|
9
|
-
version = "0.2.
|
|
9
|
+
version = "0.2.2"
|
|
10
10
|
description = "Physics-aware credibility checks for AI surrogates in multiphysics simulation: closure validity, surrogate observability, and causal materiality."
|
|
11
11
|
readme = "README.md"
|
|
12
12
|
requires-python = ">=3.10"
|
|
@@ -103,6 +103,34 @@ def coverage_note() -> str:
|
|
|
103
103
|
return "\n".join(lines)
|
|
104
104
|
|
|
105
105
|
|
|
106
|
+
def _what_physmap_adds(cells: list[dict | None]) -> list[str]:
|
|
107
|
+
"""Per vehicle, at the shipped reference operating percentile: the wrong predictions the
|
|
108
|
+
closure-validity check caught while the input-based detectors stayed quiet, and the right
|
|
109
|
+
predictions it flagged anyway. Row counts, never pooled into a rate across vehicles."""
|
|
110
|
+
pcts = {int(c["ref_pct"]) for c in cells if c is not None and c.get("ref_pct")}
|
|
111
|
+
at = f"percentile {', '.join(str(p) for p in sorted(pcts))}" if pcts else "percentile"
|
|
112
|
+
out = [
|
|
113
|
+
"What the closure-validity check adds to the input-based detectors",
|
|
114
|
+
f" at the reference operating {at}. Counts are rows of each dataset, not",
|
|
115
|
+
" independent cases, and the datasets are not pooled.",
|
|
116
|
+
"",
|
|
117
|
+
f" {'vehicle':34} {'failure variable':17} {'wrong: caught only by PhysMAP':31} "
|
|
118
|
+
f"right: flagged anyway",
|
|
119
|
+
]
|
|
120
|
+
for cell in cells:
|
|
121
|
+
if cell is None:
|
|
122
|
+
continue
|
|
123
|
+
ref = (cell.get("per_pct") or {}).get(str(int(cell.get("ref_pct") or 0)))
|
|
124
|
+
head = f" {cell['vehicle_id']:34} {cell['failure_observability']:17} "
|
|
125
|
+
if not ref:
|
|
126
|
+
out.append(head + f"not tested -- {cell.get('n_train', 0)} training row(s), "
|
|
127
|
+
f"no detector fit")
|
|
128
|
+
continue
|
|
129
|
+
caught = f"{ref['clean_lift']} of {ref['n_wrong']}"
|
|
130
|
+
out.append(head + f"{caught:31} {ref['misaligned']}")
|
|
131
|
+
return out
|
|
132
|
+
|
|
133
|
+
|
|
106
134
|
def render_report(rerun_results: dict[str, Any] | None = None) -> str:
|
|
107
135
|
"""The full seven-vehicle table.
|
|
108
136
|
|
|
@@ -165,6 +193,9 @@ def render_report(rerun_results: dict[str, Any] | None = None) -> str:
|
|
|
165
193
|
out.append("")
|
|
166
194
|
out.append(f"observability guards: {'all passed' if matrix['all_guards_passed'] else 'FAILED'}")
|
|
167
195
|
out.append("")
|
|
196
|
+
out.extend(_what_physmap_adds(
|
|
197
|
+
[rerun_results.get(v.vehicle_id) or cells.get(v.vehicle_id) for v in VEHICLES]))
|
|
198
|
+
out.append("")
|
|
168
199
|
unlicensed = unlicensed_shipped_ids()
|
|
169
200
|
if unlicensed:
|
|
170
201
|
out.append("Redistribution basis -- read this before reusing any of this data")
|