physiotrace 0.1.0a1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- physiotrace-0.1.0a1/LICENSE +21 -0
- physiotrace-0.1.0a1/PKG-INFO +211 -0
- physiotrace-0.1.0a1/README.md +171 -0
- physiotrace-0.1.0a1/pyproject.toml +55 -0
- physiotrace-0.1.0a1/setup.cfg +4 -0
- physiotrace-0.1.0a1/src/physiotrace/__init__.py +67 -0
- physiotrace-0.1.0a1/src/physiotrace/_validation.py +54 -0
- physiotrace-0.1.0a1/src/physiotrace/audit.py +627 -0
- physiotrace-0.1.0a1/src/physiotrace/bootstrap.py +87 -0
- physiotrace-0.1.0a1/src/physiotrace/diagnostics.py +188 -0
- physiotrace-0.1.0a1/src/physiotrace/heads.py +65 -0
- physiotrace-0.1.0a1/src/physiotrace/intervene.py +239 -0
- physiotrace-0.1.0a1/src/physiotrace/metrics.py +55 -0
- physiotrace-0.1.0a1/src/physiotrace/recover.py +278 -0
- physiotrace-0.1.0a1/src/physiotrace/report.py +123 -0
- physiotrace-0.1.0a1/src/physiotrace/stress.py +57 -0
- physiotrace-0.1.0a1/src/physiotrace/synthetic.py +138 -0
- physiotrace-0.1.0a1/src/physiotrace/verify.py +77 -0
- physiotrace-0.1.0a1/src/physiotrace/weights.py +238 -0
- physiotrace-0.1.0a1/src/physiotrace.egg-info/PKG-INFO +211 -0
- physiotrace-0.1.0a1/src/physiotrace.egg-info/SOURCES.txt +27 -0
- physiotrace-0.1.0a1/src/physiotrace.egg-info/dependency_links.txt +1 -0
- physiotrace-0.1.0a1/src/physiotrace.egg-info/requires.txt +16 -0
- physiotrace-0.1.0a1/src/physiotrace.egg-info/top_level.txt +1 -0
- physiotrace-0.1.0a1/tests/test_audit.py +74 -0
- physiotrace-0.1.0a1/tests/test_intervene_recover.py +98 -0
- physiotrace-0.1.0a1/tests/test_metrics_bootstrap.py +52 -0
- physiotrace-0.1.0a1/tests/test_split_and_checks.py +111 -0
- physiotrace-0.1.0a1/tests/test_weights.py +103 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ayana Mussabayeva and Anuar Aimoldin
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: physiotrace
|
|
3
|
+
Version: 0.1.0a1
|
|
4
|
+
Summary: Behavioral audit of provenance reliance in frozen-encoder task heads (Recover, Stress, Intervene, Verify).
|
|
5
|
+
Author: Anuar Aimoldin
|
|
6
|
+
Author-email: Ayana Mussabayeva <ayana.mussabayeva@gmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Project-URL: Homepage, https://github.com/AyanaMussabayeva/physiotrace
|
|
9
|
+
Project-URL: Documentation, https://github.com/AyanaMussabayeva/physiotrace/blob/main/docs/tutorial.ipynb
|
|
10
|
+
Project-URL: Source, https://github.com/AyanaMussabayeva/physiotrace
|
|
11
|
+
Project-URL: Issues, https://github.com/AyanaMussabayeva/physiotrace/issues
|
|
12
|
+
Project-URL: Changelog, https://github.com/AyanaMussabayeva/physiotrace/blob/main/CHANGELOG.md
|
|
13
|
+
Keywords: foundation models,shortcut learning,provenance,EEG,ECG,robustness,audit
|
|
14
|
+
Classifier: Development Status :: 3 - Alpha
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Programming Language :: Python :: 3
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: numpy>=1.24
|
|
27
|
+
Requires-Dist: scipy>=1.10
|
|
28
|
+
Requires-Dist: scikit-learn>=1.3
|
|
29
|
+
Requires-Dist: pandas>=2.0
|
|
30
|
+
Provides-Extra: plot
|
|
31
|
+
Requires-Dist: matplotlib>=3.7; extra == "plot"
|
|
32
|
+
Provides-Extra: dev
|
|
33
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
34
|
+
Provides-Extra: docs
|
|
35
|
+
Requires-Dist: matplotlib>=3.7; extra == "docs"
|
|
36
|
+
Requires-Dist: nbformat>=5; extra == "docs"
|
|
37
|
+
Requires-Dist: nbclient>=0.7; extra == "docs"
|
|
38
|
+
Requires-Dist: ipykernel>=6; extra == "docs"
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
|
|
41
|
+
# physiotrace
|
|
42
|
+
|
|
43
|
+
**Does your task head rely on how the data were recorded?**
|
|
44
|
+
|
|
45
|
+
`physiotrace` audits a linear task head on a frozen encoder for reliance on
|
|
46
|
+
acquisition provenance — device, site, reference, montage, scanner, batch —
|
|
47
|
+
and certifies a remedy only when it is specific and safe. It works on
|
|
48
|
+
embeddings, so any frozen model (EEG, ECG, imaging, omics) can be audited
|
|
49
|
+
without retraining.
|
|
50
|
+
|
|
51
|
+
Two common measurements do not answer this question. Shifted-transfer
|
|
52
|
+
benchmarks can fail for unrelated reasons, and a provenance probe shows only
|
|
53
|
+
that provenance is decodable, not that a head uses it. PhysioTRACE tests the
|
|
54
|
+
head's behavior directly:
|
|
55
|
+
|
|
56
|
+
| Step | Question | Output |
|
|
57
|
+
|---|---|---|
|
|
58
|
+
| **Recover** | How decodable is provenance from the embedding? | calibrated Linear Provenance Information (bits per unit) or conditional accessibility AUROC |
|
|
59
|
+
| **Stress** | Does a head fitted under the provenance–target association degrade more than a control head when only that association is reversed on the same records? | excess vulnerability `G` with an identity-aware interval |
|
|
60
|
+
| **Intervene** | Does removing a train-localized provenance component remove the failure? | `G` after removal |
|
|
61
|
+
| **Verify** | Does the removal beat rank- and energy-matched random controls, and keep utility within a declared margin `δ`? | verdict |
|
|
62
|
+
|
|
63
|
+
Every audit ends in one of three verdicts: **no reliance**, **reliance – remedy
|
|
64
|
+
certified**, or **reliance – remedy not certified**.
|
|
65
|
+
|
|
66
|
+
**New here? Start with the [tutorial](https://github.com/AyanaMussabayeva/physiotrace/blob/main/docs/tutorial.ipynb)**: bring your own
|
|
67
|
+
embeddings, choose the settings, read the report, and avoid the mistakes that
|
|
68
|
+
make an audit misleading.
|
|
69
|
+
|
|
70
|
+
## Install
|
|
71
|
+
|
|
72
|
+
```bash
|
|
73
|
+
pip install --pre physiotrace # 0.1.0a1 is a pre-release
|
|
74
|
+
pip install --pre "physiotrace[plot]" # with matplotlib, for the tutorial figures
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Python 3.10–3.13. Tested on every push from the oldest supported dependencies
|
|
78
|
+
(numpy 1.24, scipy 1.10, scikit-learn 1.3, pandas 2.0) to the latest.
|
|
79
|
+
|
|
80
|
+
## Quickstart
|
|
81
|
+
|
|
82
|
+
### Observational data (for example device or site)
|
|
83
|
+
|
|
84
|
+
```python
|
|
85
|
+
import physiotrace as pt
|
|
86
|
+
|
|
87
|
+
train = pt.Split(embeddings, y, u, clusters=patient_id) # u: e.g. device A / device B
|
|
88
|
+
test = pt.Split(test_embeddings, test_y, test_u, clusters=test_patient_id)
|
|
89
|
+
|
|
90
|
+
report = pt.audit_observational(train, test, epsilon=0.01, delta=0.01)
|
|
91
|
+
print(report)
|
|
92
|
+
report.verdict # pt.Verdict.REMEDY_CERTIFIED / REMEDY_NOT_CERTIFIED / NO_RELIANCE
|
|
93
|
+
report.to_dict() # JSON-serializable summary
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
```text
|
|
97
|
+
PhysioTRACE audit (observational)
|
|
98
|
+
verdict: reliance, remedy certified
|
|
99
|
+
Recover conditional_accessibility_auroc = 0.9946
|
|
100
|
+
Stress G = 0.1384 [0.1228, 0.1539] (epsilon = 0.01; 95% interval)
|
|
101
|
+
Intervene G after rank-1 removal = -0.0000 [-0.0013, 0.0012] (100% reduction)
|
|
102
|
+
Verify specific: yes (beats 100/100 random controls)
|
|
103
|
+
safe: yes (control-head AUROC, independent regime change -0.0000 [-0.0004, 0.0003], delta = 0.01)
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
(Output of [`examples/quickstart_synthetic.py`](https://github.com/AyanaMussabayeva/physiotrace/blob/main/examples/quickstart_synthetic.py), which also shows the other two
|
|
107
|
+
verdicts, a cross-fitted audit, and a paired audit.)
|
|
108
|
+
|
|
109
|
+
`y` and `u` can hold any two labels each (strings or numbers). `clusters`
|
|
110
|
+
identifies who each record belongs to (patient, subject); it is the bootstrap
|
|
111
|
+
unit, and a cluster on both sides of the split is rejected. `epsilon` is the
|
|
112
|
+
smallest `G` that counts as reliance and `delta` the tolerated utility loss;
|
|
113
|
+
declare both, and the localization `rank`, before looking at test results.
|
|
114
|
+
|
|
115
|
+
The exposed head is fitted on the natural association and the control head
|
|
116
|
+
with independence weights. Provenance is localized conditionally on the
|
|
117
|
+
target, so removal does not by default discard population signal that sources
|
|
118
|
+
differ in.
|
|
119
|
+
|
|
120
|
+
To cross-fit over folds instead, as in the paper's PTB-XL audit:
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
report = pt.audit_observational_crossfit(embeddings, y, u, folds=folds, clusters=patient_id)
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
### Checks
|
|
127
|
+
|
|
128
|
+
Every report lists validity checks under `report.findings` and in
|
|
129
|
+
`print(report)`: too few records in a `(y, u)` cell, a small effective sample
|
|
130
|
+
or few effective clusters after reweighting, a very large weight, a stressed
|
|
131
|
+
association too weak to test, a head that did not converge, missing clusters,
|
|
132
|
+
and too few bootstrap draws. Checks never change the verdict; those of
|
|
133
|
+
severity `warning` are also raised as `pt.PhysioTraceWarning`. Thresholds live
|
|
134
|
+
in `pt.Gates`, and `pt.Gates.paper()` gives the stricter gates of the paper:
|
|
135
|
+
|
|
136
|
+
```python
|
|
137
|
+
report = pt.audit_observational(train, test, gates=pt.Gates.paper())
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
Common input mistakes fail early with a specific message: a target with more
|
|
141
|
+
than two classes, a provenance factor with more than two groups, a patient in
|
|
142
|
+
two folds or on both sides of the split, or a `(y, u)` combination with no
|
|
143
|
+
records.
|
|
144
|
+
|
|
145
|
+
### Paired views (for example EEG references of the same record)
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
train = pt.PairedSplit(embeddings, y, views, records) # one row per (record, view)
|
|
149
|
+
report = pt.audit_paired(train, test, val=val, p_shortcut=0.6, delta=0.02)
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
The association is induced by a designed coupling at fixed marginals; each
|
|
153
|
+
view serves in turn as the shortcut and `G` averages over them. Provenance is
|
|
154
|
+
localized from within-record view contrasts without task labels.
|
|
155
|
+
`report.details` adds decision flip rates across views and the same-record
|
|
156
|
+
logit contrast.
|
|
157
|
+
|
|
158
|
+
### Building blocks
|
|
159
|
+
|
|
160
|
+
Every step is also available on its own:
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
pt.lpi(train_h, train_u, val_h, val_u, test_h, test_u, test_clusters=subject) # Recover
|
|
164
|
+
tables = pt.observational_regimes(y_train, u_train) # Stress regimes
|
|
165
|
+
w = pt.cell_weights(y_test, u_test, tables["reversed"])
|
|
166
|
+
pt.weighted_auroc(y_test, scores, w)
|
|
167
|
+
subspace, probe = pt.conditional_direction(z_train, y_train, u_train) # Intervene
|
|
168
|
+
subspace.remove(z_test)
|
|
169
|
+
pt.random_subspaces(subspace.whitening, rank=1, n=100, seed=0) # Verify controls
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
`physiotrace.synthetic` generates embeddings with a known shortcut that
|
|
173
|
+
reproduce each of the three verdicts on purpose.
|
|
174
|
+
|
|
175
|
+
## Interpreting the result
|
|
176
|
+
|
|
177
|
+
* A certified remedy supports the evaluated association and the linear
|
|
178
|
+
intervention, not safety under every deployment shift.
|
|
179
|
+
* LPI and accessibility never yield a verdict on their own: a head can ignore
|
|
180
|
+
what a probe decodes.
|
|
181
|
+
* Declare `epsilon`, `delta`, the rank, and the number of random controls
|
|
182
|
+
before looking at test results.
|
|
183
|
+
* Keep every identity (patient, subject, record) inside one fold and pass it
|
|
184
|
+
as `clusters`; resampling windows or trials instead overstates precision.
|
|
185
|
+
|
|
186
|
+
## Validation against the paper
|
|
187
|
+
|
|
188
|
+
[`validation/`](https://github.com/AyanaMussabayeva/physiotrace/blob/main/validation) reproduces the published numbers of the PhysioTRACE paper from
|
|
189
|
+
its saved artifacts:
|
|
190
|
+
|
|
191
|
+
| Script | What is compared | Agreement |
|
|
192
|
+
|---|---|---|
|
|
193
|
+
| `validate_ecg_stress.py` | PTB-XL device audit: `G` before/after removal, its reduction, and the utility change for ECGFounder, CLEF-Small, metadata comparators, sensitivity cohorts, and the weak-association pair, with bootstrap intervals (18 quantities) | ≤ 1e-15 |
|
|
194
|
+
| `validate_ecg_end_to_end.py` | The same audit end to end from ECGFounder embeddings recomputed with the paper's pipeline: `G` before/after removal with intervals, utility change, conditional accessibility, all 100 random-control `G`, four cohorts | ≤ 2e-5 (embeddings recomputed on different hardware) |
|
|
195
|
+
| `validate_nmt_paired.py` | Paired NMT audit end to end from frozen embeddings (8 encoders): `G` before/after removal, all random-control `G`, paired logit contrast, selected `C`, verdict | ≤ 4e-8 |
|
|
196
|
+
| | Calibrated LPI from the paper's exported probe states, including bootstrap intervals | ≤ 4e-8 |
|
|
197
|
+
| | Calibrated LPI with probes refitted by `physiotrace` | ≤ 8e-4 bits (optimizer tolerance) |
|
|
198
|
+
|
|
199
|
+
The paired NMT run whitened with an SVD; pass `whitening="svd"` to draw the
|
|
200
|
+
identical random controls. Both methods give the same projections.
|
|
201
|
+
|
|
202
|
+
## Status
|
|
203
|
+
|
|
204
|
+
Version 0.1 (development). Binary targets with binary provenance
|
|
205
|
+
(observational, target-conditional localization of any rank; rank 1 as in the
|
|
206
|
+
paper) or `K` paired views. Multiclass targets, regression
|
|
207
|
+
endpoints, and multi-encoder pooling are planned.
|
|
208
|
+
|
|
209
|
+
## License
|
|
210
|
+
|
|
211
|
+
MIT.
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# physiotrace
|
|
2
|
+
|
|
3
|
+
**Does your task head rely on how the data were recorded?**
|
|
4
|
+
|
|
5
|
+
`physiotrace` audits a linear task head on a frozen encoder for reliance on
|
|
6
|
+
acquisition provenance — device, site, reference, montage, scanner, batch —
|
|
7
|
+
and certifies a remedy only when it is specific and safe. It works on
|
|
8
|
+
embeddings, so any frozen model (EEG, ECG, imaging, omics) can be audited
|
|
9
|
+
without retraining.
|
|
10
|
+
|
|
11
|
+
Two common measurements do not answer this question. Shifted-transfer
|
|
12
|
+
benchmarks can fail for unrelated reasons, and a provenance probe shows only
|
|
13
|
+
that provenance is decodable, not that a head uses it. PhysioTRACE tests the
|
|
14
|
+
head's behavior directly:
|
|
15
|
+
|
|
16
|
+
| Step | Question | Output |
|
|
17
|
+
|---|---|---|
|
|
18
|
+
| **Recover** | How decodable is provenance from the embedding? | calibrated Linear Provenance Information (bits per unit) or conditional accessibility AUROC |
|
|
19
|
+
| **Stress** | Does a head fitted under the provenance–target association degrade more than a control head when only that association is reversed on the same records? | excess vulnerability `G` with an identity-aware interval |
|
|
20
|
+
| **Intervene** | Does removing a train-localized provenance component remove the failure? | `G` after removal |
|
|
21
|
+
| **Verify** | Does the removal beat rank- and energy-matched random controls, and keep utility within a declared margin `δ`? | verdict |
|
|
22
|
+
|
|
23
|
+
Every audit ends in one of three verdicts: **no reliance**, **reliance – remedy
|
|
24
|
+
certified**, or **reliance – remedy not certified**.
|
|
25
|
+
|
|
26
|
+
**New here? Start with the [tutorial](https://github.com/AyanaMussabayeva/physiotrace/blob/main/docs/tutorial.ipynb)**: bring your own
|
|
27
|
+
embeddings, choose the settings, read the report, and avoid the mistakes that
|
|
28
|
+
make an audit misleading.
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install --pre physiotrace # 0.1.0a1 is a pre-release
|
|
34
|
+
pip install --pre "physiotrace[plot]" # with matplotlib, for the tutorial figures
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Python 3.10–3.13. Tested on every push from the oldest supported dependencies
|
|
38
|
+
(numpy 1.24, scipy 1.10, scikit-learn 1.3, pandas 2.0) to the latest.
|
|
39
|
+
|
|
40
|
+
## Quickstart
|
|
41
|
+
|
|
42
|
+
### Observational data (for example device or site)
|
|
43
|
+
|
|
44
|
+
```python
|
|
45
|
+
import physiotrace as pt
|
|
46
|
+
|
|
47
|
+
train = pt.Split(embeddings, y, u, clusters=patient_id) # u: e.g. device A / device B
|
|
48
|
+
test = pt.Split(test_embeddings, test_y, test_u, clusters=test_patient_id)
|
|
49
|
+
|
|
50
|
+
report = pt.audit_observational(train, test, epsilon=0.01, delta=0.01)
|
|
51
|
+
print(report)
|
|
52
|
+
report.verdict # pt.Verdict.REMEDY_CERTIFIED / REMEDY_NOT_CERTIFIED / NO_RELIANCE
|
|
53
|
+
report.to_dict() # JSON-serializable summary
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
```text
|
|
57
|
+
PhysioTRACE audit (observational)
|
|
58
|
+
verdict: reliance, remedy certified
|
|
59
|
+
Recover conditional_accessibility_auroc = 0.9946
|
|
60
|
+
Stress G = 0.1384 [0.1228, 0.1539] (epsilon = 0.01; 95% interval)
|
|
61
|
+
Intervene G after rank-1 removal = -0.0000 [-0.0013, 0.0012] (100% reduction)
|
|
62
|
+
Verify specific: yes (beats 100/100 random controls)
|
|
63
|
+
safe: yes (control-head AUROC, independent regime change -0.0000 [-0.0004, 0.0003], delta = 0.01)
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
(Output of [`examples/quickstart_synthetic.py`](https://github.com/AyanaMussabayeva/physiotrace/blob/main/examples/quickstart_synthetic.py), which also shows the other two
|
|
67
|
+
verdicts, a cross-fitted audit, and a paired audit.)
|
|
68
|
+
|
|
69
|
+
`y` and `u` can hold any two labels each (strings or numbers). `clusters`
|
|
70
|
+
identifies who each record belongs to (patient, subject); it is the bootstrap
|
|
71
|
+
unit, and a cluster on both sides of the split is rejected. `epsilon` is the
|
|
72
|
+
smallest `G` that counts as reliance and `delta` the tolerated utility loss;
|
|
73
|
+
declare both, and the localization `rank`, before looking at test results.
|
|
74
|
+
|
|
75
|
+
The exposed head is fitted on the natural association and the control head
|
|
76
|
+
with independence weights. Provenance is localized conditionally on the
|
|
77
|
+
target, so removal does not by default discard population signal that sources
|
|
78
|
+
differ in.
|
|
79
|
+
|
|
80
|
+
To cross-fit over folds instead, as in the paper's PTB-XL audit:
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
report = pt.audit_observational_crossfit(embeddings, y, u, folds=folds, clusters=patient_id)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
### Checks
|
|
87
|
+
|
|
88
|
+
Every report lists validity checks under `report.findings` and in
|
|
89
|
+
`print(report)`: too few records in a `(y, u)` cell, a small effective sample
|
|
90
|
+
or few effective clusters after reweighting, a very large weight, a stressed
|
|
91
|
+
association too weak to test, a head that did not converge, missing clusters,
|
|
92
|
+
and too few bootstrap draws. Checks never change the verdict; those of
|
|
93
|
+
severity `warning` are also raised as `pt.PhysioTraceWarning`. Thresholds live
|
|
94
|
+
in `pt.Gates`, and `pt.Gates.paper()` gives the stricter gates of the paper:
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
report = pt.audit_observational(train, test, gates=pt.Gates.paper())
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
Common input mistakes fail early with a specific message: a target with more
|
|
101
|
+
than two classes, a provenance factor with more than two groups, a patient in
|
|
102
|
+
two folds or on both sides of the split, or a `(y, u)` combination with no
|
|
103
|
+
records.
|
|
104
|
+
|
|
105
|
+
### Paired views (for example EEG references of the same record)
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
train = pt.PairedSplit(embeddings, y, views, records) # one row per (record, view)
|
|
109
|
+
report = pt.audit_paired(train, test, val=val, p_shortcut=0.6, delta=0.02)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The association is induced by a designed coupling at fixed marginals; each
|
|
113
|
+
view serves in turn as the shortcut and `G` averages over them. Provenance is
|
|
114
|
+
localized from within-record view contrasts without task labels.
|
|
115
|
+
`report.details` adds decision flip rates across views and the same-record
|
|
116
|
+
logit contrast.
|
|
117
|
+
|
|
118
|
+
### Building blocks
|
|
119
|
+
|
|
120
|
+
Every step is also available on its own:
|
|
121
|
+
|
|
122
|
+
```python
|
|
123
|
+
pt.lpi(train_h, train_u, val_h, val_u, test_h, test_u, test_clusters=subject) # Recover
|
|
124
|
+
tables = pt.observational_regimes(y_train, u_train) # Stress regimes
|
|
125
|
+
w = pt.cell_weights(y_test, u_test, tables["reversed"])
|
|
126
|
+
pt.weighted_auroc(y_test, scores, w)
|
|
127
|
+
subspace, probe = pt.conditional_direction(z_train, y_train, u_train) # Intervene
|
|
128
|
+
subspace.remove(z_test)
|
|
129
|
+
pt.random_subspaces(subspace.whitening, rank=1, n=100, seed=0) # Verify controls
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
`physiotrace.synthetic` generates embeddings with a known shortcut that
|
|
133
|
+
reproduce each of the three verdicts on purpose.
|
|
134
|
+
|
|
135
|
+
## Interpreting the result
|
|
136
|
+
|
|
137
|
+
* A certified remedy supports the evaluated association and the linear
|
|
138
|
+
intervention, not safety under every deployment shift.
|
|
139
|
+
* LPI and accessibility never yield a verdict on their own: a head can ignore
|
|
140
|
+
what a probe decodes.
|
|
141
|
+
* Declare `epsilon`, `delta`, the rank, and the number of random controls
|
|
142
|
+
before looking at test results.
|
|
143
|
+
* Keep every identity (patient, subject, record) inside one fold and pass it
|
|
144
|
+
as `clusters`; resampling windows or trials instead overstates precision.
|
|
145
|
+
|
|
146
|
+
## Validation against the paper
|
|
147
|
+
|
|
148
|
+
[`validation/`](https://github.com/AyanaMussabayeva/physiotrace/blob/main/validation) reproduces the published numbers of the PhysioTRACE paper from
|
|
149
|
+
its saved artifacts:
|
|
150
|
+
|
|
151
|
+
| Script | What is compared | Agreement |
|
|
152
|
+
|---|---|---|
|
|
153
|
+
| `validate_ecg_stress.py` | PTB-XL device audit: `G` before/after removal, its reduction, and the utility change for ECGFounder, CLEF-Small, metadata comparators, sensitivity cohorts, and the weak-association pair, with bootstrap intervals (18 quantities) | ≤ 1e-15 |
|
|
154
|
+
| `validate_ecg_end_to_end.py` | The same audit end to end from ECGFounder embeddings recomputed with the paper's pipeline: `G` before/after removal with intervals, utility change, conditional accessibility, all 100 random-control `G`, four cohorts | ≤ 2e-5 (embeddings recomputed on different hardware) |
|
|
155
|
+
| `validate_nmt_paired.py` | Paired NMT audit end to end from frozen embeddings (8 encoders): `G` before/after removal, all random-control `G`, paired logit contrast, selected `C`, verdict | ≤ 4e-8 |
|
|
156
|
+
| | Calibrated LPI from the paper's exported probe states, including bootstrap intervals | ≤ 4e-8 |
|
|
157
|
+
| | Calibrated LPI with probes refitted by `physiotrace` | ≤ 8e-4 bits (optimizer tolerance) |
|
|
158
|
+
|
|
159
|
+
The paired NMT run whitened with an SVD; pass `whitening="svd"` to draw the
|
|
160
|
+
identical random controls. Both methods give the same projections.
|
|
161
|
+
|
|
162
|
+
## Status
|
|
163
|
+
|
|
164
|
+
Version 0.1 (development). Binary targets with binary provenance
|
|
165
|
+
(observational, target-conditional localization of any rank; rank 1 as in the
|
|
166
|
+
paper) or `K` paired views. Multiclass targets, regression
|
|
167
|
+
endpoints, and multi-encoder pooling are planned.
|
|
168
|
+
|
|
169
|
+
## License
|
|
170
|
+
|
|
171
|
+
MIT.
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "physiotrace"
|
|
7
|
+
version = "0.1.0a1"
|
|
8
|
+
description = "Behavioral audit of provenance reliance in frozen-encoder task heads (Recover, Stress, Intervene, Verify)."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
authors = [
|
|
12
|
+
{name = "Ayana Mussabayeva", email = "ayana.mussabayeva@gmail.com"},
|
|
13
|
+
{name = "Anuar Aimoldin"},
|
|
14
|
+
]
|
|
15
|
+
keywords = ["foundation models", "shortcut learning", "provenance", "EEG", "ECG", "robustness", "audit"]
|
|
16
|
+
license = "MIT"
|
|
17
|
+
license-files = ["LICENSE"]
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Development Status :: 3 - Alpha",
|
|
20
|
+
"Intended Audience :: Science/Research",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.10",
|
|
23
|
+
"Programming Language :: Python :: 3.11",
|
|
24
|
+
"Programming Language :: Python :: 3.12",
|
|
25
|
+
"Programming Language :: Python :: 3.13",
|
|
26
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
27
|
+
"Topic :: Scientific/Engineering :: Medical Science Apps.",
|
|
28
|
+
]
|
|
29
|
+
dependencies = [
|
|
30
|
+
"numpy>=1.24",
|
|
31
|
+
"scipy>=1.10",
|
|
32
|
+
"scikit-learn>=1.3",
|
|
33
|
+
"pandas>=2.0",
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
plot = ["matplotlib>=3.7"]
|
|
38
|
+
dev = ["pytest>=7"]
|
|
39
|
+
docs = ["matplotlib>=3.7", "nbformat>=5", "nbclient>=0.7", "ipykernel>=6"]
|
|
40
|
+
|
|
41
|
+
[project.urls]
|
|
42
|
+
Homepage = "https://github.com/AyanaMussabayeva/physiotrace"
|
|
43
|
+
Documentation = "https://github.com/AyanaMussabayeva/physiotrace/blob/main/docs/tutorial.ipynb"
|
|
44
|
+
Source = "https://github.com/AyanaMussabayeva/physiotrace"
|
|
45
|
+
Issues = "https://github.com/AyanaMussabayeva/physiotrace/issues"
|
|
46
|
+
Changelog = "https://github.com/AyanaMussabayeva/physiotrace/blob/main/CHANGELOG.md"
|
|
47
|
+
|
|
48
|
+
[tool.setuptools.packages.find]
|
|
49
|
+
where = ["src"]
|
|
50
|
+
|
|
51
|
+
[tool.pytest.ini_options]
|
|
52
|
+
testpaths = ["tests"]
|
|
53
|
+
# Small synthetic audits legitimately trip the validity checks; tests that
|
|
54
|
+
# assert on warnings use pytest.warns.
|
|
55
|
+
filterwarnings = ["ignore::physiotrace.diagnostics.PhysioTraceWarning"]
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"""PhysioTRACE: behavioral audit of provenance reliance in frozen-encoder task heads.
|
|
2
|
+
|
|
3
|
+
Recover how decodable provenance is, Stress fixed heads by reversing only the
|
|
4
|
+
provenance--target association, Intervene by removing a train-localized
|
|
5
|
+
provenance component, and Verify the remedy against matched random controls
|
|
6
|
+
within a declared utility margin.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from . import bootstrap, intervene, metrics, recover, stress, synthetic, weights
|
|
10
|
+
from .audit import PairedSplit, Split, audit_observational, audit_observational_crossfit, audit_paired
|
|
11
|
+
from .diagnostics import Finding, Gates, PhysioTraceWarning
|
|
12
|
+
from .intervene import Subspace, Whitening, conditional_direction, paired_subspace, random_subspaces
|
|
13
|
+
from .metrics import weighted_auroc
|
|
14
|
+
from .recover import LPIResult, fit_calibrated_probe, lpi
|
|
15
|
+
from .report import AuditReport
|
|
16
|
+
from .stress import excess_vulnerability, regime_aurocs
|
|
17
|
+
from .verify import Verdict
|
|
18
|
+
from .weights import (
|
|
19
|
+
cell_weights,
|
|
20
|
+
designed_coupling_table,
|
|
21
|
+
fixed_marginal_table,
|
|
22
|
+
independence_weights,
|
|
23
|
+
observational_regimes,
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
try:
|
|
27
|
+
from importlib.metadata import PackageNotFoundError, version as _version
|
|
28
|
+
|
|
29
|
+
__version__ = _version("physiotrace")
|
|
30
|
+
except PackageNotFoundError: # running from a source tree without installation
|
|
31
|
+
__version__ = "0+unknown"
|
|
32
|
+
|
|
33
|
+
__all__ = [
|
|
34
|
+
"AuditReport",
|
|
35
|
+
"Finding",
|
|
36
|
+
"Gates",
|
|
37
|
+
"PhysioTraceWarning",
|
|
38
|
+
"Split",
|
|
39
|
+
"audit_observational_crossfit",
|
|
40
|
+
"LPIResult",
|
|
41
|
+
"PairedSplit",
|
|
42
|
+
"Subspace",
|
|
43
|
+
"Verdict",
|
|
44
|
+
"Whitening",
|
|
45
|
+
"audit_observational",
|
|
46
|
+
"audit_paired",
|
|
47
|
+
"bootstrap",
|
|
48
|
+
"cell_weights",
|
|
49
|
+
"conditional_direction",
|
|
50
|
+
"designed_coupling_table",
|
|
51
|
+
"excess_vulnerability",
|
|
52
|
+
"fit_calibrated_probe",
|
|
53
|
+
"fixed_marginal_table",
|
|
54
|
+
"independence_weights",
|
|
55
|
+
"intervene",
|
|
56
|
+
"lpi",
|
|
57
|
+
"metrics",
|
|
58
|
+
"observational_regimes",
|
|
59
|
+
"paired_subspace",
|
|
60
|
+
"random_subspaces",
|
|
61
|
+
"recover",
|
|
62
|
+
"regime_aurocs",
|
|
63
|
+
"stress",
|
|
64
|
+
"synthetic",
|
|
65
|
+
"weighted_auroc",
|
|
66
|
+
"weights",
|
|
67
|
+
]
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"""Small input checks shared across modules."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def as_matrix(values, *, name: str) -> np.ndarray:
|
|
9
|
+
array = np.asarray(values, dtype=np.float64)
|
|
10
|
+
if array.ndim != 2:
|
|
11
|
+
raise ValueError(f"{name} must be a two-dimensional array, got shape {array.shape}.")
|
|
12
|
+
if not np.isfinite(array).all():
|
|
13
|
+
raise ValueError(f"{name} contains non-finite values.")
|
|
14
|
+
return array
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def as_codes(values, *, name: str, n_levels: int | None = None) -> np.ndarray:
|
|
18
|
+
"""Return non-negative integer codes; values must already be encoded."""
|
|
19
|
+
|
|
20
|
+
array = np.asarray(values)
|
|
21
|
+
if array.ndim != 1:
|
|
22
|
+
raise ValueError(f"{name} must be one-dimensional.")
|
|
23
|
+
if not np.issubdtype(array.dtype, np.integer):
|
|
24
|
+
if not np.issubdtype(array.dtype, np.number) or not np.all(np.equal(array, np.floor(array))):
|
|
25
|
+
raise ValueError(f"{name} must contain integer codes.")
|
|
26
|
+
codes = array.astype(np.int64)
|
|
27
|
+
if np.any(codes < 0):
|
|
28
|
+
raise ValueError(f"{name} must contain non-negative codes.")
|
|
29
|
+
if n_levels is not None and np.any(codes >= n_levels):
|
|
30
|
+
raise ValueError(f"{name} contains a code outside [0, {n_levels}).")
|
|
31
|
+
return codes
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def encode(values, levels=None) -> tuple[np.ndarray, tuple]:
|
|
35
|
+
"""Encode arbitrary labels as integer codes in the order of ``levels``."""
|
|
36
|
+
|
|
37
|
+
array = np.asarray(values)
|
|
38
|
+
if levels is None:
|
|
39
|
+
levels = tuple(np.unique(array).tolist())
|
|
40
|
+
else:
|
|
41
|
+
levels = tuple(levels)
|
|
42
|
+
lookup = {level: code for code, level in enumerate(levels)}
|
|
43
|
+
try:
|
|
44
|
+
codes = np.fromiter((lookup[value] for value in array.tolist()), dtype=np.int64, count=len(array))
|
|
45
|
+
except KeyError as error:
|
|
46
|
+
raise ValueError(f"Value {error.args[0]!r} is not among the declared levels {levels}.") from None
|
|
47
|
+
return codes, levels
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def same_length(**arrays) -> int:
|
|
51
|
+
lengths = {name: len(value) for name, value in arrays.items()}
|
|
52
|
+
if len(set(lengths.values())) != 1:
|
|
53
|
+
raise ValueError(f"Inputs must have the same length, got {lengths}.")
|
|
54
|
+
return next(iter(lengths.values()))
|