icuflow 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- icuflow-0.1.0/PKG-INFO +129 -0
- icuflow-0.1.0/README.md +107 -0
- icuflow-0.1.0/icuflow/__init__.py +30 -0
- icuflow-0.1.0/icuflow/alarms.py +56 -0
- icuflow-0.1.0/icuflow/calibration.py +101 -0
- icuflow-0.1.0/icuflow/cli.py +73 -0
- icuflow-0.1.0/icuflow/splits.py +65 -0
- icuflow-0.1.0/icuflow/tasks.py +53 -0
- icuflow-0.1.0/icuflow.egg-info/PKG-INFO +129 -0
- icuflow-0.1.0/icuflow.egg-info/SOURCES.txt +16 -0
- icuflow-0.1.0/icuflow.egg-info/dependency_links.txt +1 -0
- icuflow-0.1.0/icuflow.egg-info/entry_points.txt +2 -0
- icuflow-0.1.0/icuflow.egg-info/requires.txt +8 -0
- icuflow-0.1.0/icuflow.egg-info/top_level.txt +1 -0
- icuflow-0.1.0/pyproject.toml +34 -0
- icuflow-0.1.0/setup.cfg +4 -0
- icuflow-0.1.0/tests/test_data_layer.py +170 -0
- icuflow-0.1.0/tests/test_icuflow.py +163 -0
icuflow-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: icuflow
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Leak-safe cohort splitting, calibration and alarm metrics for clinical prediction, in the shape PyHealth and RHealth tasks expect.
|
|
5
|
+
Author: Wenke Du
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Source, https://github.com/evelyyyyynnnnn/2.0-Healthcare-Ai-Systems
|
|
8
|
+
Keywords: clinical,icu,pyhealth,calibration,early-warning
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
Requires-Dist: numpy>=1.24
|
|
17
|
+
Provides-Extra: pyhealth
|
|
18
|
+
Requires-Dist: pyhealth>=1.1.4; extra == "pyhealth"
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
21
|
+
Requires-Dist: scikit-learn>=1.3; extra == "dev"
|
|
22
|
+
|
|
23
|
+
# icuflow
|
|
24
|
+
|
|
25
|
+
> An installable package for the three parts of a clinical prediction pipeline that are easiest to get wrong: subject-grouped splits, calibration, and alarm comparison at matched sensitivity.
|
|
26
|
+
|
|
27
|
+
**Repository:** `2.0-Healthcare-Ai-Systems` · **Pillar:** Healthcare Safety
|
|
28
|
+
|
|
29
|
+
## Status
|
|
30
|
+
|
|
31
|
+
Working code, **29 passing tests**, and one measured run on real ICU charting.
|
|
32
|
+
Earlier versions of this page said the experiments ran on synthetic subjects;
|
|
33
|
+
the reportable run does not.
|
|
34
|
+
|
|
35
|
+
`icuflow` 0.1.0 is an installable package with 14 exports and a working CLI
|
|
36
|
+
(`check-split`, `score`). It is **not published to PyPI** — `published` is
|
|
37
|
+
`false` in the results file — so it has no download count and no users, and
|
|
38
|
+
neither is claimed.
|
|
39
|
+
|
|
40
|
+
**What it measures.** Using the **PhysioNet MIMIC-IV open-access demo**, the same cohort, features and model are split two ways. A random **row-level** split reports **AUROC 0.9058**. A **subject-grouped** split reports **0.4989** — chance. The reported figure is inflated by **81.57%**, and the honest one says the model learned nothing.
|
|
41
|
+
|
|
42
|
+
Cohort: 97 subjects, 134 stays, 37 readmissions, 22,512 rows, positive rate
|
|
43
|
+
0.02838. about 100 ICU stays. The direction of the leakage effect is
|
|
44
|
+
meaningful; its magnitude, from a cohort this small, is not a population
|
|
45
|
+
estimate.
|
|
46
|
+
|
|
47
|
+
Last reportable run: `2026-09-06T23:57:16+00:00`
|
|
48
|
+
|
|
49
|
+
## Quick start
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install -r requirements.txt
|
|
53
|
+
python -m pytest tests/ -q # 29 tests
|
|
54
|
+
python -m data.fetch # needs the sources in docs/DATA.md
|
|
55
|
+
python -m src.demo --real # the measured run -> results/latest-real.json
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Layout
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
README.md
|
|
62
|
+
data/
|
|
63
|
+
|-- README.md
|
|
64
|
+
|-- manifests/
|
|
65
|
+
|-- sample/
|
|
66
|
+
docs/
|
|
67
|
+
|-- DATA.md
|
|
68
|
+
|-- METHOD.md
|
|
69
|
+
icuflow/
|
|
70
|
+
|-- __init__.py
|
|
71
|
+
|-- alarms.py
|
|
72
|
+
|-- calibration.py
|
|
73
|
+
|-- cli.py
|
|
74
|
+
|-- splits.py
|
|
75
|
+
|-- tasks.py
|
|
76
|
+
icuflow.egg-info/
|
|
77
|
+
|-- PKG-INFO
|
|
78
|
+
|-- SOURCES.txt
|
|
79
|
+
|-- dependency_links.txt
|
|
80
|
+
|-- entry_points.txt
|
|
81
|
+
|-- requires.txt
|
|
82
|
+
|-- top_level.txt
|
|
83
|
+
pyproject.toml
|
|
84
|
+
requirements.txt
|
|
85
|
+
results/
|
|
86
|
+
|-- README.md
|
|
87
|
+
|-- latest.json
|
|
88
|
+
src/
|
|
89
|
+
|-- .gitkeep
|
|
90
|
+
|-- __init__.py
|
|
91
|
+
|-- demo.py
|
|
92
|
+
|-- site.py
|
|
93
|
+
|-- sitekit.py
|
|
94
|
+
tests/
|
|
95
|
+
|-- .gitkeep
|
|
96
|
+
|-- test_icuflow.py
|
|
97
|
+
website/
|
|
98
|
+
|-- README.md
|
|
99
|
+
|-- index.html
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
- `src/` — the implementation.
|
|
103
|
+
- `tests/` — pytest suite. These guard behaviour, not just imports.
|
|
104
|
+
- `results/latest.json` — the output of the last demo run. Every figure quoted
|
|
105
|
+
anywhere in this project traces back to this file.
|
|
106
|
+
- `website/` — a self-contained static site, deployable to Vercel by copying the
|
|
107
|
+
folder into its own repository. See `website/README.md`.
|
|
108
|
+
|
|
109
|
+
## The website
|
|
110
|
+
|
|
111
|
+
`website/` has no build step. To deploy it independently:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
cp -r website/ ../my-4-pyhealth-rhealth-extension-site && cd ../my-4-pyhealth-rhealth-extension-site
|
|
115
|
+
git init && git add -A && git commit -m "site"
|
|
116
|
+
vercel deploy --prod
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
The page is regenerated from `results.json` on every `python -m src.demo`, so the
|
|
120
|
+
figures on the site and the figures the code produces cannot drift apart. Do not edit
|
|
121
|
+
numbers on the page by hand.
|
|
122
|
+
|
|
123
|
+
## Honesty note
|
|
124
|
+
|
|
125
|
+
Everything in this project runs on clearly-labelled synthetic or authored data.
|
|
126
|
+
Swap in the real source and the same pipeline reports real numbers — that is
|
|
127
|
+
what the structure is for. Until that happens, nothing here should be cited as a
|
|
128
|
+
measured result, and the site's closing section states explicitly what the project
|
|
129
|
+
does not establish.
|
icuflow-0.1.0/README.md
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# icuflow
|
|
2
|
+
|
|
3
|
+
> An installable package for the three parts of a clinical prediction pipeline that are easiest to get wrong: subject-grouped splits, calibration, and alarm comparison at matched sensitivity.
|
|
4
|
+
|
|
5
|
+
**Repository:** `2.0-Healthcare-Ai-Systems` · **Pillar:** Healthcare Safety
|
|
6
|
+
|
|
7
|
+
## Status
|
|
8
|
+
|
|
9
|
+
Working code, **29 passing tests**, and one measured run on real ICU charting.
|
|
10
|
+
Earlier versions of this page said the experiments ran on synthetic subjects;
|
|
11
|
+
the reportable run does not.
|
|
12
|
+
|
|
13
|
+
`icuflow` 0.1.0 is an installable package with 14 exports and a working CLI
|
|
14
|
+
(`check-split`, `score`). It is **not published to PyPI** — `published` is
|
|
15
|
+
`false` in the results file — so it has no download count and no users, and
|
|
16
|
+
neither is claimed.
|
|
17
|
+
|
|
18
|
+
**What it measures.** Using the **PhysioNet MIMIC-IV open-access demo**, the same cohort, features and model are split two ways. A random **row-level** split reports **AUROC 0.9058**. A **subject-grouped** split reports **0.4989** — chance. The reported figure is inflated by **81.57%**, and the honest one says the model learned nothing.
|
|
19
|
+
|
|
20
|
+
Cohort: 97 subjects, 134 stays, 37 readmissions, 22,512 rows, positive rate
|
|
21
|
+
0.02838. about 100 ICU stays. The direction of the leakage effect is
|
|
22
|
+
meaningful; its magnitude, from a cohort this small, is not a population
|
|
23
|
+
estimate.
|
|
24
|
+
|
|
25
|
+
Last reportable run: `2026-09-06T23:57:16+00:00`
|
|
26
|
+
|
|
27
|
+
## Quick start
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
pip install -r requirements.txt
|
|
31
|
+
python -m pytest tests/ -q # 29 tests
|
|
32
|
+
python -m data.fetch # needs the sources in docs/DATA.md
|
|
33
|
+
python -m src.demo --real # the measured run -> results/latest-real.json
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## Layout
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
README.md
|
|
40
|
+
data/
|
|
41
|
+
|-- README.md
|
|
42
|
+
|-- manifests/
|
|
43
|
+
|-- sample/
|
|
44
|
+
docs/
|
|
45
|
+
|-- DATA.md
|
|
46
|
+
|-- METHOD.md
|
|
47
|
+
icuflow/
|
|
48
|
+
|-- __init__.py
|
|
49
|
+
|-- alarms.py
|
|
50
|
+
|-- calibration.py
|
|
51
|
+
|-- cli.py
|
|
52
|
+
|-- splits.py
|
|
53
|
+
|-- tasks.py
|
|
54
|
+
icuflow.egg-info/
|
|
55
|
+
|-- PKG-INFO
|
|
56
|
+
|-- SOURCES.txt
|
|
57
|
+
|-- dependency_links.txt
|
|
58
|
+
|-- entry_points.txt
|
|
59
|
+
|-- requires.txt
|
|
60
|
+
|-- top_level.txt
|
|
61
|
+
pyproject.toml
|
|
62
|
+
requirements.txt
|
|
63
|
+
results/
|
|
64
|
+
|-- README.md
|
|
65
|
+
|-- latest.json
|
|
66
|
+
src/
|
|
67
|
+
|-- .gitkeep
|
|
68
|
+
|-- __init__.py
|
|
69
|
+
|-- demo.py
|
|
70
|
+
|-- site.py
|
|
71
|
+
|-- sitekit.py
|
|
72
|
+
tests/
|
|
73
|
+
|-- .gitkeep
|
|
74
|
+
|-- test_icuflow.py
|
|
75
|
+
website/
|
|
76
|
+
|-- README.md
|
|
77
|
+
|-- index.html
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
- `src/` — the implementation.
|
|
81
|
+
- `tests/` — pytest suite. These guard behaviour, not just imports.
|
|
82
|
+
- `results/latest.json` — the output of the last demo run. Every figure quoted
|
|
83
|
+
anywhere in this project traces back to this file.
|
|
84
|
+
- `website/` — a self-contained static site, deployable to Vercel by copying the
|
|
85
|
+
folder into its own repository. See `website/README.md`.
|
|
86
|
+
|
|
87
|
+
## The website
|
|
88
|
+
|
|
89
|
+
`website/` has no build step. To deploy it independently:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
cp -r website/ ../my-4-pyhealth-rhealth-extension-site && cd ../my-4-pyhealth-rhealth-extension-site
|
|
93
|
+
git init && git add -A && git commit -m "site"
|
|
94
|
+
vercel deploy --prod
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
The page is regenerated from `results.json` on every `python -m src.demo`, so the
|
|
98
|
+
figures on the site and the figures the code produces cannot drift apart. Do not edit
|
|
99
|
+
numbers on the page by hand.
|
|
100
|
+
|
|
101
|
+
## Honesty note
|
|
102
|
+
|
|
103
|
+
Everything in this project runs on clearly-labelled synthetic or authored data.
|
|
104
|
+
Swap in the real source and the same pipeline reports real numbers — that is
|
|
105
|
+
what the structure is for. Until that happens, nothing here should be cited as a
|
|
106
|
+
measured result, and the site's closing section states explicitly what the project
|
|
107
|
+
does not establish.
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""icuflow — the parts of a clinical prediction pipeline that are easy to get wrong.
|
|
2
|
+
|
|
3
|
+
Three of them, packaged because they are re-implemented badly in most clinical
|
|
4
|
+
prediction code:
|
|
5
|
+
|
|
6
|
+
splits grouping by patient, so a subject never spans train and test
|
|
7
|
+
calibration turning a score into a probability that means what it says
|
|
8
|
+
alarms comparing alerting systems at matched sensitivity
|
|
9
|
+
|
|
10
|
+
Designed to sit alongside PyHealth and RHealth rather than replace them: the
|
|
11
|
+
functions take and return plain arrays, so they drop into an existing task
|
|
12
|
+
pipeline without adopting a framework.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from .splits import (GroupSplit, group_kfold, split_by_subject,
|
|
16
|
+
assert_no_subject_leak)
|
|
17
|
+
from .calibration import (IsotonicCalibrator, PlattCalibrator, ece,
|
|
18
|
+
reliability_table)
|
|
19
|
+
from .alarms import (alarm_stats, threshold_at_sensitivity,
|
|
20
|
+
compare_at_matched_sensitivity)
|
|
21
|
+
from .tasks import horizon_label, sliding_windows
|
|
22
|
+
|
|
23
|
+
__version__ = "0.1.0"
|
|
24
|
+
|
|
25
|
+
__all__ = [
|
|
26
|
+
"GroupSplit", "group_kfold", "split_by_subject", "assert_no_subject_leak",
|
|
27
|
+
"IsotonicCalibrator", "PlattCalibrator", "ece", "reliability_table",
|
|
28
|
+
"alarm_stats", "threshold_at_sensitivity", "compare_at_matched_sensitivity",
|
|
29
|
+
"horizon_label", "sliding_windows", "__version__",
|
|
30
|
+
]
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
"""Alarm-burden comparison at matched sensitivity.
|
|
2
|
+
|
|
3
|
+
Any alerting system can cut false alarms by catching fewer events. Comparing
|
|
4
|
+
two systems at whatever operating point each happens to sit on is the most
|
|
5
|
+
common way an alerting result is made to look better than it is, so this module
|
|
6
|
+
only compares at a sensitivity you have to name.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def threshold_at_sensitivity(y, scores, target: float = 0.80) -> float:
|
|
15
|
+
y = np.asarray(y)
|
|
16
|
+
s = np.asarray(scores, float)
|
|
17
|
+
pos = s[y == 1]
|
|
18
|
+
if len(pos) == 0:
|
|
19
|
+
return float("nan")
|
|
20
|
+
return float(np.quantile(pos, 1.0 - target))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def alarm_stats(y, scores, threshold: float) -> dict:
|
|
24
|
+
y = np.asarray(y)
|
|
25
|
+
s = np.asarray(scores, float)
|
|
26
|
+
pred = s >= threshold
|
|
27
|
+
tp = int(np.sum(pred & (y == 1)))
|
|
28
|
+
fp = int(np.sum(pred & (y == 0)))
|
|
29
|
+
fn = int(np.sum(~pred & (y == 1)))
|
|
30
|
+
tn = int(np.sum(~pred & (y == 0)))
|
|
31
|
+
return {
|
|
32
|
+
"threshold": float(threshold),
|
|
33
|
+
"tp": tp, "fp": fp, "fn": fn, "tn": tn,
|
|
34
|
+
"sensitivity": tp / (tp + fn) if (tp + fn) else 0.0,
|
|
35
|
+
"specificity": tn / (tn + fp) if (tn + fp) else 0.0,
|
|
36
|
+
"ppv": tp / (tp + fp) if (tp + fp) else 0.0,
|
|
37
|
+
"false_alerts": fp,
|
|
38
|
+
"false_alerts_per_100": 100.0 * fp / len(y) if len(y) else 0.0,
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def compare_at_matched_sensitivity(y, baseline_scores, model_scores,
|
|
43
|
+
target: float = 0.80) -> dict:
|
|
44
|
+
"""Set both systems to the same sensitivity, then count false alerts."""
|
|
45
|
+
b = alarm_stats(y, baseline_scores,
|
|
46
|
+
threshold_at_sensitivity(y, baseline_scores, target))
|
|
47
|
+
m = alarm_stats(y, model_scores,
|
|
48
|
+
threshold_at_sensitivity(y, model_scores, target))
|
|
49
|
+
denom = b["false_alerts"] or 1
|
|
50
|
+
return {
|
|
51
|
+
"target_sensitivity": target,
|
|
52
|
+
"baseline": b,
|
|
53
|
+
"model": m,
|
|
54
|
+
"false_alert_reduction_pct": 100.0 * (b["false_alerts"] - m["false_alerts"]) / denom,
|
|
55
|
+
"sensitivity_gap": m["sensitivity"] - b["sensitivity"],
|
|
56
|
+
}
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
"""Calibration: making a score mean what it says.
|
|
2
|
+
|
|
3
|
+
A model can rank patients correctly and still emit numbers that are not
|
|
4
|
+
probabilities. A clinician acting on "82% risk" needs that to be right 82% of
|
|
5
|
+
the time, so calibration is reported beside discrimination, never instead of it.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class PlattCalibrator:
|
|
14
|
+
"""Logistic (Platt) scaling. Parametric, data-efficient, assumes sigmoid."""
|
|
15
|
+
|
|
16
|
+
def __init__(self, max_iter: int = 200, lr: float = 0.1):
|
|
17
|
+
self.a, self.b = 1.0, 0.0
|
|
18
|
+
self.max_iter, self.lr = max_iter, lr
|
|
19
|
+
|
|
20
|
+
def fit(self, scores, y):
|
|
21
|
+
s = np.asarray(scores, float)
|
|
22
|
+
y = np.asarray(y, float)
|
|
23
|
+
s = (s - s.mean()) / (s.std() + 1e-12)
|
|
24
|
+
self._mu, self._sd = float(np.mean(scores)), float(np.std(scores) + 1e-12)
|
|
25
|
+
for _ in range(self.max_iter):
|
|
26
|
+
p = 1 / (1 + np.exp(-(self.a * s + self.b)))
|
|
27
|
+
ga = float(np.mean((p - y) * s))
|
|
28
|
+
gb = float(np.mean(p - y))
|
|
29
|
+
self.a -= self.lr * ga
|
|
30
|
+
self.b -= self.lr * gb
|
|
31
|
+
return self
|
|
32
|
+
|
|
33
|
+
def transform(self, scores):
|
|
34
|
+
s = (np.asarray(scores, float) - self._mu) / self._sd
|
|
35
|
+
return 1 / (1 + np.exp(-(self.a * s + self.b)))
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class IsotonicCalibrator:
|
|
39
|
+
"""Isotonic regression via pool-adjacent-violators.
|
|
40
|
+
|
|
41
|
+
Non-parametric, so it fixes the S-shaped miscalibration boosted trees
|
|
42
|
+
reliably produce. Costs more data than Platt, which is the trade.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(self):
|
|
46
|
+
self.x_ = None
|
|
47
|
+
self.y_ = None
|
|
48
|
+
|
|
49
|
+
def fit(self, scores, y):
|
|
50
|
+
s = np.asarray(scores, float)
|
|
51
|
+
t = np.asarray(y, float)
|
|
52
|
+
order = np.argsort(s, kind="mergesort")
|
|
53
|
+
s, t = s[order], t[order]
|
|
54
|
+
|
|
55
|
+
# PAVA: merge adjacent blocks until the fit is non-decreasing.
|
|
56
|
+
vals = list(t)
|
|
57
|
+
weights = [1.0] * len(t)
|
|
58
|
+
i = 0
|
|
59
|
+
while i < len(vals) - 1:
|
|
60
|
+
if vals[i] <= vals[i + 1] + 1e-12:
|
|
61
|
+
i += 1
|
|
62
|
+
continue
|
|
63
|
+
w = weights[i] + weights[i + 1]
|
|
64
|
+
v = (vals[i] * weights[i] + vals[i + 1] * weights[i + 1]) / w
|
|
65
|
+
vals[i:i + 2] = [v]
|
|
66
|
+
weights[i:i + 2] = [w]
|
|
67
|
+
if i > 0:
|
|
68
|
+
i -= 1
|
|
69
|
+
fitted, k = [], 0
|
|
70
|
+
for v, w in zip(vals, weights):
|
|
71
|
+
fitted.extend([v] * int(round(w)))
|
|
72
|
+
self.x_ = s
|
|
73
|
+
self.y_ = np.asarray(fitted[:len(s)], float)
|
|
74
|
+
return self
|
|
75
|
+
|
|
76
|
+
def transform(self, scores):
|
|
77
|
+
if self.x_ is None:
|
|
78
|
+
raise RuntimeError("fit before transform")
|
|
79
|
+
return np.interp(np.asarray(scores, float), self.x_, self.y_)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def reliability_table(y, p, bins: int = 10) -> list:
|
|
83
|
+
y = np.asarray(y, float)
|
|
84
|
+
p = np.asarray(p, float)
|
|
85
|
+
edges = np.linspace(0, 1, bins + 1)
|
|
86
|
+
out = []
|
|
87
|
+
for i in range(bins):
|
|
88
|
+
lo, hi = edges[i], edges[i + 1]
|
|
89
|
+
m = (p >= lo) & (p < hi if i < bins - 1 else p <= hi)
|
|
90
|
+
if m.any():
|
|
91
|
+
out.append({"bin_lo": float(lo), "bin_hi": float(hi),
|
|
92
|
+
"n": int(m.sum()),
|
|
93
|
+
"predicted": float(p[m].mean()),
|
|
94
|
+
"observed": float(y[m].mean())})
|
|
95
|
+
return out
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def ece(y, p, bins: int = 10) -> float:
|
|
99
|
+
rows = reliability_table(y, p, bins)
|
|
100
|
+
n = len(np.asarray(y))
|
|
101
|
+
return float(sum(r["n"] / n * abs(r["predicted"] - r["observed"]) for r in rows))
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""icuflow command line: check a split, or score a set of predictions."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import csv
|
|
7
|
+
import json
|
|
8
|
+
import sys
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
from .alarms import compare_at_matched_sensitivity
|
|
13
|
+
from .calibration import ece, reliability_table
|
|
14
|
+
from .splits import assert_no_subject_leak, SubjectLeakError, split_by_subject
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def _read_csv(path):
|
|
18
|
+
with open(path, newline="", encoding="utf8") as fh:
|
|
19
|
+
return list(csv.DictReader(fh))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def cmd_check_split(args) -> int:
|
|
23
|
+
rows = _read_csv(args.csv)
|
|
24
|
+
subj = np.array([r[args.subject_col] for r in rows])
|
|
25
|
+
split = np.array([r[args.split_col] for r in rows])
|
|
26
|
+
try:
|
|
27
|
+
assert_no_subject_leak(subj, split == "train", split == "test")
|
|
28
|
+
except SubjectLeakError as exc:
|
|
29
|
+
print(f"LEAK: {exc}", file=sys.stderr)
|
|
30
|
+
return 1
|
|
31
|
+
print(f"ok: {len(np.unique(subj))} subjects, no overlap between splits")
|
|
32
|
+
return 0
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def cmd_score(args) -> int:
|
|
36
|
+
rows = _read_csv(args.csv)
|
|
37
|
+
y = np.array([int(float(r[args.label_col])) for r in rows])
|
|
38
|
+
p = np.array([float(r[args.score_col]) for r in rows])
|
|
39
|
+
out = {"n": len(y), "event_rate": float(y.mean()),
|
|
40
|
+
"ece": ece(y, p), "reliability": reliability_table(y, p)}
|
|
41
|
+
if args.baseline_col:
|
|
42
|
+
b = np.array([float(r[args.baseline_col]) for r in rows])
|
|
43
|
+
out["matched_sensitivity"] = compare_at_matched_sensitivity(
|
|
44
|
+
y, b, p, args.target_sensitivity)
|
|
45
|
+
json.dump(out, sys.stdout, indent=2)
|
|
46
|
+
print()
|
|
47
|
+
return 0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def main(argv=None) -> int:
|
|
51
|
+
ap = argparse.ArgumentParser(prog="icuflow", description=__doc__)
|
|
52
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
53
|
+
|
|
54
|
+
a = sub.add_parser("check-split", help="fail if a subject spans train and test")
|
|
55
|
+
a.add_argument("csv")
|
|
56
|
+
a.add_argument("--subject-col", default="subject_id")
|
|
57
|
+
a.add_argument("--split-col", default="split")
|
|
58
|
+
a.set_defaults(fn=cmd_check_split)
|
|
59
|
+
|
|
60
|
+
b = sub.add_parser("score", help="calibration and alarm metrics for predictions")
|
|
61
|
+
b.add_argument("csv")
|
|
62
|
+
b.add_argument("--label-col", default="label")
|
|
63
|
+
b.add_argument("--score-col", default="score")
|
|
64
|
+
b.add_argument("--baseline-col", default=None)
|
|
65
|
+
b.add_argument("--target-sensitivity", type=float, default=0.80)
|
|
66
|
+
b.set_defaults(fn=cmd_score)
|
|
67
|
+
|
|
68
|
+
args = ap.parse_args(argv)
|
|
69
|
+
return args.fn(args)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
if __name__ == "__main__":
|
|
73
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Subject-grouped splitting.
|
|
2
|
+
|
|
3
|
+
The most common defect in clinical prediction code, and the one that most
|
|
4
|
+
inflates published numbers: splitting rows instead of patients. Two windows
|
|
5
|
+
from one admission share a baseline, a trajectory and often the label, so a
|
|
6
|
+
random row split lets the model memorise the patient.
|
|
7
|
+
|
|
8
|
+
`assert_no_subject_leak` exists so that this can be a test in a pipeline rather
|
|
9
|
+
than a convention someone remembers.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class SubjectLeakError(AssertionError):
|
|
20
|
+
"""Raised when the same subject appears in two folds."""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
@dataclass
|
|
24
|
+
class GroupSplit:
|
|
25
|
+
train: np.ndarray # boolean mask
|
|
26
|
+
test: np.ndarray
|
|
27
|
+
|
|
28
|
+
def sizes(self) -> tuple:
|
|
29
|
+
return int(self.train.sum()), int(self.test.sum())
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def split_by_subject(subjects, test_frac: float = 0.3,
|
|
33
|
+
seed: int = 0) -> GroupSplit:
|
|
34
|
+
subjects = np.asarray(subjects)
|
|
35
|
+
uniq = np.unique(subjects)
|
|
36
|
+
rng = np.random.default_rng(seed)
|
|
37
|
+
order = rng.permutation(len(uniq))
|
|
38
|
+
n_test = max(1, int(round(len(uniq) * test_frac)))
|
|
39
|
+
test_ids = set(uniq[order[:n_test]].tolist())
|
|
40
|
+
test = np.fromiter((s in test_ids for s in subjects), bool, len(subjects))
|
|
41
|
+
return GroupSplit(train=~test, test=test)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def group_kfold(subjects, n_splits: int = 5, seed: int = 0) -> list:
|
|
45
|
+
"""K folds that never split a subject across them."""
|
|
46
|
+
subjects = np.asarray(subjects)
|
|
47
|
+
uniq = np.unique(subjects)
|
|
48
|
+
rng = np.random.default_rng(seed)
|
|
49
|
+
order = rng.permutation(len(uniq))
|
|
50
|
+
folds = np.array_split(uniq[order], n_splits)
|
|
51
|
+
out = []
|
|
52
|
+
for f in folds:
|
|
53
|
+
ids = set(f.tolist())
|
|
54
|
+
test = np.fromiter((s in ids for s in subjects), bool, len(subjects))
|
|
55
|
+
out.append(GroupSplit(train=~test, test=test))
|
|
56
|
+
return out
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def assert_no_subject_leak(subjects, train_mask, test_mask) -> None:
|
|
60
|
+
subjects = np.asarray(subjects)
|
|
61
|
+
shared = set(subjects[train_mask].tolist()) & set(subjects[test_mask].tolist())
|
|
62
|
+
if shared:
|
|
63
|
+
raise SubjectLeakError(
|
|
64
|
+
f"{len(shared)} subject(s) appear in both train and test, "
|
|
65
|
+
f"e.g. {sorted(shared)[:5]}")
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Task construction in the shape PyHealth and RHealth pipelines expect.
|
|
2
|
+
|
|
3
|
+
Both frameworks model a task as: take a patient record, emit samples. These
|
|
4
|
+
helpers cover the two steps every ICU horizon task needs and that every project
|
|
5
|
+
rewrites -- windowing a series, and labelling a prediction horizon without
|
|
6
|
+
leaking the present into the future.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def sliding_windows(n: int, window: int, stride: int = 1):
|
|
15
|
+
"""Yield (start, end) index pairs. End is exclusive."""
|
|
16
|
+
if window <= 0 or stride <= 0:
|
|
17
|
+
raise ValueError("window and stride must be positive")
|
|
18
|
+
for a in range(0, max(0, n - window + 1), stride):
|
|
19
|
+
yield a, a + window
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def horizon_label(events, index: int, horizon_steps: int,
|
|
23
|
+
exclude_active: bool = True):
|
|
24
|
+
"""Does an event occur in the horizon strictly after `index`?
|
|
25
|
+
|
|
26
|
+
Returns None when the row should be dropped, which happens when the event
|
|
27
|
+
is already active. Predicting an event in progress is not early warning,
|
|
28
|
+
and including those rows inflates every metric that follows.
|
|
29
|
+
"""
|
|
30
|
+
ev = np.asarray(events, bool)
|
|
31
|
+
if exclude_active and ev[index]:
|
|
32
|
+
return None
|
|
33
|
+
future = ev[index + 1: index + 1 + horizon_steps]
|
|
34
|
+
if len(future) < horizon_steps:
|
|
35
|
+
return None # not enough follow-up to label honestly
|
|
36
|
+
return int(future.any())
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def build_horizon_task(subjects, series_by_subject, events_by_subject,
|
|
40
|
+
featurise, horizon_steps: int, min_history: int = 4):
|
|
41
|
+
"""Assemble X, y, subject arrays for a horizon-prediction task."""
|
|
42
|
+
X, y, subj = [], [], []
|
|
43
|
+
for s in subjects:
|
|
44
|
+
ev = np.asarray(events_by_subject[s], bool)
|
|
45
|
+
n = len(ev)
|
|
46
|
+
for i in range(min_history, n):
|
|
47
|
+
lab = horizon_label(ev, i, horizon_steps)
|
|
48
|
+
if lab is None:
|
|
49
|
+
continue
|
|
50
|
+
X.append(featurise(series_by_subject[s], i))
|
|
51
|
+
y.append(lab)
|
|
52
|
+
subj.append(s)
|
|
53
|
+
return np.asarray(X, float), np.asarray(y, int), np.asarray(subj)
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: icuflow
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Leak-safe cohort splitting, calibration and alarm metrics for clinical prediction, in the shape PyHealth and RHealth tasks expect.
|
|
5
|
+
Author: Wenke Du
|
|
6
|
+
License: MIT
|
|
7
|
+
Project-URL: Source, https://github.com/evelyyyyynnnnn/2.0-Healthcare-Ai-Systems
|
|
8
|
+
Keywords: clinical,icu,pyhealth,calibration,early-warning
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
|
|
14
|
+
Requires-Python: >=3.9
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
Requires-Dist: numpy>=1.24
|
|
17
|
+
Provides-Extra: pyhealth
|
|
18
|
+
Requires-Dist: pyhealth>=1.1.4; extra == "pyhealth"
|
|
19
|
+
Provides-Extra: dev
|
|
20
|
+
Requires-Dist: pytest>=7.0; extra == "dev"
|
|
21
|
+
Requires-Dist: scikit-learn>=1.3; extra == "dev"
|
|
22
|
+
|
|
23
|
+
# icuflow
|
|
24
|
+
|
|
25
|
+
> An installable package for the three parts of a clinical prediction pipeline that are easiest to get wrong: subject-grouped splits, calibration, and alarm comparison at matched sensitivity.
|
|
26
|
+
|
|
27
|
+
**Repository:** `2.0-Healthcare-Ai-Systems` · **Pillar:** Healthcare Safety
|
|
28
|
+
|
|
29
|
+
## Status
|
|
30
|
+
|
|
31
|
+
Working code, **29 passing tests**, and one measured run on real ICU charting.
|
|
32
|
+
Earlier versions of this page said the experiments ran on synthetic subjects;
|
|
33
|
+
the reportable run does not.
|
|
34
|
+
|
|
35
|
+
`icuflow` 0.1.0 is an installable package with 14 exports and a working CLI
|
|
36
|
+
(`check-split`, `score`). It is **not published to PyPI** — `published` is
|
|
37
|
+
`false` in the results file — so it has no download count and no users, and
|
|
38
|
+
neither is claimed.
|
|
39
|
+
|
|
40
|
+
**What it measures.** Using the **PhysioNet MIMIC-IV open-access demo**, the same cohort, features and model are split two ways. A random **row-level** split reports **AUROC 0.9058**. A **subject-grouped** split reports **0.4989** — chance. The reported figure is inflated by **81.57%**, and the honest one says the model learned nothing.
|
|
41
|
+
|
|
42
|
+
Cohort: 97 subjects, 134 stays, 37 readmissions, 22,512 rows, positive rate
|
|
43
|
+
0.02838. about 100 ICU stays. The direction of the leakage effect is
|
|
44
|
+
meaningful; its magnitude, from a cohort this small, is not a population
|
|
45
|
+
estimate.
|
|
46
|
+
|
|
47
|
+
Last reportable run: `2026-09-06T23:57:16+00:00`
|
|
48
|
+
|
|
49
|
+
## Quick start
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
pip install -r requirements.txt
|
|
53
|
+
python -m pytest tests/ -q # 29 tests
|
|
54
|
+
python -m data.fetch # needs the sources in docs/DATA.md
|
|
55
|
+
python -m src.demo --real # the measured run -> results/latest-real.json
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
## Layout
|
|
59
|
+
|
|
60
|
+
```
|
|
61
|
+
README.md
|
|
62
|
+
data/
|
|
63
|
+
|-- README.md
|
|
64
|
+
|-- manifests/
|
|
65
|
+
|-- sample/
|
|
66
|
+
docs/
|
|
67
|
+
|-- DATA.md
|
|
68
|
+
|-- METHOD.md
|
|
69
|
+
icuflow/
|
|
70
|
+
|-- __init__.py
|
|
71
|
+
|-- alarms.py
|
|
72
|
+
|-- calibration.py
|
|
73
|
+
|-- cli.py
|
|
74
|
+
|-- splits.py
|
|
75
|
+
|-- tasks.py
|
|
76
|
+
icuflow.egg-info/
|
|
77
|
+
|-- PKG-INFO
|
|
78
|
+
|-- SOURCES.txt
|
|
79
|
+
|-- dependency_links.txt
|
|
80
|
+
|-- entry_points.txt
|
|
81
|
+
|-- requires.txt
|
|
82
|
+
|-- top_level.txt
|
|
83
|
+
pyproject.toml
|
|
84
|
+
requirements.txt
|
|
85
|
+
results/
|
|
86
|
+
|-- README.md
|
|
87
|
+
|-- latest.json
|
|
88
|
+
src/
|
|
89
|
+
|-- .gitkeep
|
|
90
|
+
|-- __init__.py
|
|
91
|
+
|-- demo.py
|
|
92
|
+
|-- site.py
|
|
93
|
+
|-- sitekit.py
|
|
94
|
+
tests/
|
|
95
|
+
|-- .gitkeep
|
|
96
|
+
|-- test_icuflow.py
|
|
97
|
+
website/
|
|
98
|
+
|-- README.md
|
|
99
|
+
|-- index.html
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
- `src/` — the implementation.
|
|
103
|
+
- `tests/` — pytest suite. These guard behaviour, not just imports.
|
|
104
|
+
- `results/latest.json` — the output of the last demo run. Every figure quoted
|
|
105
|
+
anywhere in this project traces back to this file.
|
|
106
|
+
- `website/` — a self-contained static site, deployable to Vercel by copying the
|
|
107
|
+
folder into its own repository. See `website/README.md`.
|
|
108
|
+
|
|
109
|
+
## The website
|
|
110
|
+
|
|
111
|
+
`website/` has no build step. To deploy it independently:
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
cp -r website/ ../my-4-pyhealth-rhealth-extension-site && cd ../my-4-pyhealth-rhealth-extension-site
|
|
115
|
+
git init && git add -A && git commit -m "site"
|
|
116
|
+
vercel deploy --prod
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
The page is regenerated from `results.json` on every `python -m src.demo`, so the
|
|
120
|
+
figures on the site and the figures the code produces cannot drift apart. Do not edit
|
|
121
|
+
numbers on the page by hand.
|
|
122
|
+
|
|
123
|
+
## Honesty note
|
|
124
|
+
|
|
125
|
+
Everything in this project runs on clearly-labelled synthetic or authored data.
|
|
126
|
+
Swap in the real source and the same pipeline reports real numbers — that is
|
|
127
|
+
what the structure is for. Until that happens, nothing here should be cited as a
|
|
128
|
+
measured result, and the site's closing section states explicitly what the project
|
|
129
|
+
does not establish.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
README.md
|
|
2
|
+
pyproject.toml
|
|
3
|
+
icuflow/__init__.py
|
|
4
|
+
icuflow/alarms.py
|
|
5
|
+
icuflow/calibration.py
|
|
6
|
+
icuflow/cli.py
|
|
7
|
+
icuflow/splits.py
|
|
8
|
+
icuflow/tasks.py
|
|
9
|
+
icuflow.egg-info/PKG-INFO
|
|
10
|
+
icuflow.egg-info/SOURCES.txt
|
|
11
|
+
icuflow.egg-info/dependency_links.txt
|
|
12
|
+
icuflow.egg-info/entry_points.txt
|
|
13
|
+
icuflow.egg-info/requires.txt
|
|
14
|
+
icuflow.egg-info/top_level.txt
|
|
15
|
+
tests/test_data_layer.py
|
|
16
|
+
tests/test_icuflow.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
icuflow
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=68", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "icuflow"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Leak-safe cohort splitting, calibration and alarm metrics for clinical prediction, in the shape PyHealth and RHealth tasks expect."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = {text = "MIT"}
|
|
12
|
+
authors = [{name = "Wenke Du"}]
|
|
13
|
+
keywords = ["clinical", "icu", "pyhealth", "calibration", "early-warning"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Intended Audience :: Science/Research",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Topic :: Scientific/Engineering :: Medical Science Apps.",
|
|
20
|
+
]
|
|
21
|
+
dependencies = ["numpy>=1.24"]
|
|
22
|
+
|
|
23
|
+
[project.optional-dependencies]
|
|
24
|
+
pyhealth = ["pyhealth>=1.1.4"]
|
|
25
|
+
dev = ["pytest>=7.0", "scikit-learn>=1.3"]
|
|
26
|
+
|
|
27
|
+
[project.scripts]
|
|
28
|
+
icuflow = "icuflow.cli:main"
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Source = "https://github.com/evelyyyyynnnnn/2.0-Healthcare-Ai-Systems"
|
|
32
|
+
|
|
33
|
+
[tool.setuptools.packages.find]
|
|
34
|
+
include = ["icuflow*"]
|
icuflow-0.1.0/setup.cfg
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
"""Tests for the real-MIMIC leakage measurement.
|
|
2
|
+
|
|
3
|
+
The property that must not break: rows are grouped by the PERSON, not by the
|
|
4
|
+
ICU stay. A readmitted patient has two stay_ids and one physiological baseline,
|
|
5
|
+
so splitting on stay_id puts that baseline on both sides of the split -- the
|
|
6
|
+
precise leak this package exists to prevent, reintroduced by the data loader.
|
|
7
|
+
"""
|
|
8
|
+
import datetime as dt
|
|
9
|
+
import gzip
|
|
10
|
+
import io
|
|
11
|
+
import pathlib
|
|
12
|
+
import sys
|
|
13
|
+
|
|
14
|
+
import numpy as np
|
|
15
|
+
import pytest
|
|
16
|
+
|
|
17
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
|
18
|
+
|
|
19
|
+
from data import datakit
|
|
20
|
+
from data.load import load_xy
|
|
21
|
+
from data.mimicvitals import PROJECT, resample
|
|
22
|
+
|
|
23
|
+
PROJ = PROJECT
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _gz(rows, header):
|
|
27
|
+
buf = io.StringIO()
|
|
28
|
+
buf.write(",".join(header) + "\n")
|
|
29
|
+
for r in rows:
|
|
30
|
+
buf.write(",".join(str(r.get(h, "")) for h in header) + "\n")
|
|
31
|
+
return gzip.compress(buf.getvalue().encode())
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _seed(tmp_path, n_subjects=8, stays_per=(1,), hours=30):
|
|
35
|
+
"""Build a MIMIC-shaped cache. Some subjects get two stays (readmissions)."""
|
|
36
|
+
f = datakit.Fetcher(tmp_path)
|
|
37
|
+
man = f.load_manifest()
|
|
38
|
+
rng = np.random.default_rng(11)
|
|
39
|
+
|
|
40
|
+
stays, patients, chart = [], [], []
|
|
41
|
+
base = dt.datetime(2180, 7, 23, 0, 0)
|
|
42
|
+
stay_id = 100
|
|
43
|
+
for s in range(n_subjects):
|
|
44
|
+
subject = 200 + s
|
|
45
|
+
patients.append({"subject_id": subject, "anchor_age": 50 + s})
|
|
46
|
+
# A per-patient baseline, which is exactly what a row split can memorise.
|
|
47
|
+
base_hr = 60 + 4 * s
|
|
48
|
+
n_stays = stays_per[s % len(stays_per)]
|
|
49
|
+
for _ in range(n_stays):
|
|
50
|
+
stays.append({"stay_id": stay_id, "subject_id": subject})
|
|
51
|
+
for h in range(hours * 2):
|
|
52
|
+
ts = (base + dt.timedelta(minutes=30 * h)).strftime(
|
|
53
|
+
"%Y-%m-%d %H:%M:%S")
|
|
54
|
+
spo2 = 97 + rng.normal(0, 1.2)
|
|
55
|
+
if 20 <= h % 40 <= 24 and s % 3 == 0:
|
|
56
|
+
spo2 -= 11
|
|
57
|
+
for itemid, val in ((220277, spo2),
|
|
58
|
+
(220045, base_hr + rng.normal(0, 2)),
|
|
59
|
+
(220181, 85 + rng.normal(0, 6)),
|
|
60
|
+
(220210, 16 + rng.normal(0, 2)),
|
|
61
|
+
(223762, 36.8), (220179, 120), (220180, 70)):
|
|
62
|
+
chart.append({"stay_id": stay_id, "itemid": itemid,
|
|
63
|
+
"charttime": ts,
|
|
64
|
+
"valuenum": round(float(val), 2)})
|
|
65
|
+
stay_id += 1
|
|
66
|
+
|
|
67
|
+
files = {
|
|
68
|
+
f"{PROJ}/icu/icustays.csv.gz": _gz(stays, ["stay_id", "subject_id"]),
|
|
69
|
+
f"{PROJ}/hosp/patients.csv.gz": _gz(patients,
|
|
70
|
+
["subject_id", "anchor_age"]),
|
|
71
|
+
f"{PROJ}/icu/chartevents.csv.gz": _gz(
|
|
72
|
+
chart, ["stay_id", "itemid", "charttime", "valuenum"]),
|
|
73
|
+
}
|
|
74
|
+
for dest, raw in files.items():
|
|
75
|
+
p = f.raw / dest
|
|
76
|
+
p.parent.mkdir(parents=True, exist_ok=True)
|
|
77
|
+
p.write_bytes(raw)
|
|
78
|
+
man["files"][dest] = {
|
|
79
|
+
"source": dest, "url": f"https://physionet.org/files/{dest}",
|
|
80
|
+
"publisher": "PhysioNet", "terms": "open access",
|
|
81
|
+
"sha256": datakit.sha256_file(p), "bytes": len(raw),
|
|
82
|
+
"retrieved_utc": datakit.utc_now()}
|
|
83
|
+
f._write_manifest(man)
|
|
84
|
+
return f
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_refuses_without_the_real_files(tmp_path):
|
|
88
|
+
with pytest.raises(datakit.FetchError, match="missing real MIMIC-IV files"):
|
|
89
|
+
load_xy(root=tmp_path)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def test_grouping_key_is_the_patient_not_the_stay(tmp_path):
|
|
93
|
+
"""The test this whole module exists for."""
|
|
94
|
+
_seed(tmp_path, n_subjects=6, stays_per=(2,)) # every patient readmitted
|
|
95
|
+
X, y, subjects, stay_ids, names, prov = load_xy(root=tmp_path)
|
|
96
|
+
|
|
97
|
+
assert len(np.unique(stay_ids)) == 12
|
|
98
|
+
assert len(np.unique(subjects)) == 6
|
|
99
|
+
assert prov["grouping_key"].startswith("subject_id")
|
|
100
|
+
# Each subject really does span more than one stay in this cohort.
|
|
101
|
+
for s in np.unique(subjects):
|
|
102
|
+
assert len(np.unique(stay_ids[subjects == s])) == 2
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_a_subject_split_keeps_readmissions_together(tmp_path):
|
|
106
|
+
_seed(tmp_path, n_subjects=6, stays_per=(2,))
|
|
107
|
+
from icuflow import assert_no_subject_leak, split_by_subject
|
|
108
|
+
_, _, subjects, stay_ids, _, _ = load_xy(root=tmp_path)
|
|
109
|
+
|
|
110
|
+
sp = split_by_subject(subjects, test_frac=0.34, seed=0)
|
|
111
|
+
assert_no_subject_leak(subjects, sp.train, sp.test)
|
|
112
|
+
# No patient on both sides, and therefore no stay of theirs either.
|
|
113
|
+
assert not (set(subjects[sp.train]) & set(subjects[sp.test]))
|
|
114
|
+
assert not (set(stay_ids[sp.train]) & set(stay_ids[sp.test]))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def test_labels_are_strictly_in_the_future(tmp_path):
|
|
118
|
+
"""Rows already in the event are dropped, so the label is not the feature."""
|
|
119
|
+
_seed(tmp_path)
|
|
120
|
+
X, y, subjects, _, names, prov = load_xy(root=tmp_path)
|
|
121
|
+
spo2 = X[:, names.index("spo2_pct")]
|
|
122
|
+
# Every row used for prediction is a row where the patient is not currently
|
|
123
|
+
# hypoxemic; otherwise predicting "hypoxemia" would be reading it off.
|
|
124
|
+
assert (spo2 >= 90.0).all()
|
|
125
|
+
assert prov["horizon_hours"] == 4.0
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def test_features_have_no_missing_values(tmp_path):
|
|
129
|
+
_seed(tmp_path)
|
|
130
|
+
X, y, *_ = load_xy(root=tmp_path)
|
|
131
|
+
assert np.isfinite(X).all()
|
|
132
|
+
assert set(np.unique(y)) <= {0, 1}
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def test_provenance_reports_cohort_size_and_caveat(tmp_path):
|
|
136
|
+
_seed(tmp_path)
|
|
137
|
+
*_, prov = load_xy(root=tmp_path)
|
|
138
|
+
assert prov["cohort_is_a_demonstration_not_a_study"] is True
|
|
139
|
+
assert prov["n_subjects"] >= 1
|
|
140
|
+
assert 0.0 <= prov["positive_rate"] <= 1.0
|
|
141
|
+
assert all(len(v["sha256"]) == 16 for v in prov["files"].values())
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def test_an_event_with_no_usable_rows_is_refused(tmp_path):
|
|
145
|
+
"""Better to say the cohort is too small than to score three events."""
|
|
146
|
+
_seed(tmp_path, n_subjects=2, hours=13)
|
|
147
|
+
with pytest.raises(datakit.FetchError):
|
|
148
|
+
load_xy(root=tmp_path, event="hypotension", min_hours=40.0)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def test_the_leakage_experiment_runs_end_to_end_on_real_shaped_data(tmp_path):
|
|
152
|
+
_seed(tmp_path, n_subjects=10, stays_per=(1, 2))
|
|
153
|
+
from src.demo import leakage_experiment_real
|
|
154
|
+
out = leakage_experiment_real(tmp_path)
|
|
155
|
+
|
|
156
|
+
for k in ("row_split_auroc", "subject_split_auroc", "inflation"):
|
|
157
|
+
assert k in out
|
|
158
|
+
assert 0.0 <= out["row_split_auroc"] <= 1.0
|
|
159
|
+
assert 0.0 <= out["subject_split_auroc"] <= 1.0
|
|
160
|
+
assert out["n_readmissions"] == out["n_stays"] - out["n_subjects"]
|
|
161
|
+
# The result is whatever it is -- the test pins the plumbing, not a
|
|
162
|
+
# direction, because on real data the direction is the finding.
|
|
163
|
+
assert isinstance(out["inflation"], float)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def test_resample_never_moves_a_value_backwards(tmp_path):
|
|
167
|
+
grid = np.arange(0.0, 5.0, 1.0)
|
|
168
|
+
out = resample([(3.0, 120.0)], grid, default=None)
|
|
169
|
+
assert out[3] == 120.0
|
|
170
|
+
assert set(out[:3]) == {120.0} # prefix back-fill only, no future leak
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import pathlib
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
import numpy as np
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent.parent))
|
|
8
|
+
|
|
9
|
+
from icuflow import (IsotonicCalibrator, PlattCalibrator, alarm_stats,
|
|
10
|
+
assert_no_subject_leak, compare_at_matched_sensitivity,
|
|
11
|
+
ece, group_kfold, horizon_label, reliability_table,
|
|
12
|
+
sliding_windows, split_by_subject)
|
|
13
|
+
from icuflow.splits import SubjectLeakError
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
# --- splits --------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
def test_split_never_shares_a_subject():
|
|
19
|
+
subj = np.repeat(np.arange(50), 10)
|
|
20
|
+
sp = split_by_subject(subj, 0.3, seed=0)
|
|
21
|
+
assert not (set(subj[sp.train].tolist()) & set(subj[sp.test].tolist()))
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def test_split_covers_every_row():
|
|
25
|
+
subj = np.repeat(np.arange(30), 7)
|
|
26
|
+
sp = split_by_subject(subj, 0.25, seed=1)
|
|
27
|
+
assert (sp.train | sp.test).all()
|
|
28
|
+
assert not (sp.train & sp.test).any()
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def test_split_is_deterministic_for_a_seed():
|
|
32
|
+
subj = np.repeat(np.arange(40), 5)
|
|
33
|
+
a = split_by_subject(subj, 0.3, seed=7)
|
|
34
|
+
b = split_by_subject(subj, 0.3, seed=7)
|
|
35
|
+
assert np.array_equal(a.test, b.test)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def test_group_kfold_partitions_subjects_exactly_once():
|
|
39
|
+
subj = np.repeat(np.arange(40), 6)
|
|
40
|
+
folds = group_kfold(subj, n_splits=4, seed=2)
|
|
41
|
+
assert len(folds) == 4
|
|
42
|
+
seen = np.zeros(len(subj), int)
|
|
43
|
+
for f in folds:
|
|
44
|
+
seen += f.test.astype(int)
|
|
45
|
+
assert not (set(subj[f.train].tolist()) & set(subj[f.test].tolist()))
|
|
46
|
+
assert (seen == 1).all(), "every row must be tested exactly once"
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def test_leak_guard_raises():
|
|
50
|
+
subj = np.repeat(np.arange(10), 4)
|
|
51
|
+
with pytest.raises(SubjectLeakError):
|
|
52
|
+
assert_no_subject_leak(subj, np.ones(len(subj), bool),
|
|
53
|
+
np.ones(len(subj), bool))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def test_leak_guard_passes_a_clean_split():
|
|
57
|
+
subj = np.repeat(np.arange(10), 4)
|
|
58
|
+
sp = split_by_subject(subj, 0.3, seed=0)
|
|
59
|
+
assert_no_subject_leak(subj, sp.train, sp.test) # must not raise
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# --- calibration ---------------------------------------------------------
|
|
63
|
+
|
|
64
|
+
def test_isotonic_is_monotone():
|
|
65
|
+
"""Calibration must move probabilities without reordering patients."""
|
|
66
|
+
rng = np.random.default_rng(0)
|
|
67
|
+
s = rng.uniform(0, 1, 2000)
|
|
68
|
+
y = (rng.uniform(size=2000) < s).astype(int)
|
|
69
|
+
cal = IsotonicCalibrator().fit(s, y)
|
|
70
|
+
grid = np.linspace(0, 1, 200)
|
|
71
|
+
out = cal.transform(grid)
|
|
72
|
+
assert np.all(np.diff(out) >= -1e-9)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def test_isotonic_improves_a_distorted_score():
|
|
76
|
+
rng = np.random.default_rng(1)
|
|
77
|
+
p = rng.uniform(0.02, 0.98, 6000)
|
|
78
|
+
y = (rng.uniform(size=6000) < p).astype(int)
|
|
79
|
+
bad = p ** 3 / (p ** 3 + (1 - p) ** 3)
|
|
80
|
+
cal = IsotonicCalibrator().fit(bad[:3000], y[:3000])
|
|
81
|
+
assert ece(y[3000:], cal.transform(bad[3000:])) < ece(y[3000:], bad[3000:])
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def test_platt_improves_a_distorted_score():
|
|
85
|
+
rng = np.random.default_rng(2)
|
|
86
|
+
p = rng.uniform(0.05, 0.95, 6000)
|
|
87
|
+
y = (rng.uniform(size=6000) < p).astype(int)
|
|
88
|
+
bad = p ** 3 / (p ** 3 + (1 - p) ** 3)
|
|
89
|
+
cal = PlattCalibrator().fit(bad[:3000], y[:3000])
|
|
90
|
+
assert ece(y[3000:], cal.transform(bad[3000:])) < ece(y[3000:], bad[3000:])
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_ece_near_zero_for_a_calibrated_forecast():
|
|
94
|
+
rng = np.random.default_rng(3)
|
|
95
|
+
p = rng.uniform(0, 1, 50000)
|
|
96
|
+
y = (rng.uniform(size=50000) < p).astype(int)
|
|
97
|
+
assert ece(y, p) < 0.02
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_reliability_bins_sum_to_the_sample():
|
|
101
|
+
rng = np.random.default_rng(4)
|
|
102
|
+
p = rng.uniform(0, 1, 1000)
|
|
103
|
+
y = (rng.uniform(size=1000) < p).astype(int)
|
|
104
|
+
assert sum(r["n"] for r in reliability_table(y, p)) == 1000
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def test_transform_before_fit_raises():
|
|
108
|
+
with pytest.raises(RuntimeError):
|
|
109
|
+
IsotonicCalibrator().transform(np.array([0.5]))
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
# --- alarms --------------------------------------------------------------
|
|
113
|
+
|
|
114
|
+
def test_matched_comparison_equalises_sensitivity():
|
|
115
|
+
rng = np.random.default_rng(5)
|
|
116
|
+
y = (rng.uniform(size=5000) < 0.15).astype(int)
|
|
117
|
+
base = rng.uniform(size=5000) + y * 0.2
|
|
118
|
+
model = rng.uniform(size=5000) + y * 0.7
|
|
119
|
+
cmp = compare_at_matched_sensitivity(y, base, model, 0.80)
|
|
120
|
+
assert abs(cmp["sensitivity_gap"]) < 0.05
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def test_better_model_cuts_false_alerts_at_matched_sensitivity():
|
|
124
|
+
rng = np.random.default_rng(6)
|
|
125
|
+
y = (rng.uniform(size=5000) < 0.15).astype(int)
|
|
126
|
+
base = rng.uniform(size=5000) + y * 0.15
|
|
127
|
+
model = rng.uniform(size=5000) + y * 0.9
|
|
128
|
+
cmp = compare_at_matched_sensitivity(y, base, model, 0.80)
|
|
129
|
+
assert cmp["false_alert_reduction_pct"] > 0
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_alarm_counts_partition_the_sample():
|
|
133
|
+
y = np.array([0, 1, 1, 0, 1])
|
|
134
|
+
s = np.array([0.1, 0.9, 0.4, 0.6, 0.8])
|
|
135
|
+
a = alarm_stats(y, s, 0.5)
|
|
136
|
+
assert a["tp"] + a["fp"] + a["fn"] + a["tn"] == len(y)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
# --- tasks ---------------------------------------------------------------
|
|
140
|
+
|
|
141
|
+
def test_horizon_label_drops_active_events():
|
|
142
|
+
assert horizon_label(np.array([0, 1, 0, 0, 0], bool), 1, 2) is None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def test_horizon_label_drops_short_followup():
|
|
146
|
+
assert horizon_label(np.array([0, 0, 0], bool), 2, 3) is None
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def test_horizon_label_sees_only_the_future():
|
|
150
|
+
ev = np.array([1, 0, 0, 0, 1, 0], bool)
|
|
151
|
+
assert horizon_label(ev, 1, 2) == 0 # events at 0 and 4, neither in 2..3
|
|
152
|
+
assert horizon_label(ev, 2, 2) == 1 # event at 4 is inside 3..4
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def test_sliding_windows_are_in_range():
|
|
156
|
+
n = 25
|
|
157
|
+
for a, b in sliding_windows(n, 10, 5):
|
|
158
|
+
assert 0 <= a < b <= n
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def test_sliding_windows_rejects_bad_arguments():
|
|
162
|
+
with pytest.raises(ValueError):
|
|
163
|
+
list(sliding_windows(10, 0, 1))
|