Multi-MIMIC 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- multi_mimic-0.0.1/.github/workflows/live.yaml +29 -0
- multi_mimic-0.0.1/.github/workflows/python-build.yaml +93 -0
- multi_mimic-0.0.1/.github/workflows/tests.yaml +32 -0
- multi_mimic-0.0.1/.gitignore +16 -0
- multi_mimic-0.0.1/PKG-INFO +389 -0
- multi_mimic-0.0.1/README.md +360 -0
- multi_mimic-0.0.1/pyproject.toml +47 -0
- multi_mimic-0.0.1/src/multi_mimic/__init__.py +7 -0
- multi_mimic-0.0.1/src/multi_mimic/__main__.py +5 -0
- multi_mimic-0.0.1/src/multi_mimic/check.py +145 -0
- multi_mimic-0.0.1/src/multi_mimic/cli.py +235 -0
- multi_mimic-0.0.1/src/multi_mimic/download.py +444 -0
- multi_mimic-0.0.1/src/multi_mimic/figure.py +257 -0
- multi_mimic-0.0.1/src/multi_mimic/http.py +114 -0
- multi_mimic-0.0.1/src/multi_mimic/listing.py +101 -0
- multi_mimic-0.0.1/src/multi_mimic/manifest.py +93 -0
- multi_mimic-0.0.1/src/multi_mimic/origins.py +136 -0
- multi_mimic-0.0.1/src/multi_mimic/sources.py +91 -0
- multi_mimic-0.0.1/src/multi_mimic/tables.py +94 -0
- multi_mimic-0.0.1/tests/conftest.py +6 -0
- multi_mimic-0.0.1/tests/mock_physionet.py +255 -0
- multi_mimic-0.0.1/tests/test_download.py +279 -0
- multi_mimic-0.0.1/tests/test_live.py +125 -0
- multi_mimic-0.0.1/uv.lock +1852 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# The live test against the real PhysioNet (tests/test_live.py): open data only, about 2 MB.
|
|
2
|
+
# Never give this workflow a PhysioNet sign-in (no PHYSIONET_TOKEN or password secrets):
|
|
3
|
+
# credentialed MIMIC data must not be downloaded to GitHub's machines, and the
|
|
4
|
+
# credentialed part of the test stays skipped without one.
|
|
5
|
+
|
|
6
|
+
name: Live PhysioNet test
|
|
7
|
+
|
|
8
|
+
on:
|
|
9
|
+
workflow_dispatch:
|
|
10
|
+
schedule:
|
|
11
|
+
- cron: "0 6 * * 1" # Mondays: notices when PhysioNet or the physionet package changes
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
live:
|
|
15
|
+
runs-on: ubuntu-latest
|
|
16
|
+
timeout-minutes: 20
|
|
17
|
+
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/checkout@v4
|
|
20
|
+
|
|
21
|
+
- name: Install uv
|
|
22
|
+
uses: astral-sh/setup-uv@v6
|
|
23
|
+
with:
|
|
24
|
+
enable-cache: true
|
|
25
|
+
|
|
26
|
+
- name: Run the live test
|
|
27
|
+
env:
|
|
28
|
+
MULTI_MIMIC_LIVE: "1"
|
|
29
|
+
run: uv run --extra test pytest tests/test_live.py -v
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
name: Publish Python distribution to PyPI
|
|
2
|
+
|
|
3
|
+
on: push
|
|
4
|
+
|
|
5
|
+
jobs:
|
|
6
|
+
build:
|
|
7
|
+
name: Build distribution
|
|
8
|
+
runs-on: ubuntu-latest
|
|
9
|
+
|
|
10
|
+
steps:
|
|
11
|
+
- uses: actions/checkout@v4
|
|
12
|
+
with:
|
|
13
|
+
fetch-depth: 0 # hatch-vcs needs tag history to compute the version
|
|
14
|
+
|
|
15
|
+
- name: Install uv
|
|
16
|
+
uses: astral-sh/setup-uv@v6
|
|
17
|
+
with:
|
|
18
|
+
enable-cache: true
|
|
19
|
+
|
|
20
|
+
- name: Build
|
|
21
|
+
run: uv build
|
|
22
|
+
|
|
23
|
+
- name: Store the distribution packages
|
|
24
|
+
uses: actions/upload-artifact@v4
|
|
25
|
+
with:
|
|
26
|
+
name: python-package-distributions
|
|
27
|
+
path: dist/
|
|
28
|
+
|
|
29
|
+
publish-to-pypi:
|
|
30
|
+
name: Publish Python distribution to PyPI
|
|
31
|
+
if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes
|
|
32
|
+
needs:
|
|
33
|
+
- build
|
|
34
|
+
runs-on: ubuntu-latest
|
|
35
|
+
environment:
|
|
36
|
+
name: pypi
|
|
37
|
+
url: https://pypi.org/p/multi-mimic
|
|
38
|
+
permissions:
|
|
39
|
+
id-token: write # IMPORTANT: mandatory for trusted publishing
|
|
40
|
+
|
|
41
|
+
steps:
|
|
42
|
+
- name: Download all the dists
|
|
43
|
+
uses: actions/download-artifact@v4
|
|
44
|
+
with:
|
|
45
|
+
name: python-package-distributions
|
|
46
|
+
path: dist/
|
|
47
|
+
|
|
48
|
+
- name: Publish distribution to PyPI
|
|
49
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
50
|
+
|
|
51
|
+
github-release:
|
|
52
|
+
name: >-
|
|
53
|
+
Sign the Python distribution with Sigstore
|
|
54
|
+
and upload them to GitHub Release
|
|
55
|
+
needs:
|
|
56
|
+
- publish-to-pypi
|
|
57
|
+
runs-on: ubuntu-latest
|
|
58
|
+
|
|
59
|
+
permissions:
|
|
60
|
+
contents: write # IMPORTANT: mandatory for making GitHub Releases
|
|
61
|
+
id-token: write # IMPORTANT: mandatory for sigstore
|
|
62
|
+
|
|
63
|
+
steps:
|
|
64
|
+
- name: Download all the dists
|
|
65
|
+
uses: actions/download-artifact@v4
|
|
66
|
+
with:
|
|
67
|
+
name: python-package-distributions
|
|
68
|
+
path: dist/
|
|
69
|
+
|
|
70
|
+
- name: Sign the dists with Sigstore
|
|
71
|
+
uses: sigstore/gh-action-sigstore-python@v3.0.0
|
|
72
|
+
with:
|
|
73
|
+
inputs: >-
|
|
74
|
+
./dist/*.tar.gz
|
|
75
|
+
./dist/*.whl
|
|
76
|
+
- name: Create GitHub Release
|
|
77
|
+
env:
|
|
78
|
+
GITHUB_TOKEN: ${{ github.token }}
|
|
79
|
+
run: >-
|
|
80
|
+
gh release create
|
|
81
|
+
'${{ github.ref_name }}'
|
|
82
|
+
--repo '${{ github.repository }}'
|
|
83
|
+
--notes ""
|
|
84
|
+
- name: Upload artifact signatures to GitHub Release
|
|
85
|
+
env:
|
|
86
|
+
GITHUB_TOKEN: ${{ github.token }}
|
|
87
|
+
# Upload to GitHub Release using the `gh` CLI.
|
|
88
|
+
# `dist/` contains the built packages, and the
|
|
89
|
+
# sigstore-produced signatures and certificates.
|
|
90
|
+
run: >-
|
|
91
|
+
gh release upload
|
|
92
|
+
'${{ github.ref_name }}' dist/**
|
|
93
|
+
--repo '${{ github.repository }}'
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
name: Tests
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
|
|
9
|
+
jobs:
|
|
10
|
+
tests:
|
|
11
|
+
runs-on: ubuntu-latest
|
|
12
|
+
|
|
13
|
+
strategy:
|
|
14
|
+
matrix:
|
|
15
|
+
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
|
16
|
+
fail-fast: false
|
|
17
|
+
|
|
18
|
+
timeout-minutes: 20
|
|
19
|
+
|
|
20
|
+
steps:
|
|
21
|
+
- uses: actions/checkout@v4
|
|
22
|
+
|
|
23
|
+
- name: Install uv
|
|
24
|
+
uses: astral-sh/setup-uv@v6
|
|
25
|
+
with:
|
|
26
|
+
enable-cache: true
|
|
27
|
+
python-version: ${{ matrix.python-version }}
|
|
28
|
+
|
|
29
|
+
# The end-to-end tests run against a local mock of PhysioNet (tests/mock_physionet.py):
|
|
30
|
+
# no network, no real data.
|
|
31
|
+
- name: Run tests
|
|
32
|
+
run: uv run --extra test pytest -v
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Generated from local (partly credentialed) data: never commit
|
|
2
|
+
docs/modality-overview-*.svg
|
|
3
|
+
docs/modality-overview-*.png
|
|
4
|
+
mimic-demo-multimodal/
|
|
5
|
+
multi-mimic.manifest.json
|
|
6
|
+
multi-mimic.check-report.txt
|
|
7
|
+
# the same, under the names of the pre-release mimic-demo-mm
|
|
8
|
+
mimic-demo-mm.manifest.json
|
|
9
|
+
mimic-demo-mm.check-report.txt
|
|
10
|
+
|
|
11
|
+
# Python / uv
|
|
12
|
+
__pycache__/
|
|
13
|
+
*.pyc
|
|
14
|
+
.venv/
|
|
15
|
+
.pytest_cache/
|
|
16
|
+
dist/
|
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: Multi-MIMIC
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Assemble a local multimodal MIMIC-IV demo: the 100 demo patients across EHR, ECG, ICU waveforms, chest X-ray, echo and notes
|
|
5
|
+
Project-URL: Homepage, https://github.com/rvandewater/Multi-MIMIC
|
|
6
|
+
Project-URL: Issues, https://github.com/rvandewater/Multi-MIMIC/issues
|
|
7
|
+
Author: Robin van de Water
|
|
8
|
+
Keywords: ECG,MIMIC-IV,PhysioNet,chest X-ray,clinical notes,echocardiography,electronic health records,multimodal,waveforms
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Medical Science Apps.
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Requires-Dist: physionet>=0.3.0
|
|
21
|
+
Requires-Dist: requests>=2.28
|
|
22
|
+
Requires-Dist: tqdm>=4.60
|
|
23
|
+
Provides-Extra: figure
|
|
24
|
+
Requires-Dist: matplotlib>=3.6; extra == 'figure'
|
|
25
|
+
Provides-Extra: test
|
|
26
|
+
Requires-Dist: matplotlib>=3.6; extra == 'test'
|
|
27
|
+
Requires-Dist: pytest>=7; extra == 'test'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# Multi-MIMIC: MIMIC-IV Multimodal Demo
|
|
31
|
+
|
|
32
|
+
[](https://github.com/rvandewater/Multi-MIMIC/actions/workflows/tests.yaml)
|
|
33
|
+
[](https://github.com/rvandewater/Multi-MIMIC/actions/workflows/python-build.yaml)
|
|
34
|
+
[](https://github.com/rvandewater/Multi-MIMIC/actions/workflows/live.yaml)
|
|
35
|
+
[](https://pypi.org/project/multi-mimic/)
|
|
36
|
+

|
|
37
|
+
[](https://physionet.org/)
|
|
38
|
+
[](#citing)
|
|
39
|
+
|
|
40
|
+
**Multi-MIMIC** assembles a **local multimodal MIMIC-IV demo**: the 100 patients of the open
|
|
41
|
+
MIMIC-IV demo, gathered across every MIMIC module. One command downloads (or
|
|
42
|
+
extracts from local copies), shows progress, and finishes with a file check
|
|
43
|
+
that verifies every file against its checksum.
|
|
44
|
+
|
|
45
|
+
Downloads go through the official [`physionet`](https://github.com/MIT-LCP/physionet)
|
|
46
|
+
package (sign-in, resume, retries, checksums). See
|
|
47
|
+
[How and where data is saved](#how-and-where-data-is-saved) for exactly what
|
|
48
|
+
ends up on your disk.
|
|
49
|
+
|
|
50
|
+
| source | access | what is fetched |
|
|
51
|
+
|---|---|---|
|
|
52
|
+
| [`mimic-iv-demo` 2.2](https://physionet.org/content/mimic-iv-demo/2.2/) | open | everything: structured, categorical and numeric EHR events |
|
|
53
|
+
| [`mimic-iv-ed-demo` 2.2](https://physionet.org/content/mimic-iv-ed-demo/2.2/) | open | everything: ED stays, triage, vitals |
|
|
54
|
+
| [`mimic-iv-ecg-demo` 0.1](https://physionet.org/content/mimic-iv-ecg-demo/0.1/) | open | everything: 12-lead ECG waveforms |
|
|
55
|
+
| [`mimic-iv-ecg` 1.0](https://physionet.org/content/mimic-iv-ecg/1.0/) | open | tables only (machine measurements, ECG report text), demo patients' rows |
|
|
56
|
+
| [`mimic4wdb` 0.1.0](https://physionet.org/content/mimic4wdb/0.1.0/) | open | demo patients' ICU bedside-monitor waveforms and numerics (WFDB records under `waves/`) |
|
|
57
|
+
| [`mimic-cxr-jpg` 2.1.0](https://physionet.org/content/mimic-cxr-jpg/2.1.0/) | credentialed | demo patients' chest X-ray JPGs; metadata and label tables filtered to them |
|
|
58
|
+
| [`mimic-iv-echo` 1.0.1](https://physionet.org/content/mimic-iv-echo/1.0.1/) | credentialed | demo patients' echo DICOMs; record list and measurements filtered to them |
|
|
59
|
+
| [`mimic-iv-note` 2.2](https://physionet.org/content/mimic-iv-note/2.2/) | credentialed | discharge summaries and radiology reports (plus detail tables), demo patients' rows |
|
|
60
|
+
| [`mimic-cxr` 2.0.0](https://physionet.org/content/mimic-cxr/2.0.0/) | credentialed | optional (`--include mimic-cxr`): chest X-ray DICOMs and per-study report text |
|
|
61
|
+
|
|
62
|
+
The waveform database is an early pre-release with about 200 patients, so only
|
|
63
|
+
a handful of the 100 demo patients (4 in 0.1.0, about 330 MB) have waveforms.
|
|
64
|
+
Its top-level `RECORDS` file is copied as is, so it still lists every record in
|
|
65
|
+
the release, including ones that aren't downloaded.
|
|
66
|
+
|
|
67
|
+
## What the demo patients have
|
|
68
|
+
|
|
69
|
+
How many of the demo patients have each modality, and which modalities occur
|
|
70
|
+
together. Each column of the co-occurrence plot counts the patients with exactly
|
|
71
|
+
that set of modalities.
|
|
72
|
+
|
|
73
|
+
<!-- modality-overview:start -->
|
|
74
|
+
_Not generated yet: run `uv run --extra figure multi-mimic figure --readme README.md`
|
|
75
|
+
after a complete download (see [Overview figure](#overview-figure))._
|
|
76
|
+
<!-- modality-overview:end -->
|
|
77
|
+
|
|
78
|
+
## Access
|
|
79
|
+
|
|
80
|
+
For the credentialed sources you need a credentialed PhysioNet account, and you
|
|
81
|
+
must have signed each project's data use agreement. Python 3.10 or later.
|
|
82
|
+
|
|
83
|
+
## Run
|
|
84
|
+
|
|
85
|
+
With [uv](https://docs.astral.sh/uv/), no installation step is needed:
|
|
86
|
+
|
|
87
|
+
```bash
|
|
88
|
+
pip install multi-mimic # or "multi-mimic[figure]" for the overview figure
|
|
89
|
+
physionet login # once: sign in through your browser, no password stored
|
|
90
|
+
multi-mimic download
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
Or, without installing anything: `uvx multi-mimic download`. In a clone of this repository, use
|
|
94
|
+
`uv run multi-mimic ...`.
|
|
95
|
+
|
|
96
|
+
The output goes to `./mimic-demo-multimodal`; change it with `--out DIR`.
|
|
97
|
+
|
|
98
|
+
**Signing in.** Only the credentialed sources need it. The first of these that
|
|
99
|
+
is set is used:
|
|
100
|
+
1. `--user NAME` (or `PHYSIONET_USER` / `PHYSIONET_USERNAME`), with the
|
|
101
|
+
password from `PHYSIONET_PASSWORD` or asked once;
|
|
102
|
+
2. `PHYSIONET_TOKEN`, a personal access token from PhysioNet (Settings > Tokens),
|
|
103
|
+
for unattended runs;
|
|
104
|
+
3. the sign-in saved by `physionet login`;
|
|
105
|
+
4. otherwise the tool asks for a username and password.
|
|
106
|
+
|
|
107
|
+
Credentials are only sent to physionet.org, and this tool stores none of them.
|
|
108
|
+
`physionet login` keeps its sign-in in `~/.config/physionet/credentials.json`
|
|
109
|
+
(the `physionet` package manages that file).
|
|
110
|
+
|
|
111
|
+
The run goes through four stages:
|
|
112
|
+
|
|
113
|
+
1. **Listing.** It reads each project's `SHA256SUMS.txt`, which gives exact
|
|
114
|
+
paths and checksums. Lists with hundreds of thousands of files are filtered
|
|
115
|
+
to the demo patients as they stream in. A project without that file falls
|
|
116
|
+
back to its folder listings.
|
|
117
|
+
2. **Plan.** For each source it shows how many files there are, how many are
|
|
118
|
+
already on disk, and how many still need fetching. It also shows the size
|
|
119
|
+
to store and your free space, then asks before starting. `--yes` skips the
|
|
120
|
+
question; `--plan` stops here.
|
|
121
|
+
3. **Download.**
|
|
122
|
+
- **Files** are downloaded by `physionet.download`, one source at a time,
|
|
123
|
+
with one overall progress bar. Each file is checked against PhysioNet's
|
|
124
|
+
checksum. Open projects that are on the AWS open-data mirror come from
|
|
125
|
+
there (`--source auto`, the default); `--source physionet` always uses
|
|
126
|
+
physionet.org.
|
|
127
|
+
- **Interrupted files** resume where they stopped. Ctrl-C is safe: run the
|
|
128
|
+
same command again to continue.
|
|
129
|
+
- **Tables** are streamed and filtered on the fly, keeping only demo-patient
|
|
130
|
+
rows. Each table gets its own progress bar, and the full tables never
|
|
131
|
+
touch your disk. The `physionet` package can only download whole files,
|
|
132
|
+
so this part, and projects without a `SHA256SUMS.txt`, use the tool's own
|
|
133
|
+
code with the package's sign-in.
|
|
134
|
+
4. **File check.** Every expected file is verified against its checksum.
|
|
135
|
+
- **Per-source result:** a table of ok, missing, partial and corrupt counts,
|
|
136
|
+
with COMPLETE or INCOMPLETE for each source.
|
|
137
|
+
- **Coverage:** how many demo patients have each modality.
|
|
138
|
+
|
|
139
|
+
### Local copies instead of downloading
|
|
140
|
+
|
|
141
|
+
If you already have a full project on disk, extract the demo subset from it:
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
uv run multi-mimic download \
|
|
145
|
+
--local mimic-iv-note=/data/mimic-iv-note/2.2 \
|
|
146
|
+
--local mimic-cxr-jpg=/data/mimic-cxr-jpg/2.1.0 \
|
|
147
|
+
--link hardlink # or symlink; default is copy
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
- **Where checksums come from:** the local `SHA256SUMS.txt` is used when
|
|
151
|
+
present. Otherwise the tool reads the folders directly, and only the demo
|
|
152
|
+
patients' parts of `files/`.
|
|
153
|
+
- **No password needed** when every credentialed source is local.
|
|
154
|
+
- **Saving space:** `--link hardlink` or `--link symlink` avoids storing
|
|
155
|
+
images twice. A hard link falls back to a copy when the two folders are on
|
|
156
|
+
different disks.
|
|
157
|
+
|
|
158
|
+
### Choosing sources
|
|
159
|
+
|
|
160
|
+
```bash
|
|
161
|
+
uv run multi-mimic sources # list them
|
|
162
|
+
uv run multi-mimic download --only mimic-iv-demo mimic-cxr-jpg
|
|
163
|
+
uv run multi-mimic download --skip mimic-iv-echo
|
|
164
|
+
uv run multi-mimic download --include mimic-cxr # add the DICOM release
|
|
165
|
+
uv run multi-mimic download --subjects 10000032 10039708 # only these demo patients
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
`--subjects` limits the per-patient files and filtered tables to the patients
|
|
169
|
+
you name, for a quick test or a partial copy. Sources fetched whole (the open
|
|
170
|
+
demos) stay whole.
|
|
171
|
+
|
|
172
|
+
### Checking later
|
|
173
|
+
|
|
174
|
+
```bash
|
|
175
|
+
uv run multi-mimic check # verify every checksum
|
|
176
|
+
uv run multi-mimic check --quick # sizes only
|
|
177
|
+
uv run multi-mimic check --fix # re-fetch anything missing, partial or corrupt
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
The exit code is 0 when complete, 1 when incomplete, and 2 on an error such as
|
|
181
|
+
a wrong password or an unsigned agreement. Paths of problem files go to
|
|
182
|
+
`multi-mimic.check-report.txt` next to the data.
|
|
183
|
+
|
|
184
|
+
### Overview figure
|
|
185
|
+
|
|
186
|
+
```bash
|
|
187
|
+
uv run --extra figure multi-mimic figure # writes docs/modality-overview-{light,dark}.svg
|
|
188
|
+
uv run --extra figure multi-mimic figure --readme README.md # and refreshes the overview above
|
|
189
|
+
```
|
|
190
|
+
|
|
191
|
+
It counts the files that pass a size check, so run it after a complete
|
|
192
|
+
download. Modalities are grouped:
|
|
193
|
+
- **ECG:** waveforms or machine reports
|
|
194
|
+
- **Chest X-ray:** JPG or DICOM
|
|
195
|
+
- **Echo:** DICOMs or measurements
|
|
196
|
+
- **Notes:** discharge summaries or radiology reports
|
|
197
|
+
|
|
198
|
+
EHR is left out of the co-occurrence plot because every demo patient has it.
|
|
199
|
+
The figure and table show patient counts only, never IDs.
|
|
200
|
+
|
|
201
|
+
**Small counts are hidden.** A count below 5 is shown as `<5`, with no bar.
|
|
202
|
+
Combinations with fewer than 5 patients are merged into one "other" column.
|
|
203
|
+
Change the threshold with `--min-count N`; `--min-count 1` shows everything.
|
|
204
|
+
|
|
205
|
+
## How and where data is saved
|
|
206
|
+
|
|
207
|
+
Everything goes into one folder (`--out`, default `./mimic-demo-multimodal`).
|
|
208
|
+
Nothing is written anywhere else: no cache, no temporary copies of full tables.
|
|
209
|
+
|
|
210
|
+
### Folder layout
|
|
211
|
+
|
|
212
|
+
One folder per project, named **`<project>-<version>`**, the name
|
|
213
|
+
`physionet.download` gives it. Inside, every path is the same as on PhysioNet.
|
|
214
|
+
|
|
215
|
+
```
|
|
216
|
+
mimic-demo-multimodal/
|
|
217
|
+
mimic-iv-demo-2.2/ hosp/ icu/ demo_subject_id.csv ... (whole project)
|
|
218
|
+
mimic-iv-ed-demo-2.2/ ed/ ... (whole project)
|
|
219
|
+
mimic-iv-ecg-demo-0.1/ files/p1000/p1000xxxx/s.../... (whole project)
|
|
220
|
+
mimic-iv-ecg-1.0/ machine_measurements.csv ... (demo rows only)
|
|
221
|
+
mimic4wdb-0.1.0/ waves/p100/p100xxxxx/<record>/... (demo patients only)
|
|
222
|
+
mimic-cxr-jpg-2.1.0/ files/p10/p10xxxxxx/s.../*.jpg (demo patients only)
|
|
223
|
+
mimic-cxr-2.0.0-metadata.csv.gz ... (demo rows only)
|
|
224
|
+
mimic-iv-echo-1.0.1/ files/... echo-record-list.csv ...
|
|
225
|
+
mimic-iv-note-2.2/ note/discharge.csv.gz ... (demo rows only)
|
|
226
|
+
multi-mimic.manifest.json what was planned and fetched
|
|
227
|
+
multi-mimic.check-report.txt only while the check finds problems
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
So a file at `https://physionet.org/files/mimic-iv-note/2.2/note/discharge.csv.gz`
|
|
231
|
+
is saved as `mimic-iv-note-2.2/note/discharge.csv.gz`.
|
|
232
|
+
|
|
233
|
+
### What each file is
|
|
234
|
+
|
|
235
|
+
| kind | what is saved | how it is verified |
|
|
236
|
+
|---|---|---|
|
|
237
|
+
| **Exact copy** | the file as published: everything in the open demos, and the demo patients' files under `files/` or `waves/` | PhysioNet's SHA-256 checksum |
|
|
238
|
+
| **Filtered table** | a top-level table (`.csv` / `.csv.gz`) with a `subject_id` column: same name, same columns, same compression, only the demo patients' rows | the checksum of the filtered file, recorded in the manifest when it was written |
|
|
239
|
+
| **Kept whole** | a table without a `subject_id` column, 200 MB or smaller (data dictionaries, the echo study list) | PhysioNet's checksum |
|
|
240
|
+
| **Not saved** | other top-level files over 10 MB (such as `IMAGE_FILENAMES`), and tables without a `subject_id` column over 200 MB | listed in the manifest as skipped |
|
|
241
|
+
|
|
242
|
+
### While downloading
|
|
243
|
+
|
|
244
|
+
- **Filtered tables** are written to `<name>.part` and renamed when complete,
|
|
245
|
+
so a table under its real name is always complete.
|
|
246
|
+
- **Files from `physionet.download`** are written under their final name. If a
|
|
247
|
+
download is interrupted, a shorter file stays there until the next run
|
|
248
|
+
resumes it. `check` reports such a file as *partial*.
|
|
249
|
+
- **Files from local copies** (`--local`) are copied, hard-linked or symlinked
|
|
250
|
+
(`--link`). A symlink points into your local copy, so moving that copy breaks it.
|
|
251
|
+
|
|
252
|
+
### The manifest
|
|
253
|
+
|
|
254
|
+
`multi-mimic.manifest.json` lists every item the tool planned, per source:
|
|
255
|
+
- the path inside the project folder, and whether it's an exact copy or a
|
|
256
|
+
filtered table;
|
|
257
|
+
- PhysioNet's checksum and size, and its status (done, skipped or failed);
|
|
258
|
+
- for a filtered table: the checksum and size of the filtered file, how many
|
|
259
|
+
rows were kept of how many, and which demo patients it covers.
|
|
260
|
+
|
|
261
|
+
`check` and `figure` read it. Because it names demo patients and their files,
|
|
262
|
+
treat it as credentialed data, like the rest of the folder.
|
|
263
|
+
|
|
264
|
+
### Upgrading from mimic-demo-mm
|
|
265
|
+
|
|
266
|
+
Before its first release, this tool was called `mimic-demo-mm` and saved projects
|
|
267
|
+
as `<project>/<version>/` (for example `mimic4wdb/0.1.0/`). It now uses
|
|
268
|
+
`<project>-<version>/` to match the `physionet` package. The tool never renames folders on its own. If it finds the
|
|
269
|
+
old layout, `download`, `check` and `figure` stop and tell you which folders are
|
|
270
|
+
affected. Then:
|
|
271
|
+
|
|
272
|
+
```bash
|
|
273
|
+
uv run multi-mimic migrate --out DIR --dry-run # show what would be renamed
|
|
274
|
+
uv run multi-mimic migrate --out DIR # rename them
|
|
275
|
+
```
|
|
276
|
+
|
|
277
|
+
Only the project folders are renamed: nothing inside them changes. Update any
|
|
278
|
+
code that reads the old paths. `migrate` only works on a folder that has this
|
|
279
|
+
tool's manifest, so it never touches a copy of PhysioNet made some other way.
|
|
280
|
+
|
|
281
|
+
`mimic-demo-mm` also named its files `mimic-demo-mm.manifest.json` and `mimic-demo-mm.check-report.txt`. The tool
|
|
282
|
+
still reads a manifest under the old name and renames it to
|
|
283
|
+
`multi-mimic.manifest.json` the next time it saves.
|
|
284
|
+
|
|
285
|
+
## Privacy and data use
|
|
286
|
+
|
|
287
|
+
- The console shows **counts only**: never IDs, paths of individual patients, or note text.
|
|
288
|
+
- Everything under the output folder, apart from the open demo sources, is **credentialed data**.
|
|
289
|
+
- **Keep it on approved machines.** The PhysioNet Credentialed Health Data
|
|
290
|
+
Use Agreement applies to the filtered tables and the manifest too.
|
|
291
|
+
- **Don't upload it to online AI services.** See PhysioNet's guidance:
|
|
292
|
+
https://physionet.org/news/post/llm-responsible-use/
|
|
293
|
+
- **Cite every dataset you use:** see [Citing](#citing).
|
|
294
|
+
|
|
295
|
+
## Citing
|
|
296
|
+
|
|
297
|
+
Cite every dataset you use, as its PhysioNet page asks. The citations below are copied from those pages (retrieved 2026-11-13); check the page for the version you use.
|
|
298
|
+
|
|
299
|
+
**PhysioNet** (every project page asks for this standard citation):
|
|
300
|
+
|
|
301
|
+
- Pollard, T., Moody, B. E., Lehman, L., Gow, B., Fernandes, C., Xie, C., Johnson, A., Mark, R. G., & Heldt, T. (2026). PhysioNet as a global platform for biomedical research. Nature Health. <https://doi.org/10.1038/s44360-026-00096-z>. Available from: <https://pmc.ncbi.nlm.nih.gov/articles/PMC13641116/>
|
|
302
|
+
|
|
303
|
+
**The datasets this tool downloads:**
|
|
304
|
+
|
|
305
|
+
- **[MIMIC-IV Clinical Database Demo, version 2.2](https://physionet.org/content/mimic-iv-demo/2.2/)** (`mimic-iv-demo`)
|
|
306
|
+
- Johnson, A., Bulgarelli, L., Pollard, T., Horng, S., Celi, L. A., & Mark, R. (2023). MIMIC-IV Clinical Database Demo (version 2.2). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/dp1f-ex47>
|
|
307
|
+
- Derived from [MIMIC-IV 1.0](https://physionet.org/content/mimiciv/1.0/): the page asks you to cite it too (see below).
|
|
308
|
+
|
|
309
|
+
- **[MIMIC-IV-ED Demo, version 2.2](https://physionet.org/content/mimic-iv-ed-demo/2.2/)** (`mimic-iv-ed-demo`)
|
|
310
|
+
- Johnson, A., Bulgarelli, L., Pollard, T., Celi, L. A., Horng, S., & Mark, R. (2023). MIMIC-IV-ED Demo (version 2.2). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/jzz5-vs76>
|
|
311
|
+
- Derived from [MIMIC-IV-ED 2.2](https://physionet.org/content/mimic-iv-ed/2.2/): the page asks you to cite it too (see below).
|
|
312
|
+
|
|
313
|
+
- **[MIMIC-IV-ECG Demo - Diagnostic Electrocardiogram Matched Subset Demo, version 0.1](https://physionet.org/content/mimic-iv-ecg-demo/0.1/)** (`mimic-iv-ecg-demo`)
|
|
314
|
+
- Gow, B., Pollard, T., Nathanson, L. A., Moody, B., Johnson, A., Moukheiber, D., Greenbaum, N., Berkowitz, S., Eslami, P., Herbst, E., Mark, R., & Horng, S. (2022). MIMIC-IV-ECG Demo - Diagnostic Electrocardiogram Matched Subset Demo (version 0.1). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/4eqn-kt76>
|
|
315
|
+
- Derived from [MIMIC-IV 2.0](https://physionet.org/content/mimiciv/2.0/), [MIMIC-IV Clinical Database Demo 1.0](https://physionet.org/content/mimic-iv-demo/1.0/): the page asks you to cite them too (see below).
|
|
316
|
+
|
|
317
|
+
- **[MIMIC-IV-ECG: Diagnostic Electrocardiogram Matched Subset, version 1.0](https://physionet.org/content/mimic-iv-ecg/1.0/)** (`mimic-iv-ecg`)
|
|
318
|
+
- Gow, B., Pollard, T., Nathanson, L. A., Johnson, A., Moody, B., Fernandes, C., Greenbaum, N., Waks, J. W., Eslami, P., Carbonati, T., Chaudhari, A., Herbst, E., Moukheiber, D., Berkowitz, S., Mark, R., & Horng, S. (2023). MIMIC-IV-ECG: Diagnostic Electrocardiogram Matched Subset (version 1.0). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/4nqg-sb35>
|
|
319
|
+
- Derived from [MIMIC-IV 2.2](https://physionet.org/content/mimiciv/2.2/): the page asks you to cite it too (see below).
|
|
320
|
+
|
|
321
|
+
- **[MIMIC-IV Waveform Database, version 0.1.0](https://physionet.org/content/mimic4wdb/0.1.0/)** (`mimic4wdb`)
|
|
322
|
+
- Moody, B., Hao, S., Gow, B., Pollard, T., Zong, W., & Mark, R. (2022). MIMIC-IV Waveform Database (version 0.1.0). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/a2mw-f949>
|
|
323
|
+
- Derived from [MIMIC-IV 2.0](https://physionet.org/content/mimiciv/2.0/): the page asks you to cite it too (see below).
|
|
324
|
+
|
|
325
|
+
- **[MIMIC-CXR-JPG - chest radiographs with structured labels, version 2.1.0](https://physionet.org/content/mimic-cxr-jpg/2.1.0/)** (`mimic-cxr-jpg`)
|
|
326
|
+
- Johnson, A., Lungren, M., Peng, Y., Lu, Z., Mark, R., Berkowitz, S., & Horng, S. (2024). MIMIC-CXR-JPG - chest radiographs with structured labels (version 2.1.0). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/jsn5-t979>
|
|
327
|
+
- Original publication, which the page also asks you to cite: Johnson AE, Pollard TJ, Berkowitz S, Greenbaum NR, Lungren MP, Deng CY, Mark RG, Horng S. MIMIC-CXR: A large publicly available database of labeled chest radiographs. arXiv preprint arXiv:1901.07042. 2019 Jan 21. <https://arxiv.org/abs/1901.07042>
|
|
328
|
+
- Derived from [MIMIC-CXR Database 2.0.0](https://physionet.org/content/mimic-cxr/2.0.0/): the page asks you to cite it too (see below).
|
|
329
|
+
|
|
330
|
+
- **[MIMIC-IV-Echo: Echocardiogram Matched Subset, version 1.0.1](https://physionet.org/content/mimic-iv-echo/1.0.1/)** (`mimic-iv-echo`)
|
|
331
|
+
- Gow, B., Pollard, T., Greenbaum, N., Moody, B., Han, A., Waks, J. W., Johnson, A., Herbst, E., Eslami, P., Chaudhari, A., Carbonati, T., Berkowitz, S., Mark, R., & Horng, S. (2026). MIMIC-IV-Echo: Echocardiogram Matched Subset (version 1.0.1). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/307c-mr50>
|
|
332
|
+
- Derived from [MIMIC-IV 2.2](https://physionet.org/content/mimiciv/2.2/), [MIMIC-IV 3.1](https://physionet.org/content/mimiciv/3.1/): the page asks you to cite them too (see below).
|
|
333
|
+
|
|
334
|
+
- **[MIMIC-IV-Note: Deidentified free-text clinical notes, version 2.2](https://physionet.org/content/mimic-iv-note/2.2/)** (`mimic-iv-note`)
|
|
335
|
+
- Johnson, A., Pollard, T., Horng, S., Celi, L. A., & Mark, R. (2023). MIMIC-IV-Note: Deidentified free-text clinical notes (version 2.2). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/1n74-ne17>
|
|
336
|
+
|
|
337
|
+
- **[MIMIC-CXR Database, version 2.0.0](https://physionet.org/content/mimic-cxr/2.0.0/)** (`mimic-cxr`)
|
|
338
|
+
- Johnson, A., Pollard, T., Mark, R., Berkowitz, S., & Horng, S. (2019). MIMIC-CXR Database (version 2.0.0). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/C2JT1Q>
|
|
339
|
+
- Original publication, which the page also asks you to cite: Johnson, A.E.W., Pollard, T.J., Berkowitz, S.J. et al. MIMIC-CXR, a de-identified publicly available database of chest radiographs with free-text reports. Sci Data 6, 317 (2019). <https://doi.org/10.1038/s41597-019-0322-0>
|
|
340
|
+
|
|
341
|
+
**Projects the datasets above were derived from** (their pages list them under "Parent Projects", with "Please cite them when using this project"):
|
|
342
|
+
|
|
343
|
+
- **[MIMIC-CXR Database, version 2.0.0](https://physionet.org/content/mimic-cxr/2.0.0/)**: listed above.
|
|
344
|
+
- **[MIMIC-IV Clinical Database Demo, version 1.0](https://physionet.org/content/mimic-iv-demo/1.0/)**: Johnson, A., Bulgarelli, L., Pollard, T., Horng, S., Celi, L. A., & Mark, R. (2022). MIMIC-IV Clinical Database Demo (version 1.0). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/jwtp-v091>
|
|
345
|
+
- **[MIMIC-IV-ED, version 2.2](https://physionet.org/content/mimic-iv-ed/2.2/)**: Johnson, A., Bulgarelli, L., Pollard, T., Celi, L. A., Mark, R., & Horng, S. (2023). MIMIC-IV-ED (version 2.2). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/5ntk-km72>
|
|
346
|
+
- **[MIMIC-IV, version 1.0](https://physionet.org/content/mimiciv/1.0/)**: Johnson, A., Bulgarelli, L., Pollard, T., Horng, S., Celi, L. A., & Mark, R. (2021). MIMIC-IV (version 1.0). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/s6n6-xd98>
|
|
347
|
+
- **[MIMIC-IV, version 2.0](https://physionet.org/content/mimiciv/2.0/)**: Johnson, A., Bulgarelli, L., Pollard, T., Horng, S., Celi, L. A., & Mark, R. (2022). MIMIC-IV (version 2.0). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/7vcr-e114>
|
|
348
|
+
- **[MIMIC-IV, version 2.2](https://physionet.org/content/mimiciv/2.2/)**: Johnson, A., Bulgarelli, L., Pollard, T., Horng, S., Celi, L. A., & Mark, R. (2023). MIMIC-IV (version 2.2). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/6mm1-ek67>
|
|
349
|
+
- Original publication, which the page also asks you to cite: Johnson, A.E.W., Bulgarelli, L., Shen, L. et al. MIMIC-IV, a freely accessible electronic health record dataset. Sci Data 10, 1 (2023). <https://doi.org/10.1038/s41597-022-01899-x>
|
|
350
|
+
- **[MIMIC-IV, version 3.1](https://physionet.org/content/mimiciv/3.1/)**: Johnson, A., Bulgarelli, L., Pollard, T., Gow, B., Moody, B., Horng, S., Celi, L. A., & Mark, R. (2024). MIMIC-IV (version 3.1). PhysioNet. RRID:SCR_007345. <https://doi.org/10.13026/kpb9-mt58>
|
|
351
|
+
- Original publication, which the page also asks you to cite: Johnson, A.E.W., Bulgarelli, L., Shen, L. et al. MIMIC-IV, a freely accessible electronic health record dataset. Sci Data 10, 1 (2023). <https://doi.org/10.1038/s41597-022-01899-x>
|
|
352
|
+
|
|
353
|
+
## Development
|
|
354
|
+
|
|
355
|
+
```bash
|
|
356
|
+
uv run --extra test pytest -q
|
|
357
|
+
```
|
|
358
|
+
|
|
359
|
+
The tests run end to end against `tests/mock_physionet.py`, a local stand-in
|
|
360
|
+
for PhysioNet with synthetic data. The real `physionet` package downloads
|
|
361
|
+
from it. It reproduces:
|
|
362
|
+
- credentialed-project passwords, including a rejected password and an account
|
|
363
|
+
with no signed agreement;
|
|
364
|
+
- resumable downloads, size checks and folder listings, and the API endpoint
|
|
365
|
+
that lists a project's versions;
|
|
366
|
+
- a project without checksums;
|
|
367
|
+
- a file whose connection drops halfway.
|
|
368
|
+
|
|
369
|
+
**Releasing.** Push a tag such as `v0.2.0`. The `python-build.yaml` workflow
|
|
370
|
+
builds the package on every push. For a tag, it also publishes to PyPI through
|
|
371
|
+
trusted publishing, signs the files with Sigstore, and creates a GitHub release.
|
|
372
|
+
The version comes from the tag (hatch-vcs).
|
|
373
|
+
|
|
374
|
+
**Live test.** `tests/test_live.py` runs against the real PhysioNet and uses
|
|
375
|
+
about 2 MB. It takes one to two minutes, mostly spent reading the waveform
|
|
376
|
+
checksum list. It is off by default:
|
|
377
|
+
|
|
378
|
+
```bash
|
|
379
|
+
MULTI_MIMIC_LIVE=1 uv run --extra test pytest tests/test_live.py -v
|
|
380
|
+
```
|
|
381
|
+
|
|
382
|
+
It downloads the MIMIC-IV-ED demo (about 120 KB) through `physionet.download`,
|
|
383
|
+
checks it, confirms a second run fetches nothing, and draws the figure. It also
|
|
384
|
+
plans one waveform patient without downloading their files. With a PhysioNet
|
|
385
|
+
sign-in it also plans MIMIC-IV-Note, reading only each table's header.
|
|
386
|
+
|
|
387
|
+
The `live.yaml` workflow runs it weekly and on demand, with open data only.
|
|
388
|
+
Never give that workflow a PhysioNet sign-in: credentialed data must not be
|
|
389
|
+
downloaded to GitHub's machines.
|