Multi-MIMIC 0.0.1__py3-none-any.whl

multi_mimic/__init__.py

@@ -0,0 +1,7 @@
+"""Download the MIMIC-IV demo patients across modalities from PhysioNet."""
+from importlib.metadata import PackageNotFoundError, version
+
+try:
+    __version__ = version("Multi-MIMIC")   # set from the git tag when the package is built
+except PackageNotFoundError:
+    __version__ = "0.0.0"

multi_mimic/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

multi_mimic/check.py

@@ -0,0 +1,145 @@
+"""Final file check: is everything there, complete and uncorrupted?
+
+Every file is compared with its PhysioNet checksum (SHA256SUMS.txt); every
+filtered table with the checksum recorded when it was written. Results are
+printed as counts only; the paths of problem files go to a local report."""
+from __future__ import annotations
+
+import gzip
+import csv
+from collections import Counter, defaultdict
+from pathlib import Path
+
+from .download import bar, human, local_path
+from .listing import subject_of
+from .manifest import Item, Manifest
+from .sources import BY_KEY
+from .tables import sha256_file
+
+OK, MISSING, INCOMPLETE, CORRUPT, SKIPPED = "ok", "missing", "incomplete", "corrupt", "skipped"
+REPORT = "multi-mimic.check-report.txt"
+LEGACY_REPORT = "mimic-demo-mm.check-report.txt"   # written by the pre-release mimic-demo-mm
+
+
+def verify_item(root: Path, it: Item, quick: bool) -> str:
+    if it.status == "skipped":
+        return SKIPPED
+    p = local_path(root, it)
+    if not p.exists():
+        return INCOMPLETE if p.with_name(p.name + ".part").exists() else MISSING
+    size = p.stat().st_size
+    if it.kind == "table":
+        if it.out_sha256 is None:
+            return MISSING
+        if size != it.out_size:
+            return CORRUPT
+        return OK if quick or sha256_file(p) == it.out_sha256 else CORRUPT
+    if it.size is not None and size != it.size:
+        return INCOMPLETE if size < it.size else CORRUPT
+    if not quick and it.sha256 and sha256_file(p) != it.sha256:
+        return CORRUPT
+    return OK
+
+
+def verify(root: Path, man: Manifest, keys: list[str] | None = None, quick: bool = False) -> dict[tuple, str]:
+    items = [i for i in man.items.values() if keys is None or i.source in keys]
+    total = sum((i.out_size if i.kind == "table" else i.size) or 0 for i in items if i.status != "skipped")
+    res = {}
+    with bar(total=total, unit="B", unit_scale=True, desc="  checking files" + (" (sizes only)" if quick else ""),
+             leave=False) as b:
+        for it in items:
+            res[it.key] = verify_item(root, it, quick)
+            b.update((it.out_size if it.kind == "table" else it.size) or 0)
+    return res
+
+
+def _ed_subjects(root: Path) -> set[str]:
+    p = root / BY_KEY["mimic-iv-ed-demo"].rel / "ed/edstays.csv.gz"
+    if not p.exists():
+        return set()
+    with gzip.open(p, "rt", newline="") as f:
+        r = csv.DictReader(f)
+        return {row["subject_id"] for row in r}
+
+
+def coverage(root: Path, man: Manifest, res: dict) -> list[tuple[str, set[str] | None, str]]:
+    """-> [(label, demo patients with it or None for all of them, modality group)]."""
+    ok = [man.items[k] for k, v in res.items() if v == OK]
+    present = {src for src, _ in res}   # only the sources that were checked
+
+    def files_of(src, exts):
+        return {subject_of(i.path) for i in ok if i.source == src and BY_KEY[src].in_files(i.path)
+                and i.path.lower().endswith(exts)} - {None}
+
+    def tables_of(src, name):
+        s = set()
+        for i in ok:
+            if i.source == src and i.kind == "table" and name in Path(i.path).name:
+                s.update(i.subjects)
+        return s
+
+    rows = []
+    if "mimic-iv-demo" in present:
+        rows.append(("structured, categorical and numeric events (EHR)", None, "EHR"))
+    if "mimic-iv-ed-demo" in present:
+        rows.append(("emergency department stays", _ed_subjects(root), "ED stay"))
+    for key, label, kind, arg, group in (
+            ("mimic-iv-ecg-demo", "ECG waveforms", "files", (".hea",), "ECG"),
+            ("mimic-iv-ecg", "ECG machine reports (text)", "tables", "machine_measurements", "ECG"),
+            ("mimic4wdb", "ICU bedside-monitor waveforms", "files", (".hea",), "ICU waveforms"),
+            ("mimic-cxr-jpg", "chest X-ray images (JPG)", "files", (".jpg",), "Chest X-ray"),
+            ("mimic-cxr", "chest X-ray DICOMs", "files", (".dcm",), "Chest X-ray"),
+            ("mimic-iv-echo", "echo DICOMs", "files", (".dcm",), "Echo"),
+            ("mimic-iv-echo", "echo measurements", "tables", "measurement", "Echo"),
+            ("mimic-iv-note", "discharge summaries", "tables", "discharge.csv", "Notes"),
+            ("mimic-iv-note", "radiology reports", "tables", "radiology.csv", "Notes")):
+        if key in present:
+            rows.append((label, files_of(key, arg) if kind == "files" else tables_of(key, arg), group))
+    return rows
+
+
+def report(root: Path, man: Manifest, res: dict, quick: bool) -> bool:
+    per = defaultdict(Counter)
+    sizes = Counter()
+    for (src, _), st in res.items():
+        per[src][st] += 1
+        it = man.items[(src, _)]
+        if st == OK:
+            sizes[src] += (it.out_size if it.kind == "table" else it.size) or 0
+    print(f"\n== File check{' (sizes only)' if quick else ' (checksums)'} ==")
+    print(f"  {'source':<18}{'ok':>8}{'missing':>9}{'partial':>9}{'corrupt':>9}{'skipped':>9}{'size':>11}  status")
+    complete = True
+    checked = {src for src, _ in res}
+    for key in [k for k in man.sources if k in checked]:
+        c = per.get(key, Counter())
+        bad = c[MISSING] + c[INCOMPLETE] + c[CORRUPT]
+        complete &= bad == 0 and c[OK] > 0
+        status = "complete" if bad == 0 and c[OK] else ("EMPTY" if not c[OK] and not bad else "INCOMPLETE")
+        print(f"  {key:<18}{c[OK]:>8,}{c[MISSING]:>9,}{c[INCOMPLETE]:>9,}{c[CORRUPT]:>9,}{c[SKIPPED]:>9,}"
+              f"{human(sizes[key]):>11}  {status}")
+    n_demo = man.demo_subjects or 0
+    print(f"\n== Demo patients per modality (of {n_demo}) ==")
+    rows = coverage(root, man, res)
+    for label, s, _ in rows:
+        print(f"  {label:<48}{n_demo if s is None else len(s):>5}")
+    optional = ("emergency department stays", "ICU bedside-monitor waveforms")   # few patients have these
+    sets = [s for label, s, _ in rows if s is not None and label not in optional]
+    if len(sets) > 1:
+        label = "with all of the above (ED, waveforms optional)"
+        print(f"  {label:<48}{len(set.intersection(*sets)):>5}")
+
+    bad = [(k, v) for k, v in res.items() if v in (MISSING, INCOMPLETE, CORRUPT)]
+    rep = root / REPORT
+    (root / LEGACY_REPORT).unlink(missing_ok=True)   # replaced by the report below, if any
+    if bad:
+        with open(rep, "w") as f:
+            for (src, path), st in sorted(bad):
+                note = man.items[(src, path)].note
+                f.write(f"{st}\t{BY_KEY[src].rel}/{path}\t{note}\n")
+        print(f"\nINCOMPLETE: {len(bad):,} problem file(s), listed in {rep} (local file).")
+        print("Run `multi-mimic check --fix` (or the same download command again) to fetch them.")
+    else:
+        rep.unlink(missing_ok=True)
+        print("\nCOMPLETE: every expected file is present and matches its checksum." if not quick else
+              "\nCOMPLETE (sizes only): run `multi-mimic check` for full checksum verification.")
+    return complete and not bad

multi_mimic/cli.py

@@ -0,0 +1,235 @@
+"""Multi-MIMIC: assemble a local multimodal MIMIC-IV demo.
+
+  multi-mimic sources                         list the datasets
+  multi-mimic download --user NAME            download everything (asks for the password)
+  multi-mimic download --local mimic-iv-note=/data/mimic-iv-note/2.2 ...
+                                                take subsets from local copies instead
+  multi-mimic check [--fix]                   verify every file; re-fetch broken ones
+  multi-mimic figure [--readme README.md]     draw the modality coverage / co-occurrence overview
+  multi-mimic migrate [--dry-run]             rename folders from the mimic-demo-mm layout
+"""
+from __future__ import annotations
+
+import argparse
+import getpass
+import os
+import sys
+from pathlib import Path
+
+from . import __version__
+from .check import report, verify
+from .download import (Runner, demo_ids, make_origins, migrate_layout, plan, require_current_layout,
+                       summarize_plan)
+from .http import DEFAULT_BASE_URL, Client, PhysioNetError, base_url
+from .manifest import Manifest, manifest_file
+from .sources import BY_KEY, SOURCES, select
+
+
+def _local_map(pairs: list[str] | None, man: Manifest | None = None) -> dict[str, Path]:
+    out = {}
+    if man is not None:  # remember local copies used by earlier runs (for check --fix)
+        out.update({k: Path(v["local_dir"]) for k, v in man.sources.items() if v.get("local_dir")})
+    for p in pairs or []:
+        if "=" not in p:
+            raise SystemExit(f"--local expects SOURCE=FOLDER, got '{p}'")
+        k, d = p.split("=", 1)
+        if k not in BY_KEY:
+            raise SystemExit(f"--local: unknown source '{k}'. See: multi-mimic sources")
+        out[k] = Path(d)
+    return out
+
+
+def _client(sources, local, user) -> Client:
+    """Sign-in for credentialed sources, in this order: --user (or PHYSIONET_USER /
+    PHYSIONET_USERNAME) with PHYSIONET_PASSWORD or a prompt; PHYSIONET_TOKEN; the
+    sign-in saved by `physionet login`; else ask for a username and password."""
+    remote_cred = [s.key for s in sources if s.credentialed and s.key not in local]
+    if not remote_cred:
+        return Client()
+    user = user or os.environ.get("PHYSIONET_USER") or os.environ.get("PHYSIONET_USERNAME")
+    if not user:
+        if os.environ.get("PHYSIONET_TOKEN"):
+            return Client(token=os.environ["PHYSIONET_TOKEN"])
+        from physionet.auth import load_saved_login
+        if load_saved_login(base_url()) is not None:
+            return Client()   # the physionet package uses the saved sign-in
+        print("  (tip: `uv run physionet login` signs in once, without a password here)")
+        user = input("PhysioNet username: ").strip()
+    pw = os.environ.get("PHYSIONET_PASSWORD") or getpass.getpass(f"PhysioNet password for {user}: ")
+    return Client(user, pw)
+
+
+def cmd_sources(a) -> int:
+    print(f"{'source':<19}{'access':<14}{'what is fetched':<26}{'default':<9}description")
+    what = {"full": "everything", "demo": "demo patients' files+rows", "tables": "demo patients' rows"}
+    for s in SOURCES:
+        print(f"{s.key:<19}{s.access:<14}{what[s.mode]:<26}{'yes' if s.default else 'no':<9}{s.title}")
+    print("\nPhysioNet project: https://physionet.org/content/<source>/<version>/  "
+          "(versions: " + ", ".join(f"{s.key} {s.version}" for s in SOURCES) + ")")
+    return 0
+
+
+def _download(root: Path, sources, local, link, user, jobs, yes, plan_only, quick,
+              source="auto", subjects=None) -> int:
+    root.mkdir(parents=True, exist_ok=True)
+    require_current_layout(root)
+    client = _client(sources, local, user)
+    origins = make_origins(sources, client, local, link, root)
+    man = Manifest.load(root)
+    ids = demo_ids(client, root, local)
+    if subjects:
+        unknown = set(subjects) - ids
+        if unknown:
+            raise SystemExit(f"--subjects: {len(unknown)} ID(s) are not MIMIC-IV demo patients")
+        ids = set(subjects)
+    man.demo_subjects = len(ids)
+    print(f"Multi-MIMIC {__version__}: {len(ids)} demo patients -> {root}\n\nListing files")
+    items = plan(origins, sources, ids, man, jobs)
+    files, tables, store, free = summarize_plan(root, sources, items, origins)
+    man.save()
+    if plan_only:
+        return 0
+    if store > free:
+        print("  x not enough free space: choose fewer sources (--only/--skip), use --link with --local, "
+              "or --out on a bigger disk")
+        return 2
+    if (files or tables) and not yes:
+        if input("\n  Start? [Y/n] ").strip().lower() in ("n", "no"):
+            print("  Nothing downloaded. Re-run any time.")
+            return 0
+    if files or tables:
+        print()
+        if client.base_url != DEFAULT_BASE_URL:
+            source = "physionet"   # a test server or mirror: never the AWS copy of the real projects
+        Runner(origins, root, man, ids, jobs, client=client, source=source).run(files, tables)
+    res = verify(root, man, [s.key for s in sources], quick)
+    return 0 if report(root, man, res, quick) else 1
+
+
+def cmd_download(a) -> int:
+    sources = select(a.only, a.skip, a.include)
+    local = _local_map(a.local)
+    return _download(Path(a.out).resolve(), sources, local, a.link, a.user, a.jobs, a.yes, a.plan, a.quick_check,
+                     a.source, a.subjects)
+
+
+def cmd_check(a) -> int:
+    root = Path(a.out).resolve()
+    require_current_layout(root)
+    man = Manifest.load(root)
+    if not man.items:
+        print(f"No manifest in {root}. Run `multi-mimic download` first (or pass --out).")
+        return 2
+    res = verify(root, man, None, a.quick)
+    ok = report(root, man, res, a.quick)
+    if ok or not a.fix:
+        return 0 if ok else 1
+    bad = {k for k, v in res.items() if v in ("missing", "incomplete", "corrupt")}
+    for k in bad:
+        it = man.items[k]
+        it.status = "pending"
+        p = root / BY_KEY[it.source].rel / it.path
+        if res[k] == "corrupt" and p.exists():
+            p.unlink()
+    print(f"\nFixing {len(bad):,} file(s)...")
+    sources = [BY_KEY[k] for k in man.sources if k in BY_KEY]
+    local = _local_map(a.local, man)
+    return _download(root, sources, local, a.link, a.user, a.jobs, True, False, False, a.source)
+
+
+def cmd_migrate(a) -> int:
+    root = Path(a.out).resolve()
+    if manifest_file(root) is None:
+        print(f"x {root} has no Multi-MIMIC manifest: not a folder this tool made, nothing changed")
+        return 2
+    migrate_layout(root, a.dry_run)
+    return 0
+
+
+def cmd_figure(a) -> int:
+    try:
+        import matplotlib  # noqa: F401
+    except ImportError:
+        print("x the figure needs matplotlib: run `uv run --extra figure multi-mimic figure ...`")
+        return 2
+    from .figure import run
+    if a.min_count < 1:
+        print("x --min-count must be at least 1")
+        return 2
+    return run(Path(a.out).resolve(), Path(a.to), Path(a.readme) if a.readme else None, a.min_count)
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="multi-mimic", description=__doc__,
+                                 formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--version", action="version", version=__version__)
+    sub = ap.add_subparsers(dest="cmd", required=True)
+
+    sub.add_parser("sources", help="list the datasets").set_defaults(fn=cmd_sources)
+
+    def common(p):
+        p.add_argument("--out", default="mimic-demo-multimodal", help="output folder (default: %(default)s)")
+        p.add_argument("--user", help="PhysioNet username (or env PHYSIONET_USER; password: env PHYSIONET_PASSWORD "
+                                      "or prompt). Without it: PHYSIONET_TOKEN, or the sign-in saved by `physionet login`")
+        p.add_argument("--local", action="append", metavar="SOURCE=FOLDER",
+                       help="take this source from a local copy instead of PhysioNet (repeatable), "
+                            "e.g. mimic-iv-note=/data/mimic-iv-note/2.2")
+        p.add_argument("--link", choices=["copy", "hardlink", "symlink"], default="copy",
+                       help="for --local sources: copy files (default), or hard-link / symlink to save space")
+        p.add_argument("--jobs", type=int, default=4,
+                       help="parallel size checks, local copies and fallback downloads (default: %(default)s)")
+        p.add_argument("--source", choices=["auto", "physionet", "aws"], default="auto",
+                       help="where the physionet package downloads files from: auto (AWS open-data mirror "
+                            "when a project is on it, else PhysioNet), physionet or aws (default: %(default)s)")
+
+    d = sub.add_parser("download", help="download / extract everything, then check it")
+    common(d)
+    d.add_argument("--only", nargs="+", metavar="SOURCE", help="only these sources")
+    d.add_argument("--skip", nargs="+", metavar="SOURCE", help="leave these sources out")
+    d.add_argument("--include", nargs="+", metavar="SOURCE", help="add optional sources (e.g. mimic-cxr)")
+    d.add_argument("--yes", "-y", action="store_true", help="don't ask before starting")
+    d.add_argument("--plan", action="store_true", help="only list what would be fetched and how big it is")
+    d.add_argument("--quick-check", action="store_true", help="final check by size only (skip checksums)")
+    d.add_argument("--subjects", nargs="+", metavar="ID",
+                   help="only these demo patients (for a quick or partial download); full sources stay whole")
+    d.set_defaults(fn=cmd_download)
+
+    c = sub.add_parser("check", help="verify every file against its checksum")
+    common(c)
+    c.add_argument("--quick", action="store_true", help="sizes only (skip checksums)")
+    c.add_argument("--fix", action="store_true", help="re-fetch missing, partial or corrupt files")
+    c.set_defaults(fn=cmd_check)
+
+    m = sub.add_parser("migrate", help="rename folders from the mimic-demo-mm layout (<project>/<version>/) "
+                                       "to <project>-<version>/")
+    m.add_argument("--out", default="mimic-demo-multimodal", help="data folder (default: %(default)s)")
+    m.add_argument("--dry-run", action="store_true", help="only show what would be renamed")
+    m.set_defaults(fn=cmd_migrate)
+
+    g = sub.add_parser("figure", help="draw the modality coverage and co-occurrence overview (counts only)")
+    g.add_argument("--out", default="mimic-demo-multimodal", help="data folder (default: %(default)s)")
+    g.add_argument("--to", default="docs", help="folder for the SVGs (default: %(default)s)")
+    g.add_argument("--readme", help="also update the overview section of this README")
+    g.add_argument("--min-count", type=int, default=5,
+                   help="show smaller counts as '<N' and merge smaller combinations into 'other' "
+                        "(default: %(default)s; 1 shows everything)")
+    g.set_defaults(fn=cmd_figure)
+
+    a = ap.parse_args(argv)
+    try:
+        return a.fn(a)
+    except PhysioNetError as e:
+        print(f"x {e}")
+        return 2
+    except SystemExit as e:
+        if isinstance(e.code, str):   # a readable error message from deeper down
+            print(e.code)
+            return 2
+        raise
+    except KeyboardInterrupt:
+        print("\nStopped.")
+        return 130
+
+
+if __name__ == "__main__":
+    sys.exit(main())