wotann 0.5.80 → 0.5.83

package/skills/scientific/pydicom/scripts/extract_metadata.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""
+Extract and display DICOM metadata in a readable format.
+
+Usage:
+    python extract_metadata.py file.dcm
+    python extract_metadata.py file.dcm --output metadata.txt
+    python extract_metadata.py file.dcm --format json --output metadata.json
+"""
+
+import argparse
+import sys
+import json
+from pathlib import Path
+
+try:
+    import pydicom
+except ImportError:
+    print("Error: pydicom is not installed. Install it with: pip install pydicom")
+    sys.exit(1)
+
+
+def format_value(value):
+    """Format DICOM values for display."""
+    if isinstance(value, bytes):
+        try:
+            return value.decode('utf-8', errors='ignore')
+        except:
+            return str(value)
+    elif isinstance(value, pydicom.multival.MultiValue):
+        return ', '.join(str(v) for v in value)
+    elif isinstance(value, pydicom.sequence.Sequence):
+        return f"Sequence with {len(value)} item(s)"
+    else:
+        return str(value)
+
+
+def extract_metadata_text(ds, show_sequences=False):
+    """Extract metadata as formatted text."""
+    lines = []
+    lines.append("=" * 80)
+    lines.append("DICOM Metadata")
+    lines.append("=" * 80)
+
+    # File Meta Information
+    if hasattr(ds, 'file_meta'):
+        lines.append("\n[File Meta Information]")
+        for elem in ds.file_meta:
+            lines.append(f"{elem.name:40s} {format_value(elem.value)}")
+
+    # Patient Information
+    lines.append("\n[Patient Information]")
+    patient_tags = ['PatientName', 'PatientID', 'PatientBirthDate',
+                   'PatientSex', 'PatientAge', 'PatientWeight']
+    for tag in patient_tags:
+        if hasattr(ds, tag):
+            value = getattr(ds, tag)
+            lines.append(f"{tag:40s} {format_value(value)}")
+
+    # Study Information
+    lines.append("\n[Study Information]")
+    study_tags = ['StudyInstanceUID', 'StudyDate', 'StudyTime',
+                 'StudyDescription', 'AccessionNumber', 'StudyID']
+    for tag in study_tags:
+        if hasattr(ds, tag):
+            value = getattr(ds, tag)
+            lines.append(f"{tag:40s} {format_value(value)}")
+
+    # Series Information
+    lines.append("\n[Series Information]")
+    series_tags = ['SeriesInstanceUID', 'SeriesNumber', 'SeriesDescription',
+                  'Modality', 'SeriesDate', 'SeriesTime']
+    for tag in series_tags:
+        if hasattr(ds, tag):
+            value = getattr(ds, tag)
+            lines.append(f"{tag:40s} {format_value(value)}")
+
+    # Image Information
+    lines.append("\n[Image Information]")
+    image_tags = ['SOPInstanceUID', 'InstanceNumber', 'ImageType',
+                 'Rows', 'Columns', 'BitsAllocated', 'BitsStored',
+                 'PhotometricInterpretation', 'SamplesPerPixel',
+                 'PixelSpacing', 'SliceThickness', 'ImagePositionPatient',
+                 'ImageOrientationPatient', 'WindowCenter', 'WindowWidth']
+    for tag in image_tags:
+        if hasattr(ds, tag):
+            value = getattr(ds, tag)
+            lines.append(f"{tag:40s} {format_value(value)}")
+
+    # All other elements
+    if show_sequences:
+        lines.append("\n[All Elements]")
+        for elem in ds:
+            if elem.VR != 'SQ':  # Skip sequences for brevity
+                lines.append(f"{elem.name:40s} {format_value(elem.value)}")
+            else:
+                lines.append(f"{elem.name:40s} {format_value(elem.value)}")
+
+    return '\n'.join(lines)
+
+
+def extract_metadata_json(ds):
+    """Extract metadata as JSON."""
+    metadata = {}
+
+    # File Meta Information
+    if hasattr(ds, 'file_meta'):
+        metadata['file_meta'] = {}
+        for elem in ds.file_meta:
+            metadata['file_meta'][elem.keyword] = format_value(elem.value)
+
+    # All data elements (excluding sequences for simplicity)
+    metadata['dataset'] = {}
+    for elem in ds:
+        if elem.VR != 'SQ':
+            metadata['dataset'][elem.keyword] = format_value(elem.value)
+
+    return json.dumps(metadata, indent=2)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Extract and display DICOM metadata',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  python extract_metadata.py file.dcm
+  python extract_metadata.py file.dcm --output metadata.txt
+  python extract_metadata.py file.dcm --format json --output metadata.json
+  python extract_metadata.py file.dcm --show-sequences
+        """
+    )
+
+    parser.add_argument('input', type=str, help='Input DICOM file')
+    parser.add_argument('--output', '-o', type=str, help='Output file (default: print to console)')
+    parser.add_argument('--format', type=str, choices=['text', 'json'], default='text',
+                       help='Output format (default: text)')
+    parser.add_argument('--show-sequences', action='store_true',
+                       help='Include all data elements including sequences')
+
+    args = parser.parse_args()
+
+    # Validate input file exists
+    input_path = Path(args.input)
+    if not input_path.exists():
+        print(f"Error: Input file '{args.input}' not found")
+        sys.exit(1)
+
+    try:
+        # Read DICOM file
+        ds = pydicom.dcmread(args.input)
+
+        # Extract metadata
+        if args.format == 'json':
+            output = extract_metadata_json(ds)
+        else:
+            output = extract_metadata_text(ds, args.show_sequences)
+
+        # Write or print output
+        if args.output:
+            with open(args.output, 'w') as f:
+                f.write(output)
+            print(f"✓ Metadata extracted to: {args.output}")
+        else:
+            print(output)
+
+    except Exception as e:
+        print(f"✗ Error: {e}")
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

package/skills/scientific/pyhealth/SKILL.md

@@ -0,0 +1,123 @@
+---
+name: pyhealth
+description: Build clinical/healthcare deep-learning pipelines with PyHealth — loading EHR/signal/imaging datasets (MIMIC-III/IV, eICU, OMOP, SleepEDF, ChestXray14, EHRShot), defining tasks (mortality, readmission, length-of-stay, drug recommendation, sleep staging, ICD coding, EEG events), instantiating models (Transformer, RETAIN, GAMENet, SafeDrug, MICRON, StageNet, AdaCare, CNN/RNN/MLP), training with the PyHealth Trainer, computing clinical metrics, and using medical code utilities (ICD/ATC/NDC/RxNorm lookup and cross-mapping). Use this skill whenever the user mentions PyHealth, MIMIC, eICU, OMOP, EHR modeling, clinical prediction, drug recommendation, sleep staging, medical code mapping, ICD/ATC codes, or any healthcare ML pipeline that fits the dataset → task → model → trainer → metrics pattern, even if "PyHealth" isn't named explicitly.
+metadata:
+    skill-author: K-Dense Inc.
+---
+
+# PyHealth
+
+PyHealth (https://pyhealth.dev/) is a Python toolkit for clinical deep learning. It provides a unified, modular pipeline across electronic health records (EHR), physiological signals, and medical imaging.
+
+The library is built around a **5-stage pipeline** — `Dataset → Task → Model → Trainer → Metrics` — where each stage is replaceable and the interfaces between stages are stable. Code that follows this pipeline shape composes well; code that bypasses it usually fights the library.
+
+## When to use this skill
+
+Use this skill whenever the user is doing clinical/healthcare ML and any of the following are true:
+
+- They mention PyHealth, MIMIC-III/IV, eICU, OMOP-CDM, EHRShot, SleepEDF, SHHS, ISRUC, COVID19-CXR, ChestX-ray14, TUEV/TUAB.
+- They want to predict mortality, readmission, length of stay, drug recommendations, sleep stages, ICD codes, EEG events, or de-identification.
+- They need to look up or cross-map medical codes (ICD-9-CM, ICD-10-CM, ATC, NDC, RxNorm, CCS).
+- They have EHR-shaped data and want to train a clinical model without writing the plumbing themselves.
+
+PyHealth is the right tool when the workflow fits its 5 stages. If the user just wants generic PyTorch on tabular data, this skill is not necessary.
+
+## Installation (uv)
+
+PyHealth 2.0 requires Python ≥ 3.12, < 3.14. Use `uv` for environment management — it's faster and reproducible.
+
+```bash
+# Create a project with the right Python
+uv init my-pyhealth-project
+cd my-pyhealth-project
+uv python pin 3.12
+
+# Add PyHealth (this also pulls in PyTorch and friends)
+uv add pyhealth
+
+# Run scripts inside the env
+uv run python train.py
+```
+
+For a one-off script without a project, use `uv run --with pyhealth python script.py`. For the legacy 1.x line (Python 3.9+), `uv add pyhealth==1.16`. Detailed install notes, MIMIC access, and GPU/CPU device tips are in `references/installation.md`.
+
+## The 5-stage pipeline
+
+A complete pipeline is typically <20 lines. This is the canonical shape — start here and modify pieces:
+
+```python
+from pyhealth.datasets import MIMIC3Dataset, split_by_patient, get_dataloader
+from pyhealth.tasks import MortalityPredictionMIMIC3
+from pyhealth.models import Transformer
+from pyhealth.trainer import Trainer
+from pyhealth.metrics.binary import binary_metrics_fn
+
+# 1. Dataset — raw patient registry
+base = MIMIC3Dataset(
+    root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/",
+    tables=["DIAGNOSES_ICD", "PROCEDURES_ICD", "PRESCRIPTIONS"],
+)
+
+# 2. Task — converts patients into supervised samples
+samples = base.set_task(MortalityPredictionMIMIC3())
+
+# 3. Split + DataLoaders (split by patient to avoid leakage)
+train_ds, val_ds, test_ds = split_by_patient(samples, [0.8, 0.1, 0.1])
+train_loader = get_dataloader(train_ds, batch_size=32, shuffle=True)
+val_loader   = get_dataloader(val_ds,   batch_size=32, shuffle=False)
+test_loader  = get_dataloader(test_ds,  batch_size=32, shuffle=False)
+
+# 4. Model — must be passed the SampleDataset, not the BaseDataset
+model = Transformer(dataset=samples)
+
+# 5. Train + evaluate
+trainer = Trainer(model=model)
+trainer.train(
+    train_dataloader=train_loader,
+    val_dataloader=val_loader,
+    epochs=50,
+    monitor="pr_auc",
+)
+
+y_true, y_prob, _ = trainer.inference(test_loader)
+print(binary_metrics_fn(y_true, y_prob, metrics=["pr_auc", "roc_auc"]))
+```
+
+A copy-pasteable starter is in `assets/starter_pipeline.py`.
+
+## Critical things to get right
+
+These are the mistakes that PyHealth code most commonly trips on. Internalize them before writing pipelines:
+
+1. **Models take a `SampleDataset`, not a `BaseDataset`.** `MIMIC3Dataset(...)` returns a `BaseDataset` (a queryable patient registry). Only after `.set_task(task)` do you get a `SampleDataset`, which is what models, splitters, and DataLoaders expect. If you pass `base` to a model, it will fail or behave wrong.
+
+2. **Always split by patient (or visit), not by sample.** Random sample-level splits leak information across train/test because the same patient can appear in both. Use `split_by_patient` for patient-level prediction, `split_by_visit` only when visits are independent.
+
+3. **Match the task to the dataset.** Tasks are dataset-specific: `MortalityPredictionMIMIC3` won't work on MIMIC-IV — use `MortalityPredictionMIMIC4` or `InHospitalMortalityMIMIC4`. The full mapping is in `references/tasks.md`.
+
+4. **Pick `monitor` to match the task type.** For binary classification use `"pr_auc"` or `"roc_auc"`. For multilabel (drug rec) use `"pr_auc_samples"` or `"jaccard_samples"`. For multiclass use `"accuracy"` or `"f1_macro"`. Wrong monitor → checkpoint selection saves the wrong epoch.
+
+5. **MIMIC-IV uses `ehr_root=`, not `root=`.** This is the one inconsistency in the dataset constructors.
+
+6. **For reproducible work, point `cache_dir=` somewhere persistent.** PyHealth caches the parsed dataset; without `cache_dir`, you re-parse every run.
+
+## How to use this skill
+
+PyHealth has a large API surface — there's no point loading it all at once. Read the reference file that matches the user's task:
+
+| If the user is asking about… | Read |
+|---|---|
+| Installing, env setup, MIMIC access, GPU | `references/installation.md` |
+| Which dataset class to use, loading patterns, splitting | `references/datasets.md` |
+| What prediction task to choose (mortality, readmission, drug rec, sleep…) | `references/tasks.md` |
+| Picking a model architecture, model-specific arguments | `references/models.md` |
+| Looking up or cross-mapping ICD/ATC/NDC/RxNorm/CCS codes, tokenizers | `references/medcode.md` |
+| End-to-end recipes for common scenarios | `references/examples.md` |
+
+For multi-step tasks (e.g., "build a drug recommendation pipeline on MIMIC-IV"), read `tasks.md` + `models.md` + `examples.md` together — they cross-reference each other.
+
+## A note on style
+
+Write minimal, idiomatic PyHealth. The library is opinionated; lean into its abstractions instead of reimplementing them in raw PyTorch. If you find yourself writing a custom training loop, ask whether `Trainer` would do the job — it almost always will, and it handles checkpointing, logging, and best-model selection for free.
+
+When the user has private MIMIC access, point them at the local CSV root; for demos and learning, the synthetic MIMIC-III bucket (`https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/`) is fine and works without credentialing.

package/skills/scientific/pyhealth/assets/starter_pipeline.py

@@ -0,0 +1,58 @@
+"""
+PyHealth starter pipeline. Replace the four marked lines for a different
+dataset/task/model/monitor. Everything else stays the same.
+
+Run:
+    uv run python starter_pipeline.py
+"""
+
+from pyhealth.datasets import MIMIC3Dataset, split_by_patient, get_dataloader
+from pyhealth.tasks import MortalityPredictionMIMIC3
+from pyhealth.models import Transformer
+from pyhealth.trainer import Trainer
+from pyhealth.metrics.binary import binary_metrics_fn
+
+
+# ---- 1. Dataset ----------------------------------------------------------
+# Swap MIMIC3Dataset for MIMIC4Dataset / eICUDataset / OMOPDataset / etc.
+# For MIMIC-IV use ehr_root= instead of root=.
+base = MIMIC3Dataset(
+    root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/",
+    tables=["DIAGNOSES_ICD", "PROCEDURES_ICD", "PRESCRIPTIONS"],
+    cache_dir="./cache/mimic3",
+)
+
+# ---- 2. Task -------------------------------------------------------------
+# Match the suffix to the dataset (MIMIC3 / MIMIC4 / EICU / OMOP).
+task = MortalityPredictionMIMIC3()
+samples = base.set_task(task)
+
+# ---- 3. Split + DataLoaders ---------------------------------------------
+# Always split_by_patient for clinical prediction to avoid patient leakage.
+train, val, test = split_by_patient(samples, [0.8, 0.1, 0.1])
+train_loader = get_dataloader(train, batch_size=32, shuffle=True)
+val_loader = get_dataloader(val, batch_size=32, shuffle=False)
+test_loader = get_dataloader(test, batch_size=32, shuffle=False)
+
+# ---- 4. Model ------------------------------------------------------------
+# Swap for RETAIN / RNN / GAMENet / SafeDrug / StageNet / etc.
+# The model MUST receive the SampleDataset (`samples`), not the BaseDataset.
+model = Transformer(dataset=samples)
+
+# ---- 5. Trainer ----------------------------------------------------------
+# monitor:
+#   binary       -> "pr_auc" or "roc_auc"
+#   multiclass   -> "accuracy" / "f1_macro" / "cohen_kappa"
+#   multilabel   -> "pr_auc_samples" / "jaccard_samples"
+trainer = Trainer(model=model)
+trainer.train(
+    train_dataloader=train_loader,
+    val_dataloader=val_loader,
+    epochs=50,
+    monitor="pr_auc",
+    patience=5,
+)
+
+# ---- 6. Evaluate ---------------------------------------------------------
+y_true, y_prob, _ = trainer.inference(test_loader)
+print(binary_metrics_fn(y_true, y_prob, metrics=["pr_auc", "roc_auc", "f1"]))

package/skills/scientific/pyhealth/references/datasets.md

@@ -0,0 +1,126 @@
+# Datasets
+
+PyHealth datasets are **queryable patient registries**, not PyTorch `Dataset`s. The PyTorch-compatible object is the `SampleDataset` returned by `base.set_task(task)`. Don't try to index `BaseDataset` like a list — it won't work.
+
+## Two-tier object model
+
+```
+BaseDataset                         SampleDataset
+├── parses raw CSVs                 ├── one row per supervised sample
+├── one row per patient             ├── indexable, length-ed
+├── .set_task(task) → SampleDataset ├── feeds into get_dataloader(...)
+├── .get_patient(id) → Patient      └── feeds into Model(dataset=...)
+└── .iter_patients() → iterator
+```
+
+Always go `BaseDataset → set_task → SampleDataset` before doing anything else.
+
+## EHR / clinical datasets
+
+| Class | Import | Constructor signature highlights |
+|---|---|---|
+| `MIMIC3Dataset` | `from pyhealth.datasets import MIMIC3Dataset` | `root, tables, cache_dir=None, dev=False, num_workers=...` |
+| `MIMIC4Dataset` | `from pyhealth.datasets import MIMIC4Dataset` | `ehr_root, tables, ...` *(note: `ehr_root`, not `root`)* |
+| `eICUDataset` | `from pyhealth.datasets import eICUDataset` | `root, tables, ...` |
+| `OMOPDataset` | `from pyhealth.datasets import OMOPDataset` | `root, tables, ...` |
+| `EHRShotDataset` | `from pyhealth.datasets import EHRShotDataset` | few-shot benchmark |
+| `Support2Dataset` | `from pyhealth.datasets import Support2Dataset` | palliative care outcomes |
+| `MIMICExtractDataset` | `from pyhealth.datasets import MIMICExtractDataset` | pre-processed MIMIC |
+
+### Common MIMIC tables
+
+- **MIMIC-III** (uppercase): `DIAGNOSES_ICD`, `PROCEDURES_ICD`, `PRESCRIPTIONS`, `LABEVENTS`, `NOTEEVENTS`
+- **MIMIC-IV** (lowercase): `diagnoses_icd`, `procedures_icd`, `prescriptions`, `labevents`
+
+### MIMIC-III example
+
+```python
+from pyhealth.datasets import MIMIC3Dataset
+
+base = MIMIC3Dataset(
+    root="https://storage.googleapis.com/pyhealth/Synthetic_MIMIC-III/",
+    tables=["DIAGNOSES_ICD", "PROCEDURES_ICD", "PRESCRIPTIONS"],
+    cache_dir="./cache/mimic3",
+    dev=False,
+)
+```
+
+### MIMIC-IV example
+
+```python
+from pyhealth.datasets import MIMIC4Dataset
+
+base = MIMIC4Dataset(
+    ehr_root="/path/to/mimic-iv-2.2/hosp",      # NOT root=
+    tables=["diagnoses_icd", "procedures_icd", "prescriptions"],
+    cache_dir="./cache/mimic4",
+)
+```
+
+## Signal / sleep datasets
+
+| Class | Use |
+|---|---|
+| `SleepEDFDataset` | Sleep-EDF polysomnography → sleep stage classification |
+| `SHHSDataset` | Sleep Heart Health Study EEG |
+| `ISRUCDataset` | ISRUC sleep dataset |
+| `TUABDataset` | Temple University abnormal EEG |
+| `TUEVDataset` | Temple University EEG events |
+| `CardiologyDataset` | ECG / cardiology recordings |
+| `DREAMTDataset`, `BMDHSDataset` | Sleep / respiratory recordings |
+
+## Imaging datasets
+
+| Class | Use |
+|---|---|
+| `COVID19CXRDataset` | COVID-19 chest X-ray classification |
+| `ChestXray14Dataset` | NIH ChestX-ray14, multi-label |
+| `PhysioNetDeIDDataset` | De-identified clinical notes |
+
+## Genomics datasets
+
+| Class | Use |
+|---|---|
+| `ClinVarDataset` | Variant pathogenicity classification |
+| `COSMICDataset` | Mutation pathogenicity |
+| `TCGAPRADDataset` | Cancer survival, mutation burden |
+
+## Text dataset
+
+| Class | Use |
+|---|---|
+| `MedicalTranscriptionsDataset` | Clinical transcription category classification |
+
+## Splitting and DataLoaders
+
+After `set_task`, split and wrap in DataLoaders. **Always split by patient** (not by sample) for clinical prediction — random sample splits leak the same patient into train and test.
+
+```python
+from pyhealth.datasets import split_by_patient, split_by_visit, get_dataloader
+
+train, val, test = split_by_patient(samples, [0.8, 0.1, 0.1])
+
+train_loader = get_dataloader(train, batch_size=32, shuffle=True)
+val_loader   = get_dataloader(val,   batch_size=32, shuffle=False)
+test_loader  = get_dataloader(test,  batch_size=32, shuffle=False)
+```
+
+Use `split_by_visit` only when visits are independent (rare — most clinical tasks need patient-level splits). For time-aware evaluation, use `split_by_patient` with chronological cutoffs from a custom task.
+
+## Inspecting a dataset
+
+```python
+base.stats()                          # summary printout
+patient = base.get_patient("p001")    # Patient object
+events = patient.get_events()         # all events for that patient
+
+for p in base.iter_patients():        # iterate without loading all into memory
+    ...
+
+len(samples)                          # only valid AFTER set_task
+samples[0]                            # dict of features + label for one sample
+```
+
+## Custom datasets
+
+Subclass `BaseDataset` if the user has a non-standard EHR source. They must implement parsing of patients/events; `set_task` then works as usual. This is more involved than picking a built-in dataset — only suggest it when nothing else fits.