MEDS-DEV 0.0.1__py3-none-any.whl

MEDS_DEV/__init__.py

@@ -0,0 +1,13 @@
+from importlib.metadata import PackageNotFoundError, version
+
+from .datasets import DATASETS  # noqa: F401
+from .models import MODELS  # noqa: F401
+from .tasks import TASKS  # noqa: F401
+
+__all__ = ["DATASETS", "TASKS", "MODELS"]
+
+__package_name__ = "MEDS_DEV"
+try:
+    __version__ = version(__package_name__)
+except PackageNotFoundError:
+    __version__ = "unknown"

MEDS_DEV/configs/_build_dataset.yaml

@@ -0,0 +1,24 @@
+defaults:
+  - _self_
+
+dataset: ???
+output_dir: ???
+demo: False
+temp_dir: null # If null, will be determined automatically to a temporary directory.
+
+hydra:
+  job:
+    name: "meds_dev_build_{dataset}_${now:%Y-%m-%d_%H-%M-%S}"
+  run:
+    dir: "${output_dir}/.logs"
+  help:
+    app_name: "MEDS-DEV Dataset Builder Helper"
+
+    template: |-
+      == ${hydra.help.app_name} ==
+      ${hydra.help.app_name} is a command line tool for building MEDS-DEV compliant datasets.
+
+      You can specify "dataset" to point to the dataset you want to build, "demo=True" to build the demo
+      instead of the full dataset (good for testing), "output_dir" to say where the final, raw MEDS cohort
+      should be stored on disk, and "temp_dir" to overwrite the default temporary directory for storing
+      intermediated files.

MEDS_DEV/configs/_evaluate_predictions.yaml

@@ -0,0 +1,22 @@
+defaults:
+  - _self_
+
+predictions_dir: ???
+predictions_path: ${predictions_dir}/**/*.parquet
+output_dir: ???
+
+hydra:
+  job:
+    name: "meds_dev_evaluation_${now:%Y-%m-%d_%H-%M-%S}"
+  run:
+    dir: "${output_dir}/.logs"
+  help:
+    app_name: "MEDS-DEV Evaluation Helper"
+
+    template: |-
+      == ${hydra.help.app_name} ==
+      ${hydra.help.app_name} is a command line tool for evaluating predictions from MEDS-DEV models.
+
+      Refer to the usage of MEDS-evaluation for more details. You can either specify the predictions path
+      directly with `predictions_path` or use the `predictions_dir` to evaluate all predictions in a
+      directory, where this can point to the output dir of a model predict step.

MEDS_DEV/configs/_extract_task.yaml

@@ -0,0 +1,24 @@
+defaults:
+  - _self_
+
+dataset: ???
+dataset_dir: ???
+task: ???
+output_dir: ???
+
+hydra:
+  job:
+    name: "meds_dev_extract_{task}_for_{dataset}_${now:%Y-%m-%d_%H-%M-%S}"
+  run:
+    dir: "${output_dir}/.logs"
+  help:
+    app_name: "MEDS-DEV Task Extractor Helper"
+
+    template: |-
+      == ${hydra.help.app_name} ==
+      ${hydra.help.app_name} is a command line tool for extracting MEDS-DEV specified tasks from compliant
+      datasets.
+
+      You can specify "dataset" to give the name of the dataset you want to build and "dataset_dir" to point
+      to the extracted dataset on disk, "output_dir" to say where the final, labeled task cohort
+      should be stored on disk, and "task" to dictate which task should be extracted.

MEDS_DEV/configs/_run_model.yaml

@@ -0,0 +1,63 @@
+defaults:
+  - _self_
+
+dataset_type: supervised
+dataset_dir: ???
+labels_dir: null
+dataset_name: null
+task_name: null
+
+temp_dir: ${output_dir}/.temp # This is used because the venv is built here, so persisting it is useful.
+
+demo: false
+mode: prediction
+model: ???
+do_overwrite: false
+
+split: null # this is only used for training.
+
+model_initialization_dir: null
+
+output_dir: ???
+
+hydra:
+  job:
+    name: "meds_dev_model_${now:%Y-%m-%d_%H-%M-%S}"
+  run:
+    dir: "${output_dir}/.logs"
+  help:
+    app_name: "MEDS-DEV Model Running Helper"
+
+    template: |-
+      == ${hydra.help.app_name} ==
+      ${hydra.help.app_name} is a command line tool for running MEDS-DEV compliant models.
+
+      This tool is primarily designed to help model developers ensure that other users can trivially train
+      their models over local health datasets and local data owners trivially train and evaluate MEDS-DEV
+      models over their data.
+
+      Models can either be run over either (a) unsupervised data or (b) supervised data, and can be run in any
+      of (1) hyperparameter tuning, (2) training, or (3) prediction modes. Various models need not define all
+      of the combinations of these modes, and this is by design!
+
+      The assumed (global) flow is:
+        1. unsupervised, train
+        2. [optional, for zero-shot] unsupervised, predict
+        3. supervised, train
+        4. supervised, predict
+
+      In each of these steps the "model_initialization_dir" parameter is assumed to point to the directory
+      specified by the "output_dir" of the prior stage, if such a prior stage is necessary and was run.
+
+      It is fully acceptable to have no prior stages having been run, and keep model_initialization_dir as
+      null. These parameters are merely used as inputs to the customized model run scripts in the model.yaml
+      files. The only mandatory stage for participating in MEDS-DEV is supervised, prediction.
+
+      You can also run in mode=`full` to run all 4 applicable steps in order (with applicability determined by
+      what commands are present in the model), with the model_initialization_dir appropriately set in between.
+      In this case, `output_dir` should be set to a global "experiment directory" for the series of runs, and
+      appropriate sub-dirs will be created and used for the sequence. In this case, if prior stage completion
+      is flagged (e.g., if you use the same dir twice), the stage will not be re-run. The directory structure
+      used for these will depend on dataset_name and task_name, so those must be set if this mode is used.
+
+      If do_overwrite is set to true, the output dir will be cleared before anything is run.

MEDS_DEV/datasets/AUMCdb/README.md

@@ -0,0 +1,34 @@
+# AUMCdb
+
+## Description
+
+The first freely accessible intensive care database from within the European Union containing de-identified health data related to tens of thousands of European intensive care unit admissions, including demographics, vital signs, laboratory tests and medications.
+
+## Access Requirements
+
+Taken from [official website](https://amsterdammedicaldatascience.nl/amsterdamumcdb/#requesting-access):
+
+- **Access Policy**: Fill out and sign the combined Access and End User License form.
+- **License (for files)**: Specify the license under which the dataset files are distributed.
+- **Data Use Agreement**: Agreement found [here](https://amsterdammedicaldatascience.nl/content/uploads/sites/2/2022/12/arfeula_v1.6.pdf).
+- **Required training**: Valid training courses include the Data or Specimens Only Research (DSOR) course from CITI, the Basic Course for Clinical Investigators (BROK) from NFU or an equivalent course. The DSOR course may be taken free of charge and is also needed to gain access to the MIMIC and eICU intensive care databases from the USA.
+
+## Supported Tasks
+
+- `tasks/mortality/in_icu/first_24h.yaml`
+
+## MEDS-transformation
+
+[MEDS_transforms](https://github.com/mmcdermott/MEDS_transforms) now includes the AUMCdb example. Please refer to the [guide](https://github.com/mmcdermott/MEDS_transforms/tree/main/AUMC_Example).
+
+## Sources
+
+1. [AUMCdb dataset](https://amsterdammedicaldatascience.nl/amsterdamumcdb/)
+2. [AUMCdb Research Paper](https://journals.lww.com/ccmjournal/fulltext/2021/06000/sharing_icu_patient_data_responsibly_under_the.16.aspx)
+3. [AUMCdb Data Use Agreement](https://amsterdammedicaldatascience.nl/content/uploads/sites/2/2022/12/arfeula_v1.6.pdf)
+4. [Data Repository](https://easy.dans.knaw.nl/ui/home)
+5. [Code Repository](https://github.com/AmsterdamUMC/AmsterdamUMCdb)
+
+## Disclaimer
+
+Please refer to the data owners and the most up-to-date information when using this dataset in your research.

MEDS_DEV/datasets/AUMCdb/predicates.yaml

@@ -0,0 +1,8 @@
+predicates:
+  icu_admission:
+    code: { regex: "^ICU_ADMISSION//.*" }
+  icu_discharge:
+    code: { regex: "^ICU_DISCHARGE//.*" }
+
+  death:
+    code: MEDS_DEATH

MEDS_DEV/datasets/MIMIC-IV/README.md

@@ -0,0 +1,10 @@
+# MIMIC-IV
+
+TODO: Summarize MIMIC-IV
+
+TODO: Add citations properly somehow.
+
+## URLs
+
+1. https://physionet.org/content/mimiciv/
+2. https://www.nature.com/articles/s41597-022-01899-x

MEDS_DEV/datasets/MIMIC-IV/commands.yaml

@@ -0,0 +1,14 @@
+build_full: >-
+  MEDS_extract-MIMIC_IV
+  raw_input_dir="{temp_dir}/raw"
+  pre_MEDS_dir="{temp_dir}/pre_MEDS"
+  MEDS_cohort_dir="{output_dir}"
+  log_dir="{output_dir}/.pipeline_logs"
+
+build_demo: >-
+  MEDS_extract-MIMIC_IV
+  do_demo=True
+  raw_input_dir="{temp_dir}/raw"
+  pre_MEDS_dir="{temp_dir}/pre_MEDS"
+  MEDS_cohort_dir="{output_dir}"
+  log_dir="{output_dir}/.pipeline_logs"

MEDS_DEV/datasets/MIMIC-IV/predicates.yaml

@@ -0,0 +1,167 @@
+predicates:
+  hospital_admission:
+    code: { regex: "^HOSPITAL_ADMISSION//.*" }
+  hospital_discharge:
+    code: { regex: "^HOSPITAL_DISCHARGE//.*" }
+
+  ED_registration:
+    code: { regex: "^ED_REGISTRATION//.*" }
+  ED_discharge:
+    code: { regex: "^ED_OUT//.*" }
+
+  icu_admission:
+    code: { regex: "^ICU_ADMISSION//.*" }
+  icu_discharge:
+    code: { regex: "^ICU_DISCHARGE//.*" }
+
+  birth:
+    code: MEDS_BIRTH
+  death:
+    code: MEDS_DEATH
+
+  creatinine_1:
+    code: LAB//50912//mg/dL
+  creatinine_2:
+    code: LAB//52546//mg/dL
+  abnormally_high_creatinine_1:
+    code: LAB//50912//mg/dL
+    value_min: 1.3 # mg/dL
+    value_min_inclusive: False
+    value_max: null
+  abnormally_high_creatinine_2:
+    code: LAB//52546//mg/dL
+    value_min: 1.3 # mg/dL
+    value_min_inclusive: False
+    value_max: null
+  creatinine:
+    expr: or(creatinine_1, creatinine_2)
+  abnormally_high_creatinine:
+    expr: or(abnormally_high_creatinine_1, abnormally_high_creatinine_2)
+
+  sodium_1:
+    code: LAB//220645//mEq/L
+  sodium_2:
+    code: LAB//50983//mEq/L
+  sodium_3:
+    code: LAB//52623//mEq/L
+  abnormally_low_sodium_1:
+    code: LAB//220645//mEq/L
+    value_min: null
+    value_max: 135 # mEq/L
+    value_max_inclusive: False
+  abnormally_low_sodium_2:
+    code: LAB//50983//mEq/L
+    value_min: null
+    value_max: 135 # mEq/L
+    value_max_inclusive: False
+  abnormally_low_sodium_3:
+    code: LAB//52623//mEq/L
+    value_min: null
+    value_max: 135 # mEq/L
+    value_max_inclusive: False
+  sodium:
+    expr: or(sodium_1, sodium_2, sodium_3)
+  abnormally_low_sodium:
+    expr: or(abnormally_low_sodium_1, abnormally_low_sodium_2, abnormally_low_sodium_3)
+
+  bicarbonate_1:
+    code: LAB//227443//mEq/L
+  bicarbonate_2:
+    code: LAB//50882//mEq/L
+  abnormally_low_bicarbonate_1:
+    code: LAB//227443//mEq/L
+    value_min: null
+    value_max: 22 # mEq/L
+    value_max_inclusive: False
+  abnormally_low_bicarbonate_2:
+    code: LAB//50882//mEq/L
+    value_min: null
+    value_max: 22 # mEq/L
+    value_max_inclusive: False
+  bicarbonate:
+    expr: or(bicarbonate_1, bicarbonate_2)
+  abnormally_low_bicarbonate:
+    expr: or(abnormally_low_bicarbonate_1, abnormally_low_bicarbonate_2)
+
+  hemoglobin_1:
+    code: LAB//220228//g/dl
+  hemoglobin_2:
+    code: LAB//50811//g/dL
+  abnormally_low_hemoglobin_1:
+    code: LAB//220228//g/dl
+    value_min: null
+    value_max: 13 # g/dL
+    value_max_inclusive: False
+  abnormally_low_hemoglobin_2:
+    code: LAB//50811//g/dL
+    value_min: null
+    value_max: 13 # g/dL
+    value_max_inclusive: False
+  hemoglobin:
+    expr: or(hemoglobin_1, hemoglobin_2)
+  abnormally_low_hemoglobin:
+    expr: or(abnormally_low_hemoglobin_1, abnormally_low_hemoglobin_2)
+
+  wbc_1:
+    code: LAB//220546//K/uL
+  wbc_2:
+    code: LAB//51300//K/uL
+  abnormally_high_wbc_1:
+    code: LAB//220546//K/uL
+    value_min: 11 # K/uL
+    value_min_inclusive: False
+    value_max: null
+  abnormally_high_wbc_2:
+    code: LAB//51300//K/uL
+    value_min: 11 # K/uL
+    value_min_inclusive: False
+    value_max: null
+  wbc:
+    expr: or(wbc_1, wbc_2)
+  abnormally_high_wbc:
+    expr: or(abnormally_high_wbc_1, abnormally_high_wbc_2)
+
+  platelets_1:
+    code: LAB//227457//K/uL
+  platelets_2:
+    code: LAB//51265//K/uL
+  abnormally_low_platelets_1:
+    code: LAB//227457//K/uL
+    value_min: null
+    value_max: 150 # K/uL
+    value_max_inclusive: False
+  abnormally_low_platelets_2:
+    code: LAB//51265//K/uL
+    value_min: null
+    value_max: 150 # K/uL
+    value_max_inclusive: False
+  platelets:
+    expr: or(platelets_1, platelets_2)
+  abnormally_low_platelets:
+    expr: or(abnormally_low_platelets_1, abnormally_low_platelets_2)
+
+  map_1:
+    code: LAB//220052//mmHg
+  map_2:
+    code: LAB//220181//mmHg
+  map_3:
+    code: LAB//225312//mmHg
+  abnormally_low_map_1:
+    code: LAB//220052//mmHg
+    value_min: null
+    value_max: 65 # mmHg
+    value_max_inclusive: False
+  abnormally_low_map_2:
+    code: LAB//220181//mmHg
+    value_min: null
+    value_max: 65 # mmHg
+    value_max_inclusive: False
+  abnormally_low_map_3:
+    code: LAB//225312//mmHg
+    value_min: null
+    value_max: 65 # mmHg
+    value_max_inclusive: False
+  map:
+    expr: or(map_1, map_2, map_3)
+  abnormally_low_map:
+    expr: or(abnormally_low_map_1, abnormally_low_map_2, abnormally_low_map_3)

MEDS_DEV/datasets/MIMIC-IV/requirements.txt

@@ -0,0 +1 @@
+MIMIC-IV-MEDS==0.0.2

MEDS_DEV/datasets/OMOP/README.md

@@ -0,0 +1,66 @@
+# OMOP Dataset Predicates
+
+This README provides an overview of predicates defined for the OMOP dataset to represent various healthcare events. Each predicate is defined with specific codes or patterns that are used to identify events, such as admissions, discharges, and other clinical encounters, within the OMOP format.
+
+## Predicates Overview
+
+### 1. **hospital_admission**
+
+- **Codes**:
+    - `Visit/IP`: Inpatient visit.
+    - `Visit/ERIP`: Emergency room to inpatient visit.
+    - `CMS Place of Service/51`: Inpatient Psychiatric Facility.
+    - `CMS Place of Service/61`: Comprehensive Inpatient Rehabilitation Facility.
+
+### 2. **hospital_discharge**
+
+- **Codes**:
+    - `CMS Place of Service/12`: Home.
+    - `CMS Place of Service/31`: Skilled Nursing Facility.
+    - `SNOMED/371827001`: Discharge to another healthcare facility.
+    - `PCORNet/Generic-NI`: Noninstitutional (self-care).
+    - `SNOMED/397709008`: Hospice care.
+    - `Medicare Specialty/A4`: Hospice provider.
+    - `CMS Place of Service/21`: Inpatient hospital.
+    - `CMS Place of Service/61`: Comprehensive Inpatient Rehabilitation Facility.
+    - `CMS Place of Service/51`: Inpatient Psychiatric Facility.
+    - `SNOMED/225928004`: Transfer to intermediate care facility.
+    - `CMS Place of Service/34`: Hospice.
+    - `PCORNet/Generic-OT`: Other.
+    - `CMS Place of Service/27`: Ambulatory Surgical Center.
+    - `CMS Place of Service/33`: Custodial Care Facility.
+    - `CMS Place of Service/09`: Prison/Correctional Facility.
+    - `CMS Place of Service/32`: Nursing Facility.
+
+### 3. **ED_registration**
+
+- **Code**:
+    - `Visit/ER`: Emergency room visit.
+
+### 4. **ED_discharge**
+
+- **Codes**:
+    - `CMS Place of Service/12`: Home.
+    - `PCORNet/Generic-NI`: Noninstitutional (self-care).
+    - `SNOMED/371827001`: Discharge to other healthcare facility.
+    - `CMS Place of Service/21`: Inpatient hospital.
+    - `NUCC/261Q00000X`: Clinic/Center.
+    - `SNOMED/225928004`: Transfer to intermediate care facility.
+    - `CMS Place of Service/51`: Inpatient Psychiatric Facility.
+    - `CMS Place of Service/23`: Emergency Room - Hospital.
+    - `CMS Place of Service/31`: Skilled Nursing Facility.
+    - `CMS Place of Service/34`: Hospice.
+    - `SNOMED/397709008`: Hospice care.
+    - `CMS Place of Service/24`: Ambulatory Surgical Center.
+    - `SNOMED/34596002`: Discharge to home under care of a home health service.
+    - `PCORNet/Generic-OT`: Other.
+    - `CMS Place of Service/09`: Prison/Correctional Facility.
+
+### 5. **death**
+
+- **Code**:
+    - `MEDS_DEATH`: Indicator of patient death.
+
+## Usage
+
+Each predicate is defined in the YAML file, allowing for flexible querying and filtering of the OMOP dataset. The structure leverages the codes and patterns specified under each predicate to capture relevant patient events.

MEDS_DEV/datasets/OMOP/predicates.yaml

@@ -0,0 +1,52 @@
+predicates:
+  hospital_admission:
+    code:
+      any:
+        - "Visit/IP"
+        - "Visit/ERIP"
+        - "CMS Place of Service/51"
+        - "CMS Place of Service/61"
+
+  hospital_discharge:
+    code:
+      any:
+        - "CMS Place of Service/12"
+        - "CMS Place of Service/31"
+        - "SNOMED/371827001"
+        - "PCORNet/Generic-NI"
+        - "SNOMED/397709008"
+        - "Medicare Specialty/A4"
+        - "CMS Place of Service/21"
+        - "CMS Place of Service/61"
+        - "CMS Place of Service/51"
+        - "SNOMED/225928004"
+        - "CMS Place of Service/34"
+        - "PCORNet/Generic-OT"
+        - "CMS Place of Service/27"
+        - "CMS Place of Service/33"
+        - "CMS Place of Service/09"
+        - "CMS Place of Service/32"
+
+  ED_registration:
+    code: "Visit/ER"
+
+  ED_discharge:
+    code:
+      any:
+        - "CMS Place of Service/12"
+        - "PCORNet/Generic-NI"
+        - "SNOMED/371827001"
+        - "CMS Place of Service/21"
+        - "NUCC/261Q00000X"
+        - "SNOMED/225928004"
+        - "CMS Place of Service/51"
+        - "CMS Place of Service/23"
+        - "CMS Place of Service/31"
+        - "CMS Place of Service/34"
+        - "SNOMED/397709008"
+        - "CMS Place of Service/24"
+        - "SNOMED/34596002"
+        - "PCORNet/Generic-OT"
+        - "CMS Place of Service/09"
+  death:
+    code: MEDS_DEATH

MEDS_DEV/datasets/README.md

@@ -0,0 +1,19 @@
+# Datasets
+
+This folder contains details for datasets currently included in the MEDS-DEV effort.
+
+To contribute a new dataset:
+
+1. Fork this repository
+2. Add your dataset predicates file in its respective folder (see `MIMIC-IV/predicates.yaml` for an example of predicate structure)
+3. Test locally to ensure your dataset works correctly. Ideally specify the used packages and versions in the dataset information.
+4. Specify the dataset information (including supported and custom tasks) in the template README.md file in the dataset's folder.
+5. Create a pull request with your changes
+
+## Notes
+
+If you have a version of a task configuration file that is more specialized to a dataset than can be achieved
+with overwriting the predicates alone, then:
+
+1. Make a GitHub issue explaining why the existing file is not used
+2. Add a file here `../tasks/$DATASET_NAME/$TASK_NAME.yaml` with that configuration.

MEDS_DEV/datasets/UCLA/README.md

@@ -0,0 +1,33 @@
+# UCLA DDR MEDS Dataset
+
+UCLA Health serves well over half a million patients annually, and as a result, UCLA possesses a vast database of Electronic Health Records (EHR). The stemming Discovery Data Repository (DDR) serves as an invaluable resource for researchers. Within the DDR Medical record concepts and organization, and the UCLA ATLAS Precision Health Biobank which integrates genetic information and de-identified medical records enable precision health research.
+
+## Access Requirements
+
+In order to gain access you will need to clear and obtain HIPAA compliance. *HIPAA compliance is the practice of following the Health Insurance Portability and Accountability Act (HIPAA) to protect the privacy and security of health information.* The dataset is not publicly available but people within UCLA can help run analysis on your study by sending over **source code** and **model weights** if it is a pre-trained architecture.
+
+- **License (for files)**: Specify the license under which the dataset files are distributed. (Pending)
+- **Data Use Agreement**: Specify any data use agreement that must be signed to access the dataset. (Pending)
+
+## Supported Tasks
+
+Currently the tasks supported by the UCLA-MEDS format are follows:
+
+- `tasks/mortality/in_icu/first_24h.yaml`
+
+## MEDS-transformation
+
+Researchers at UCLA have transformed our DDR database into the medsformat making it quick and easy to run any analyses at our institution.
+
+## Sources
+
+Summarize the sources of the dataset. If the dataset is a combination of multiple sources, list them here.
+
+1. https://link-to-dataset.org
+
+## Contact
+
+For queries or questions. Feel free to send emails to
+
+- Simon A. Lee (simonlee711@g.ucla.edu)
+- Jeffrey N. Chiang (njchiang@g.ucla.edu)

MEDS_DEV/datasets/UCLA/predicates.yaml

@@ -0,0 +1,18 @@
+predicates:
+  hospital_admission:
+    code: { regex: "^HOSPITAL_ADMISSION//.*" }
+  hospital_discharge:
+    code: { regex: "^HOSPITAL_DISCHARGE//.*" }
+
+  ED_registration:
+    code: { regex: "^ED_REGISTRATION//.*" }
+  ED_discharge:
+    code: { regex: "^ED_DISCHARGE//.*" }
+
+  icu_admission:
+    code: { regex: "^ICU_ADMISSION//.*" }
+  icu_discharge:
+    code: { regex: "^ICU_DISCHARGE//.*" }
+
+  death:
+    code: MORTALITY

MEDS_DEV/datasets/__init__.py

@@ -0,0 +1,22 @@
+from importlib.resources import files
+
+from omegaconf import OmegaConf
+
+dataset_files = files("MEDS_DEV.datasets")
+CFG_YAML = files("MEDS_DEV.configs") / "_build_dataset.yaml"
+
+DATASETS = {}
+for path in dataset_files.iterdir():
+    if not (path.is_dir() and (path / "commands.yaml").exists()):
+        continue
+
+    commands = OmegaConf.load(path / "commands.yaml")
+    requirements_path = path / "requirements.txt"
+    predicates_path = path / "predicates.yaml"
+    DATASETS[path.name] = {
+        "commands": commands,
+        "predicates": predicates_path if predicates_path.exists() else None,
+        "requirements": requirements_path if requirements_path.exists() else None,
+    }
+
+__all__ = ["DATASETS", "CFG_YAML"]

MEDS_DEV/datasets/__main__.py

@@ -0,0 +1,36 @@
+import logging
+import subprocess
+
+logger = logging.getLogger(__name__)
+
+import hydra
+from omegaconf import DictConfig
+
+from ..utils import temp_env
+from . import CFG_YAML, DATASETS
+
+
+@hydra.main(version_base=None, config_path=str(CFG_YAML.parent), config_name=CFG_YAML.stem)
+def main(cfg: DictConfig):
+    if cfg.dataset not in DATASETS:
+        raise ValueError(f"Dataset {cfg.dataset} not currently configured!")
+
+    commands = DATASETS[cfg.dataset]["commands"]
+    requirements = DATASETS[cfg.dataset]["requirements"]
+
+    build_cmd = commands["build_demo"] if cfg.demo else commands["build_full"]
+
+    with temp_env(cfg, requirements) as (build_temp_dir, env):
+        build_cmd = build_cmd.format(output_dir=cfg.output_dir, temp_dir=str(build_temp_dir.resolve()))
+
+        logger.info(f"Running build command: {build_cmd}")
+        build_command_out = subprocess.run(
+            build_cmd, shell=True, env=env, cwd=build_temp_dir, capture_output=True
+        )
+
+        command_errored = build_command_out.returncode != 0
+        if command_errored:
+            raise RuntimeError(
+                f"Build {cfg.dataset} command {build_cmd} failed with exit code "
+                f"{build_command_out.returncode}:\n{build_command_out.stderr.decode()}"
+            )

MEDS_DEV/evaluation/README.md

@@ -0,0 +1,4 @@
+# MEDS-DEV Evaluation
+
+MEDS-DEV evaluation is standardized through the standalone
+[meds-evaluation](https://github.com/kamilest/meds-evaluation) package.

MEDS_DEV/evaluation/__init__.py

@@ -0,0 +1,5 @@
+from importlib.resources import files
+
+CFG_YAML = files("MEDS_DEV.configs") / "_evaluate_predictions.yaml"
+
+__all__ = ["CFG_YAML"]