NWICU-MEDS 0.1.dev18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- NWICU_MEDS/__init__.py +38 -0
- NWICU_MEDS/__main__.py +82 -0
- NWICU_MEDS/commands.py +79 -0
- NWICU_MEDS/configs/ETL.yaml +45 -0
- NWICU_MEDS/configs/__init__.py +0 -0
- NWICU_MEDS/configs/event_configs.yaml +324 -0
- NWICU_MEDS/configs/main.yaml +25 -0
- NWICU_MEDS/configs/pre_MEDS.yaml +15 -0
- NWICU_MEDS/configs/runner.yaml +7 -0
- NWICU_MEDS/dataset.yaml +25 -0
- NWICU_MEDS/download.py +301 -0
- NWICU_MEDS/pre_MEDS.py +352 -0
- NWICU_MEDS-0.1.dev18.dist-info/LICENSE +21 -0
- NWICU_MEDS-0.1.dev18.dist-info/METADATA +86 -0
- NWICU_MEDS-0.1.dev18.dist-info/RECORD +18 -0
- NWICU_MEDS-0.1.dev18.dist-info/WHEEL +5 -0
- NWICU_MEDS-0.1.dev18.dist-info/entry_points.txt +2 -0
- NWICU_MEDS-0.1.dev18.dist-info/top_level.txt +1 -0
NWICU_MEDS/__init__.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
from importlib.metadata import PackageNotFoundError, version
|
|
2
|
+
from importlib.resources import files
|
|
3
|
+
|
|
4
|
+
from omegaconf import OmegaConf
|
|
5
|
+
|
|
6
|
+
__package_name__ = "NWICU_MEDS"
|
|
7
|
+
try:
|
|
8
|
+
__version__ = version(__package_name__)
|
|
9
|
+
except PackageNotFoundError: # pragma: no cover
|
|
10
|
+
__version__ = "unknown"
|
|
11
|
+
|
|
12
|
+
MAIN_CFG = files(__package_name__).joinpath("configs/main.yaml")
|
|
13
|
+
EVENT_CFG = files(__package_name__).joinpath("configs/event_configs.yaml")
|
|
14
|
+
ETL_CFG = files(__package_name__).joinpath("configs/ETL.yaml")
|
|
15
|
+
RUNNER_CFG = files(__package_name__).joinpath("configs/runner.yaml")
|
|
16
|
+
PRE_MEDS_PY = files(__package_name__).joinpath("pre_MEDS.py")
|
|
17
|
+
PRE_MEDS_CFG = files(__package_name__).joinpath("configs/pre_MEDS.yaml")
|
|
18
|
+
DATASET_CFG = files(__package_name__).joinpath("dataset.yaml")
|
|
19
|
+
|
|
20
|
+
dataset_info = OmegaConf.load(DATASET_CFG)
|
|
21
|
+
|
|
22
|
+
HAS_PRE_MEDS = PRE_MEDS_PY.exists()
|
|
23
|
+
|
|
24
|
+
event_config = OmegaConf.load(EVENT_CFG)
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"event_config",
|
|
28
|
+
"EVENT_CFG",
|
|
29
|
+
"ETL_CFG",
|
|
30
|
+
"HAS_PRE_MEDS",
|
|
31
|
+
"PRE_MEDS_CFG",
|
|
32
|
+
"MAIN_CFG",
|
|
33
|
+
"RUNNER_CFG",
|
|
34
|
+
"DATASET_CFG",
|
|
35
|
+
"dataset_info",
|
|
36
|
+
"__package_name__",
|
|
37
|
+
"__version__",
|
|
38
|
+
]
|
NWICU_MEDS/__main__.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
import hydra
|
|
8
|
+
from omegaconf import DictConfig
|
|
9
|
+
|
|
10
|
+
from . import ETL_CFG, EVENT_CFG, HAS_PRE_MEDS, MAIN_CFG, PRE_MEDS_PY, RUNNER_CFG
|
|
11
|
+
from . import __version__ as PKG_VERSION
|
|
12
|
+
from . import dataset_info
|
|
13
|
+
from .commands import run_command
|
|
14
|
+
from .download import download_data
|
|
15
|
+
|
|
16
|
+
logger = logging.getLogger(__name__)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@hydra.main(version_base=None, config_path=str(MAIN_CFG.parent), config_name=MAIN_CFG.stem)
|
|
20
|
+
def main(cfg: DictConfig):
|
|
21
|
+
"""Runs the end-to-end MEDS Extraction pipeline."""
|
|
22
|
+
|
|
23
|
+
raw_input_dir = Path(cfg.raw_input_dir)
|
|
24
|
+
pre_MEDS_dir = Path(cfg.pre_MEDS_dir)
|
|
25
|
+
MEDS_cohort_dir = Path(cfg.MEDS_cohort_dir)
|
|
26
|
+
stage_runner_fp = cfg.get("stage_runner_fp", None)
|
|
27
|
+
|
|
28
|
+
# Step 0: Data downloading
|
|
29
|
+
if cfg.do_download:
|
|
30
|
+
if cfg.get("do_demo", False):
|
|
31
|
+
logger.info("Downloading demo data.")
|
|
32
|
+
download_data(raw_input_dir, dataset_info, do_demo=True)
|
|
33
|
+
else:
|
|
34
|
+
logger.info("Downloading data.")
|
|
35
|
+
download_data(raw_input_dir, dataset_info)
|
|
36
|
+
else: # pragma: no cover
|
|
37
|
+
logger.info("Skipping data download.")
|
|
38
|
+
|
|
39
|
+
# Step 1: Pre-MEDS Data Wrangling
|
|
40
|
+
if HAS_PRE_MEDS:
|
|
41
|
+
command_parts = [
|
|
42
|
+
"python",
|
|
43
|
+
str(PRE_MEDS_PY),
|
|
44
|
+
f"input_dir={raw_input_dir}",
|
|
45
|
+
f"output_dir={pre_MEDS_dir}",
|
|
46
|
+
]
|
|
47
|
+
run_command(command_parts, cfg)
|
|
48
|
+
else:
|
|
49
|
+
pre_MEDS_dir = f"{raw_input_dir}"
|
|
50
|
+
|
|
51
|
+
# Step 2: MEDS Cohort Creation
|
|
52
|
+
# First we need to set some environment variables
|
|
53
|
+
command_parts = [
|
|
54
|
+
f"DATASET_NAME={dataset_info.dataset_name}",
|
|
55
|
+
f"DATASET_VERSION={dataset_info.raw_dataset_version}:{PKG_VERSION}",
|
|
56
|
+
f"EVENT_CONVERSION_CONFIG_FP={str(EVENT_CFG.resolve())}",
|
|
57
|
+
f"PRE_MEDS_DIR={str(pre_MEDS_dir.resolve())}",
|
|
58
|
+
f"MEDS_COHORT_DIR={str(MEDS_cohort_dir.resolve())}",
|
|
59
|
+
]
|
|
60
|
+
|
|
61
|
+
# Then we construct the rest of the command
|
|
62
|
+
command_parts.extend(
|
|
63
|
+
[
|
|
64
|
+
"MEDS_transform-runner",
|
|
65
|
+
f"--config-path={str(RUNNER_CFG.parent.resolve())}",
|
|
66
|
+
f"--config-name={RUNNER_CFG.stem}",
|
|
67
|
+
f"pipeline_config_fp={str(ETL_CFG.resolve())}",
|
|
68
|
+
]
|
|
69
|
+
)
|
|
70
|
+
if int(os.getenv("N_WORKERS", 1)) <= 1:
|
|
71
|
+
logger.info("Running in serial mode as N_WORKERS is not set.")
|
|
72
|
+
command_parts.append("~parallelize")
|
|
73
|
+
|
|
74
|
+
if stage_runner_fp:
|
|
75
|
+
command_parts.append(f"stage_runner_fp={stage_runner_fp}")
|
|
76
|
+
|
|
77
|
+
command_parts.append("'hydra.searchpath=[pkg://MEDS_transforms.configs]'")
|
|
78
|
+
run_command(command_parts, cfg)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
if __name__ == "__main__":
|
|
82
|
+
main()
|
NWICU_MEDS/commands.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import subprocess
|
|
3
|
+
|
|
4
|
+
from omegaconf import DictConfig
|
|
5
|
+
|
|
6
|
+
logger = logging.getLogger(__name__)
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def run_command(
|
|
10
|
+
command_parts: list[str],
|
|
11
|
+
cfg: DictConfig | None = None,
|
|
12
|
+
runner_fn: callable = subprocess.run,
|
|
13
|
+
):
|
|
14
|
+
"""Runs a command with the specified runner function.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
command_parts: A list of the arguments to be run as a shell command.
|
|
18
|
+
cfg: The configuration for the command.
|
|
19
|
+
runner_fn: The function to run the command with (added for dependency injection).
|
|
20
|
+
|
|
21
|
+
Raises:
|
|
22
|
+
ValueError: If the command fails
|
|
23
|
+
|
|
24
|
+
Examples:
|
|
25
|
+
>>> def fake_shell_succeed(cmd, shell, capture_output):
|
|
26
|
+
... print(cmd)
|
|
27
|
+
... return subprocess.CompletedProcess(args=cmd, returncode=0, stdout=b"", stderr=b"")
|
|
28
|
+
>>> def fake_shell_fail(cmd, shell, capture_output):
|
|
29
|
+
... print(cmd)
|
|
30
|
+
... return subprocess.CompletedProcess(args=cmd, returncode=1, stdout=b"", stderr=b"")
|
|
31
|
+
>>> run_command(["echo", "hello"], runner_fn=fake_shell_succeed)
|
|
32
|
+
echo hello
|
|
33
|
+
>>> run_command(["echo", "hello"], runner_fn=fake_shell_fail)
|
|
34
|
+
Traceback (most recent call last):
|
|
35
|
+
...
|
|
36
|
+
ValueError: Command failed with return code 1.
|
|
37
|
+
>>> run_command(["hello"], cfg={"do_overwrite": True}, runner_fn=fake_shell_succeed)
|
|
38
|
+
hello ++do_overwrite=True
|
|
39
|
+
>>> run_command(["hello"], cfg={"do_overwrite": False}, runner_fn=fake_shell_succeed)
|
|
40
|
+
hello ++do_overwrite=False
|
|
41
|
+
>>> run_command(["hello"], cfg={"do_profile": True}, runner_fn=fake_shell_succeed)
|
|
42
|
+
hello ++hydra.callbacks.profiler._target_=hydra_profiler.profiler.ProfilerCallback
|
|
43
|
+
>>> run_command(["hello"], cfg={"do_profile": False}, runner_fn=fake_shell_succeed)
|
|
44
|
+
hello
|
|
45
|
+
>>> run_command(["hello"], cfg={"seed": 42}, runner_fn=fake_shell_succeed)
|
|
46
|
+
hello ++seed=42
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
if cfg is None:
|
|
50
|
+
do_overwrite = None
|
|
51
|
+
do_profile = False
|
|
52
|
+
seed = None
|
|
53
|
+
else:
|
|
54
|
+
do_overwrite = cfg.get("do_overwrite", None)
|
|
55
|
+
do_profile = cfg.get("do_profile", False)
|
|
56
|
+
seed = cfg.get("seed", None)
|
|
57
|
+
|
|
58
|
+
if do_overwrite is not None:
|
|
59
|
+
command_parts.append(f"++do_overwrite={do_overwrite}")
|
|
60
|
+
if seed is not None:
|
|
61
|
+
command_parts.append(f"++seed={seed}")
|
|
62
|
+
if do_profile:
|
|
63
|
+
command_parts.append("++hydra.callbacks.profiler._target_=hydra_profiler.profiler.ProfilerCallback")
|
|
64
|
+
|
|
65
|
+
full_cmd = " ".join(command_parts)
|
|
66
|
+
logger.info(f"Running command: {full_cmd}")
|
|
67
|
+
command_out = runner_fn(full_cmd, shell=True, capture_output=True)
|
|
68
|
+
|
|
69
|
+
# https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
|
|
70
|
+
# https://loguru.readthedocs.io/en/stable/api/logger.html#loguru._logger.Logger.parse
|
|
71
|
+
|
|
72
|
+
stderr = command_out.stderr.decode()
|
|
73
|
+
stdout = command_out.stdout.decode()
|
|
74
|
+
logger.info(f"Command output:\n{stdout}")
|
|
75
|
+
|
|
76
|
+
if command_out.returncode != 0:
|
|
77
|
+
logger.error(f"Command failed with return code {command_out.returncode}.")
|
|
78
|
+
logger.error(f"Command stderr:\n{stderr}")
|
|
79
|
+
raise ValueError(f"Command failed with return code {command_out.returncode}.")
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
defaults:
|
|
2
|
+
- _extract
|
|
3
|
+
- _self_
|
|
4
|
+
|
|
5
|
+
description: |-
|
|
6
|
+
This pipeline extracts a dataset in longitudinal, sparse form from an input dataset meeting
|
|
7
|
+
select criteria and converts them to the MEDS format.
|
|
8
|
+
|
|
9
|
+
This pipeline is for the ${etl_metadata.dataset_name} dataset version ${etl_metadata.dataset_version}.
|
|
10
|
+
|
|
11
|
+
Key variables for this pipeline are
|
|
12
|
+
- `input_dir=$PATH_TO_INPUT_DIR`
|
|
13
|
+
- `cohort_dir=$PATH_TO_OUTPUT_DIR`.
|
|
14
|
+
- `event_conversion_config_fp=$PATH_TO_EVENT_CONVERSION_CONFIG_FP`
|
|
15
|
+
|
|
16
|
+
See the MEDS-Transforms Extract documentation for configuration file details.
|
|
17
|
+
|
|
18
|
+
# The event conversion configuration file is used throughout the pipeline to define the events to extract.
|
|
19
|
+
event_conversion_config_fp: ${oc.env:EVENT_CONVERSION_CONFIG_FP}
|
|
20
|
+
|
|
21
|
+
input_dir: ${oc.env:PRE_MEDS_DIR}
|
|
22
|
+
cohort_dir: ${oc.env:MEDS_COHORT_DIR}
|
|
23
|
+
|
|
24
|
+
etl_metadata:
|
|
25
|
+
dataset_name: ${oc.env:DATASET_NAME}
|
|
26
|
+
dataset_version: ${oc.env:DATASET_VERSION}
|
|
27
|
+
|
|
28
|
+
stage_configs:
|
|
29
|
+
shard_events:
|
|
30
|
+
infer_schema_length: 999999999
|
|
31
|
+
split_and_shard_subjects:
|
|
32
|
+
n_subjects_per_shard: 1000
|
|
33
|
+
|
|
34
|
+
stages:
|
|
35
|
+
- shard_events
|
|
36
|
+
- split_and_shard_subjects
|
|
37
|
+
- convert_to_sharded_events
|
|
38
|
+
- merge_to_MEDS_cohort
|
|
39
|
+
- extract_code_metadata
|
|
40
|
+
- finalize_MEDS_metadata
|
|
41
|
+
- finalize_MEDS_data
|
|
42
|
+
|
|
43
|
+
parallelize:
|
|
44
|
+
n_workers: ${oc.env:N_WORKERS}
|
|
45
|
+
launcher: "joblib"
|
|
File without changes
|
|
@@ -0,0 +1,324 @@
|
|
|
1
|
+
subject_id_col: subject_id
|
|
2
|
+
nw_hosp/admissions:
|
|
3
|
+
ed_registration:
|
|
4
|
+
code: ED_REGISTRATION
|
|
5
|
+
time: col(edregtime)
|
|
6
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
7
|
+
ed_out:
|
|
8
|
+
code: ED_OUT
|
|
9
|
+
time: col(edouttime)
|
|
10
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
11
|
+
admission:
|
|
12
|
+
code:
|
|
13
|
+
- HOSPITAL_ADMISSION
|
|
14
|
+
- col(admission_type)
|
|
15
|
+
- col(admission_location)
|
|
16
|
+
time: col(admittime)
|
|
17
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
18
|
+
insurance: insurance
|
|
19
|
+
language: language
|
|
20
|
+
marital_status: marital_status
|
|
21
|
+
race: race
|
|
22
|
+
hadm_id: hadm_id
|
|
23
|
+
discharge:
|
|
24
|
+
code:
|
|
25
|
+
- HOSPITAL_DISCHARGE
|
|
26
|
+
- col(discharge_location)
|
|
27
|
+
time: col(dischtime)
|
|
28
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
29
|
+
hadm_id: hadm_id
|
|
30
|
+
# We omit the death event here as it is joined to the data in the subjects table in the pre-MEDS step.
|
|
31
|
+
|
|
32
|
+
nw_hosp/diagnoses_icd:
|
|
33
|
+
diagnosis:
|
|
34
|
+
code:
|
|
35
|
+
- DIAGNOSIS
|
|
36
|
+
- ICD
|
|
37
|
+
- col(icd_version)
|
|
38
|
+
- col(icd_code)
|
|
39
|
+
hadm_id: hadm_id
|
|
40
|
+
time: col(hadm_discharge_time)
|
|
41
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
42
|
+
_metadata:
|
|
43
|
+
nw_hosp/d_icd_diagnoses:
|
|
44
|
+
description: "long_title"
|
|
45
|
+
parent_codes: "ICD{icd_version}CM/{norm_icd_code}" # Single strings are templates of columns.
|
|
46
|
+
|
|
47
|
+
#nw_hosp/drgcodes:
|
|
48
|
+
# drg:
|
|
49
|
+
# code:
|
|
50
|
+
# - DRG
|
|
51
|
+
# - col(drg_type)
|
|
52
|
+
# - col(drg_code)
|
|
53
|
+
# - col(description)
|
|
54
|
+
# hadm_id: hadm_id
|
|
55
|
+
# time: col(hadm_discharge_time)
|
|
56
|
+
# time_format: "%Y-%m-%d %H:%M:%S"
|
|
57
|
+
# drg_severity: drg_severity
|
|
58
|
+
# drg_mortality: drg_mortality
|
|
59
|
+
|
|
60
|
+
nw_hosp/emar:
|
|
61
|
+
medication:
|
|
62
|
+
code:
|
|
63
|
+
- MEDICATION
|
|
64
|
+
- col(medication)
|
|
65
|
+
- col(event_txt)
|
|
66
|
+
time: col(charttime)
|
|
67
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
68
|
+
hadm_id: hadm_id
|
|
69
|
+
emar_id: emar_id
|
|
70
|
+
emar_seq: emar_seq
|
|
71
|
+
|
|
72
|
+
#nw_hosp/hcpcsevents:
|
|
73
|
+
# hcpcs:
|
|
74
|
+
# code:
|
|
75
|
+
# - HCPCS
|
|
76
|
+
# - col(short_description)
|
|
77
|
+
# hadm_id: hadm_id
|
|
78
|
+
# time: col(chartdate)
|
|
79
|
+
# time_format: "%Y-%m-%d"
|
|
80
|
+
# _metadata:
|
|
81
|
+
# # These are not all CPT codes, unfortunately
|
|
82
|
+
# nw_hosp/d_hcpcs:
|
|
83
|
+
# description: "long_description"
|
|
84
|
+
# possibly_cpt_code: "code"
|
|
85
|
+
|
|
86
|
+
nw_hosp/labevents:
|
|
87
|
+
lab:
|
|
88
|
+
code:
|
|
89
|
+
- LAB
|
|
90
|
+
- col(itemid)
|
|
91
|
+
- col(valueuom)
|
|
92
|
+
hadm_id: hadm_id
|
|
93
|
+
time: col(charttime)
|
|
94
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
95
|
+
numeric_value: valuenum
|
|
96
|
+
text_value: value
|
|
97
|
+
priority: priority
|
|
98
|
+
_metadata:
|
|
99
|
+
d_labitems_to_loinc:
|
|
100
|
+
description: ["omop_concept_name", "label"] # List of strings are columns to be collated
|
|
101
|
+
itemid: "itemid (omop_source_code)"
|
|
102
|
+
parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
|
|
103
|
+
valueuom: "valueuom"
|
|
104
|
+
|
|
105
|
+
#nw_hosp/omr:
|
|
106
|
+
# omr:
|
|
107
|
+
# code: col(result_name)
|
|
108
|
+
# text_value: col(result_value)
|
|
109
|
+
# time: col(chartdate)
|
|
110
|
+
# time_format: "%Y-%m-%d"
|
|
111
|
+
|
|
112
|
+
nw_hosp/patients:
|
|
113
|
+
gender:
|
|
114
|
+
code:
|
|
115
|
+
- GENDER
|
|
116
|
+
- col(gender)
|
|
117
|
+
time: null
|
|
118
|
+
dob:
|
|
119
|
+
code: MEDS_BIRTH # This is the MEDS official code for birth.
|
|
120
|
+
time: col(year_of_birth)
|
|
121
|
+
time_format: "%Y"
|
|
122
|
+
death:
|
|
123
|
+
code: MEDS_DEATH # This is the MEDS official code for death.
|
|
124
|
+
time: col(dod)
|
|
125
|
+
time_format:
|
|
126
|
+
- "%Y-%m-%d %H:%M:%S"
|
|
127
|
+
- "%Y-%m-%d"
|
|
128
|
+
|
|
129
|
+
#nw_hosp/pharmacy:
|
|
130
|
+
# medication_start:
|
|
131
|
+
# code:
|
|
132
|
+
# - MEDICATION
|
|
133
|
+
# - START
|
|
134
|
+
# - col(medication)
|
|
135
|
+
# time: col(starttime)
|
|
136
|
+
# route: route
|
|
137
|
+
# frequency: frequency
|
|
138
|
+
# doses_per_24_hrs: doses_per_24_hrs
|
|
139
|
+
# poe_id: poe_id
|
|
140
|
+
# time_format:
|
|
141
|
+
# - "%Y-%m-%d %H:%M:%S"
|
|
142
|
+
# - "%Y-%m-%d"
|
|
143
|
+
# medication_stop:
|
|
144
|
+
# code:
|
|
145
|
+
# - MEDICATION
|
|
146
|
+
# - STOP
|
|
147
|
+
# - col(medication)
|
|
148
|
+
# time: col(stoptime)
|
|
149
|
+
# poe_id: poe_id
|
|
150
|
+
# time_format:
|
|
151
|
+
# - "%Y-%m-%d %H:%M:%S"
|
|
152
|
+
# - "%Y-%m-%d"
|
|
153
|
+
#
|
|
154
|
+
#nw_hosp/procedures_icd:
|
|
155
|
+
# procedure:
|
|
156
|
+
# code:
|
|
157
|
+
# - PROCEDURE
|
|
158
|
+
# - ICD
|
|
159
|
+
# - col(icd_version)
|
|
160
|
+
# - col(icd_code)
|
|
161
|
+
# hadm_id: hadm_id
|
|
162
|
+
# time: col(chartdate)
|
|
163
|
+
# time_format: "%Y-%m-%d"
|
|
164
|
+
# _metadata:
|
|
165
|
+
# nw_hosp/d_icd_procedures:
|
|
166
|
+
# description: "long_title"
|
|
167
|
+
# parent_codes: # List of objects are string labels mapping to filters to be evaluated.
|
|
168
|
+
# - "ICD{icd_version}Proc/{norm_icd_code}": { icd_version: "9" }
|
|
169
|
+
# - "ICD{icd_version}PCS/{norm_icd_code}": { icd_version: "10" }
|
|
170
|
+
#
|
|
171
|
+
#nw_hosp/transfers:
|
|
172
|
+
# transfer:
|
|
173
|
+
# code:
|
|
174
|
+
# - TRANSFER_TO
|
|
175
|
+
# - col(eventtype)
|
|
176
|
+
# - col(careunit)
|
|
177
|
+
# time: col(intime)
|
|
178
|
+
# time_format: "%Y-%m-%d %H:%M:%S"
|
|
179
|
+
# hadm_id: hadm_id
|
|
180
|
+
|
|
181
|
+
nw_icu/icustays:
|
|
182
|
+
icu_admission:
|
|
183
|
+
code:
|
|
184
|
+
- ICU_ADMISSION
|
|
185
|
+
- col(first_careunit)
|
|
186
|
+
time: col(intime)
|
|
187
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
188
|
+
hadm_id: hadm_id
|
|
189
|
+
icustay_id: stay_id
|
|
190
|
+
icu_discharge:
|
|
191
|
+
code:
|
|
192
|
+
- ICU_DISCHARGE
|
|
193
|
+
- col(last_careunit)
|
|
194
|
+
time: col(outtime)
|
|
195
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
196
|
+
hadm_id: hadm_id
|
|
197
|
+
icustay_id: stay_id
|
|
198
|
+
|
|
199
|
+
nw_icu/chartevents:
|
|
200
|
+
event:
|
|
201
|
+
code:
|
|
202
|
+
- LAB
|
|
203
|
+
- col(itemid)
|
|
204
|
+
- col(valueuom)
|
|
205
|
+
time: col(charttime)
|
|
206
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
207
|
+
numeric_value: valuenum
|
|
208
|
+
text_value: value
|
|
209
|
+
hadm_id: hadm_id
|
|
210
|
+
icustay_id: stay_id
|
|
211
|
+
_metadata:
|
|
212
|
+
# TODO: make this work even with missing valueuom
|
|
213
|
+
#meas_chartevents_main:
|
|
214
|
+
# description: ["omop_concept_name", "label"] # List of strings are columns to be collated
|
|
215
|
+
# itemid: "itemid (omop_source_code)"
|
|
216
|
+
# parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
|
|
217
|
+
# TODO: I don't know if this is necessary...
|
|
218
|
+
d_labitems_to_loinc:
|
|
219
|
+
description: ["omop_concept_name", "label"] # List of strings are columns to be collated
|
|
220
|
+
itemid: "itemid (omop_source_code)"
|
|
221
|
+
parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
|
|
222
|
+
valueuom: "valueuom"
|
|
223
|
+
|
|
224
|
+
nw_icu/procedureevents:
|
|
225
|
+
start:
|
|
226
|
+
code:
|
|
227
|
+
- PROCEDURE
|
|
228
|
+
- START
|
|
229
|
+
- col(itemid)
|
|
230
|
+
time: col(starttime)
|
|
231
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
232
|
+
hadm_id: hadm_id
|
|
233
|
+
icustay_id: stay_id
|
|
234
|
+
_metadata:
|
|
235
|
+
proc_datetimeevents:
|
|
236
|
+
description: ["omop_concept_name", "label"] # List of strings are columns to be collated
|
|
237
|
+
itemid: "itemid (omop_source_code)"
|
|
238
|
+
parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
|
|
239
|
+
proc_itemid:
|
|
240
|
+
description: ["omop_concept_name", "label"] # List of strings are columns to be collated
|
|
241
|
+
itemid: "itemid (omop_source_code)"
|
|
242
|
+
parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
|
|
243
|
+
end:
|
|
244
|
+
code:
|
|
245
|
+
- PROCEDURE
|
|
246
|
+
- END
|
|
247
|
+
- col(itemid)
|
|
248
|
+
time: col(endtime)
|
|
249
|
+
time_format: "%Y-%m-%d %H:%M:%S"
|
|
250
|
+
hadm_id: hadm_id
|
|
251
|
+
icustay_id: stay_id
|
|
252
|
+
_metadata:
|
|
253
|
+
proc_datetimeevents:
|
|
254
|
+
description: ["omop_concept_name", "label"] # List of strings are columns to be collated
|
|
255
|
+
itemid: "itemid (omop_source_code)"
|
|
256
|
+
parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
|
|
257
|
+
proc_itemid:
|
|
258
|
+
description: ["omop_concept_name", "label"] # List of strings are columns to be collated
|
|
259
|
+
itemid: "itemid (omop_source_code)"
|
|
260
|
+
parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
|
|
261
|
+
|
|
262
|
+
#nw_icu/inputevents:
|
|
263
|
+
# input_start:
|
|
264
|
+
# code:
|
|
265
|
+
# - INFUSION_START
|
|
266
|
+
# - col(itemid)
|
|
267
|
+
# time: col(starttime)
|
|
268
|
+
# time_format: "%Y-%m-%d %H:%M:%S"
|
|
269
|
+
# hadm_id: hadm_id
|
|
270
|
+
# icustay_id: stay_id
|
|
271
|
+
# order_id: orderid
|
|
272
|
+
# link_order_id: linkorderid
|
|
273
|
+
# numeric_value: rate
|
|
274
|
+
# unit: rateuom
|
|
275
|
+
# ordercategorydescription: ordercategorydescription
|
|
276
|
+
# _metadata:
|
|
277
|
+
# inputevents_to_rxnorm:
|
|
278
|
+
# description: ["omop_concept_name", "label"] # List of strings are columns to be collated
|
|
279
|
+
# itemid: "itemid (omop_source_code)"
|
|
280
|
+
# parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
|
|
281
|
+
# input_end:
|
|
282
|
+
# code:
|
|
283
|
+
# - INFUSION_END
|
|
284
|
+
# - col(itemid)
|
|
285
|
+
# time: col(endtime)
|
|
286
|
+
# time_format: "%Y-%m-%d %H:%M:%S"
|
|
287
|
+
# hadm_id: hadm_id
|
|
288
|
+
# icustay_id: stay_id
|
|
289
|
+
# order_id: orderid
|
|
290
|
+
# link_order_id: linkorderid
|
|
291
|
+
# numeric_value: amount
|
|
292
|
+
# ordercategorydescription: ordercategorydescription
|
|
293
|
+
# statusdescription: statusdescription
|
|
294
|
+
# unit: amountuom
|
|
295
|
+
# _metadata:
|
|
296
|
+
# inputevents_to_rxnorm:
|
|
297
|
+
# description: ["omop_concept_name", "label"] # List of strings are columns to be collated
|
|
298
|
+
# itemid: "itemid (omop_source_code)"
|
|
299
|
+
# parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
|
|
300
|
+
# subject_weight:
|
|
301
|
+
# code:
|
|
302
|
+
# - SUBJECT_WEIGHT_AT_INFUSION
|
|
303
|
+
# - KG
|
|
304
|
+
# time: col(starttime)
|
|
305
|
+
# time_format: "%Y-%m-%d %H:%M:%S"
|
|
306
|
+
# numeric_value: patientweight
|
|
307
|
+
#
|
|
308
|
+
#nw_icu/outputevents:
|
|
309
|
+
# output:
|
|
310
|
+
# code:
|
|
311
|
+
# - SUBJECT_FLUID_OUTPUT
|
|
312
|
+
# - col(itemid)
|
|
313
|
+
# - col(valueuom)
|
|
314
|
+
# time: col(charttime)
|
|
315
|
+
# time_format: "%Y-%m-%d %H:%M:%S"
|
|
316
|
+
# hadm_id: hadm_id
|
|
317
|
+
# icustay_id: stay_id
|
|
318
|
+
# numeric_value: value
|
|
319
|
+
# _metadata:
|
|
320
|
+
# outputevents_to_loinc:
|
|
321
|
+
# description: ["omop_concept_name", "label"] # List of strings are columns to be collated
|
|
322
|
+
# itemid: "itemid (omop_source_code)"
|
|
323
|
+
# valueuom: "unitname"
|
|
324
|
+
# parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
defaults:
|
|
2
|
+
- _self_
|
|
3
|
+
|
|
4
|
+
root_output_dir: ???
|
|
5
|
+
|
|
6
|
+
raw_input_dir: ${root_output_dir}/raw_input
|
|
7
|
+
pre_MEDS_dir: ${root_output_dir}/pre_MEDS
|
|
8
|
+
MEDS_cohort_dir: ${root_output_dir}/MEDS_cohort
|
|
9
|
+
|
|
10
|
+
stage_runner_fp: null
|
|
11
|
+
|
|
12
|
+
do_download: True
|
|
13
|
+
do_overwrite: False
|
|
14
|
+
do_demo: False
|
|
15
|
+
|
|
16
|
+
log_dir: ${root_output_dir}/.logs
|
|
17
|
+
|
|
18
|
+
# Hydra
|
|
19
|
+
hydra:
|
|
20
|
+
job:
|
|
21
|
+
name: full_extraction_${now:%Y-%m-%d_%H-%M-%S}
|
|
22
|
+
run:
|
|
23
|
+
dir: ${log_dir}
|
|
24
|
+
sweep:
|
|
25
|
+
dir: ${log_dir}
|
NWICU_MEDS/dataset.yaml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
dataset_name: NWICU
|
|
2
|
+
raw_dataset_version: 0.1.0
|
|
3
|
+
dataset_metadata_version: v2.4.0
|
|
4
|
+
dataset_description: ???
|
|
5
|
+
|
|
6
|
+
metadata_url_base: https://raw.githubusercontent.com/MIT-LCP/mimic-code/${dataset_metadata_version}/mimic-iv/concepts/concept_map
|
|
7
|
+
|
|
8
|
+
urls:
|
|
9
|
+
dataset:
|
|
10
|
+
- url: https://physionet.org/files/nwicu-northwestern-icu/${raw_dataset_version}/data/
|
|
11
|
+
username: ${oc.env:DATASET_DOWNLOAD_USERNAME}
|
|
12
|
+
password: ${oc.env:DATASET_DOWNLOAD_PASSWORD}
|
|
13
|
+
# demo:
|
|
14
|
+
# - https://physionet.org/files/mimic-iv-demo/2.2/
|
|
15
|
+
common:
|
|
16
|
+
- ${metadata_url_base}/d_labitems_to_loinc.csv
|
|
17
|
+
- ${metadata_url_base}/inputevents_to_rxnorm.csv
|
|
18
|
+
- ${metadata_url_base}/lab_itemid_to_loinc.csv
|
|
19
|
+
- ${metadata_url_base}/meas_chartevents_main.csv
|
|
20
|
+
- ${metadata_url_base}/meas_chartevents_value.csv
|
|
21
|
+
- ${metadata_url_base}/numerics-summary.csv
|
|
22
|
+
- ${metadata_url_base}/outputevents_to_loinc.csv
|
|
23
|
+
- ${metadata_url_base}/proc_datetimeevents.csv
|
|
24
|
+
- ${metadata_url_base}/proc_itemid.csv
|
|
25
|
+
- ${metadata_url_base}/waveforms-summary.csv
|