NWICU-MEDS 0.1.dev18__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
NWICU_MEDS/__init__.py ADDED
@@ -0,0 +1,38 @@
1
+ from importlib.metadata import PackageNotFoundError, version
2
+ from importlib.resources import files
3
+
4
+ from omegaconf import OmegaConf
5
+
6
+ __package_name__ = "NWICU_MEDS"
7
+ try:
8
+ __version__ = version(__package_name__)
9
+ except PackageNotFoundError: # pragma: no cover
10
+ __version__ = "unknown"
11
+
12
+ MAIN_CFG = files(__package_name__).joinpath("configs/main.yaml")
13
+ EVENT_CFG = files(__package_name__).joinpath("configs/event_configs.yaml")
14
+ ETL_CFG = files(__package_name__).joinpath("configs/ETL.yaml")
15
+ RUNNER_CFG = files(__package_name__).joinpath("configs/runner.yaml")
16
+ PRE_MEDS_PY = files(__package_name__).joinpath("pre_MEDS.py")
17
+ PRE_MEDS_CFG = files(__package_name__).joinpath("configs/pre_MEDS.yaml")
18
+ DATASET_CFG = files(__package_name__).joinpath("dataset.yaml")
19
+
20
+ dataset_info = OmegaConf.load(DATASET_CFG)
21
+
22
+ HAS_PRE_MEDS = PRE_MEDS_PY.exists()
23
+
24
+ event_config = OmegaConf.load(EVENT_CFG)
25
+
26
+ __all__ = [
27
+ "event_config",
28
+ "EVENT_CFG",
29
+ "ETL_CFG",
30
+ "HAS_PRE_MEDS",
31
+ "PRE_MEDS_CFG",
32
+ "MAIN_CFG",
33
+ "RUNNER_CFG",
34
+ "DATASET_CFG",
35
+ "dataset_info",
36
+ "__package_name__",
37
+ "__version__",
38
+ ]
NWICU_MEDS/__main__.py ADDED
@@ -0,0 +1,82 @@
1
+ #!/usr/bin/env python
2
+
3
+ import logging
4
+ import os
5
+ from pathlib import Path
6
+
7
+ import hydra
8
+ from omegaconf import DictConfig
9
+
10
+ from . import ETL_CFG, EVENT_CFG, HAS_PRE_MEDS, MAIN_CFG, PRE_MEDS_PY, RUNNER_CFG
11
+ from . import __version__ as PKG_VERSION
12
+ from . import dataset_info
13
+ from .commands import run_command
14
+ from .download import download_data
15
+
16
+ logger = logging.getLogger(__name__)
17
+
18
+
19
+ @hydra.main(version_base=None, config_path=str(MAIN_CFG.parent), config_name=MAIN_CFG.stem)
20
+ def main(cfg: DictConfig):
21
+ """Runs the end-to-end MEDS Extraction pipeline."""
22
+
23
+ raw_input_dir = Path(cfg.raw_input_dir)
24
+ pre_MEDS_dir = Path(cfg.pre_MEDS_dir)
25
+ MEDS_cohort_dir = Path(cfg.MEDS_cohort_dir)
26
+ stage_runner_fp = cfg.get("stage_runner_fp", None)
27
+
28
+ # Step 0: Data downloading
29
+ if cfg.do_download:
30
+ if cfg.get("do_demo", False):
31
+ logger.info("Downloading demo data.")
32
+ download_data(raw_input_dir, dataset_info, do_demo=True)
33
+ else:
34
+ logger.info("Downloading data.")
35
+ download_data(raw_input_dir, dataset_info)
36
+ else: # pragma: no cover
37
+ logger.info("Skipping data download.")
38
+
39
+ # Step 1: Pre-MEDS Data Wrangling
40
+ if HAS_PRE_MEDS:
41
+ command_parts = [
42
+ "python",
43
+ str(PRE_MEDS_PY),
44
+ f"input_dir={raw_input_dir}",
45
+ f"output_dir={pre_MEDS_dir}",
46
+ ]
47
+ run_command(command_parts, cfg)
48
+ else:
49
+ pre_MEDS_dir = f"{raw_input_dir}"
50
+
51
+ # Step 2: MEDS Cohort Creation
52
+ # First we need to set some environment variables
53
+ command_parts = [
54
+ f"DATASET_NAME={dataset_info.dataset_name}",
55
+ f"DATASET_VERSION={dataset_info.raw_dataset_version}:{PKG_VERSION}",
56
+ f"EVENT_CONVERSION_CONFIG_FP={str(EVENT_CFG.resolve())}",
57
+ f"PRE_MEDS_DIR={str(pre_MEDS_dir.resolve())}",
58
+ f"MEDS_COHORT_DIR={str(MEDS_cohort_dir.resolve())}",
59
+ ]
60
+
61
+ # Then we construct the rest of the command
62
+ command_parts.extend(
63
+ [
64
+ "MEDS_transform-runner",
65
+ f"--config-path={str(RUNNER_CFG.parent.resolve())}",
66
+ f"--config-name={RUNNER_CFG.stem}",
67
+ f"pipeline_config_fp={str(ETL_CFG.resolve())}",
68
+ ]
69
+ )
70
+ if int(os.getenv("N_WORKERS", 1)) <= 1:
71
+ logger.info("Running in serial mode as N_WORKERS is not set.")
72
+ command_parts.append("~parallelize")
73
+
74
+ if stage_runner_fp:
75
+ command_parts.append(f"stage_runner_fp={stage_runner_fp}")
76
+
77
+ command_parts.append("'hydra.searchpath=[pkg://MEDS_transforms.configs]'")
78
+ run_command(command_parts, cfg)
79
+
80
+
81
+ if __name__ == "__main__":
82
+ main()
NWICU_MEDS/commands.py ADDED
@@ -0,0 +1,79 @@
1
+ import logging
2
+ import subprocess
3
+
4
+ from omegaconf import DictConfig
5
+
6
+ logger = logging.getLogger(__name__)
7
+
8
+
9
+ def run_command(
10
+ command_parts: list[str],
11
+ cfg: DictConfig | None = None,
12
+ runner_fn: callable = subprocess.run,
13
+ ):
14
+ """Runs a command with the specified runner function.
15
+
16
+ Args:
17
+ command_parts: A list of the arguments to be run as a shell command.
18
+ cfg: The configuration for the command.
19
+ runner_fn: The function to run the command with (added for dependency injection).
20
+
21
+ Raises:
22
+ ValueError: If the command fails
23
+
24
+ Examples:
25
+ >>> def fake_shell_succeed(cmd, shell, capture_output):
26
+ ... print(cmd)
27
+ ... return subprocess.CompletedProcess(args=cmd, returncode=0, stdout=b"", stderr=b"")
28
+ >>> def fake_shell_fail(cmd, shell, capture_output):
29
+ ... print(cmd)
30
+ ... return subprocess.CompletedProcess(args=cmd, returncode=1, stdout=b"", stderr=b"")
31
+ >>> run_command(["echo", "hello"], runner_fn=fake_shell_succeed)
32
+ echo hello
33
+ >>> run_command(["echo", "hello"], runner_fn=fake_shell_fail)
34
+ Traceback (most recent call last):
35
+ ...
36
+ ValueError: Command failed with return code 1.
37
+ >>> run_command(["hello"], cfg={"do_overwrite": True}, runner_fn=fake_shell_succeed)
38
+ hello ++do_overwrite=True
39
+ >>> run_command(["hello"], cfg={"do_overwrite": False}, runner_fn=fake_shell_succeed)
40
+ hello ++do_overwrite=False
41
+ >>> run_command(["hello"], cfg={"do_profile": True}, runner_fn=fake_shell_succeed)
42
+ hello ++hydra.callbacks.profiler._target_=hydra_profiler.profiler.ProfilerCallback
43
+ >>> run_command(["hello"], cfg={"do_profile": False}, runner_fn=fake_shell_succeed)
44
+ hello
45
+ >>> run_command(["hello"], cfg={"seed": 42}, runner_fn=fake_shell_succeed)
46
+ hello ++seed=42
47
+ """
48
+
49
+ if cfg is None:
50
+ do_overwrite = None
51
+ do_profile = False
52
+ seed = None
53
+ else:
54
+ do_overwrite = cfg.get("do_overwrite", None)
55
+ do_profile = cfg.get("do_profile", False)
56
+ seed = cfg.get("seed", None)
57
+
58
+ if do_overwrite is not None:
59
+ command_parts.append(f"++do_overwrite={do_overwrite}")
60
+ if seed is not None:
61
+ command_parts.append(f"++seed={seed}")
62
+ if do_profile:
63
+ command_parts.append("++hydra.callbacks.profiler._target_=hydra_profiler.profiler.ProfilerCallback")
64
+
65
+ full_cmd = " ".join(command_parts)
66
+ logger.info(f"Running command: {full_cmd}")
67
+ command_out = runner_fn(full_cmd, shell=True, capture_output=True)
68
+
69
+ # https://stackoverflow.com/questions/21953835/run-subprocess-and-print-output-to-logging
70
+ # https://loguru.readthedocs.io/en/stable/api/logger.html#loguru._logger.Logger.parse
71
+
72
+ stderr = command_out.stderr.decode()
73
+ stdout = command_out.stdout.decode()
74
+ logger.info(f"Command output:\n{stdout}")
75
+
76
+ if command_out.returncode != 0:
77
+ logger.error(f"Command failed with return code {command_out.returncode}.")
78
+ logger.error(f"Command stderr:\n{stderr}")
79
+ raise ValueError(f"Command failed with return code {command_out.returncode}.")
@@ -0,0 +1,45 @@
1
+ defaults:
2
+ - _extract
3
+ - _self_
4
+
5
+ description: |-
6
+ This pipeline extracts a dataset in longitudinal, sparse form from an input dataset meeting
7
+ select criteria and converts them to the MEDS format.
8
+
9
+ This pipeline is for the ${etl_metadata.dataset_name} dataset version ${etl_metadata.dataset_version}.
10
+
11
+ Key variables for this pipeline are
12
+ - `input_dir=$PATH_TO_INPUT_DIR`
13
+ - `cohort_dir=$PATH_TO_OUTPUT_DIR`.
14
+ - `event_conversion_config_fp=$PATH_TO_EVENT_CONVERSION_CONFIG_FP`
15
+
16
+ See the MEDS-Transforms Extract documentation for configuration file details.
17
+
18
+ # The event conversion configuration file is used throughout the pipeline to define the events to extract.
19
+ event_conversion_config_fp: ${oc.env:EVENT_CONVERSION_CONFIG_FP}
20
+
21
+ input_dir: ${oc.env:PRE_MEDS_DIR}
22
+ cohort_dir: ${oc.env:MEDS_COHORT_DIR}
23
+
24
+ etl_metadata:
25
+ dataset_name: ${oc.env:DATASET_NAME}
26
+ dataset_version: ${oc.env:DATASET_VERSION}
27
+
28
+ stage_configs:
29
+ shard_events:
30
+ infer_schema_length: 999999999
31
+ split_and_shard_subjects:
32
+ n_subjects_per_shard: 1000
33
+
34
+ stages:
35
+ - shard_events
36
+ - split_and_shard_subjects
37
+ - convert_to_sharded_events
38
+ - merge_to_MEDS_cohort
39
+ - extract_code_metadata
40
+ - finalize_MEDS_metadata
41
+ - finalize_MEDS_data
42
+
43
+ parallelize:
44
+ n_workers: ${oc.env:N_WORKERS}
45
+ launcher: "joblib"
File without changes
@@ -0,0 +1,324 @@
1
+ subject_id_col: subject_id
2
+ nw_hosp/admissions:
3
+ ed_registration:
4
+ code: ED_REGISTRATION
5
+ time: col(edregtime)
6
+ time_format: "%Y-%m-%d %H:%M:%S"
7
+ ed_out:
8
+ code: ED_OUT
9
+ time: col(edouttime)
10
+ time_format: "%Y-%m-%d %H:%M:%S"
11
+ admission:
12
+ code:
13
+ - HOSPITAL_ADMISSION
14
+ - col(admission_type)
15
+ - col(admission_location)
16
+ time: col(admittime)
17
+ time_format: "%Y-%m-%d %H:%M:%S"
18
+ insurance: insurance
19
+ language: language
20
+ marital_status: marital_status
21
+ race: race
22
+ hadm_id: hadm_id
23
+ discharge:
24
+ code:
25
+ - HOSPITAL_DISCHARGE
26
+ - col(discharge_location)
27
+ time: col(dischtime)
28
+ time_format: "%Y-%m-%d %H:%M:%S"
29
+ hadm_id: hadm_id
30
+ # We omit the death event here as it is joined to the data in the subjects table in the pre-MEDS step.
31
+
32
+ nw_hosp/diagnoses_icd:
33
+ diagnosis:
34
+ code:
35
+ - DIAGNOSIS
36
+ - ICD
37
+ - col(icd_version)
38
+ - col(icd_code)
39
+ hadm_id: hadm_id
40
+ time: col(hadm_discharge_time)
41
+ time_format: "%Y-%m-%d %H:%M:%S"
42
+ _metadata:
43
+ nw_hosp/d_icd_diagnoses:
44
+ description: "long_title"
45
+ parent_codes: "ICD{icd_version}CM/{norm_icd_code}" # Single strings are templates of columns.
46
+
47
+ #nw_hosp/drgcodes:
48
+ # drg:
49
+ # code:
50
+ # - DRG
51
+ # - col(drg_type)
52
+ # - col(drg_code)
53
+ # - col(description)
54
+ # hadm_id: hadm_id
55
+ # time: col(hadm_discharge_time)
56
+ # time_format: "%Y-%m-%d %H:%M:%S"
57
+ # drg_severity: drg_severity
58
+ # drg_mortality: drg_mortality
59
+
60
+ nw_hosp/emar:
61
+ medication:
62
+ code:
63
+ - MEDICATION
64
+ - col(medication)
65
+ - col(event_txt)
66
+ time: col(charttime)
67
+ time_format: "%Y-%m-%d %H:%M:%S"
68
+ hadm_id: hadm_id
69
+ emar_id: emar_id
70
+ emar_seq: emar_seq
71
+
72
+ #nw_hosp/hcpcsevents:
73
+ # hcpcs:
74
+ # code:
75
+ # - HCPCS
76
+ # - col(short_description)
77
+ # hadm_id: hadm_id
78
+ # time: col(chartdate)
79
+ # time_format: "%Y-%m-%d"
80
+ # _metadata:
81
+ # # These are not all CPT codes, unfortunately
82
+ # nw_hosp/d_hcpcs:
83
+ # description: "long_description"
84
+ # possibly_cpt_code: "code"
85
+
86
+ nw_hosp/labevents:
87
+ lab:
88
+ code:
89
+ - LAB
90
+ - col(itemid)
91
+ - col(valueuom)
92
+ hadm_id: hadm_id
93
+ time: col(charttime)
94
+ time_format: "%Y-%m-%d %H:%M:%S"
95
+ numeric_value: valuenum
96
+ text_value: value
97
+ priority: priority
98
+ _metadata:
99
+ d_labitems_to_loinc:
100
+ description: ["omop_concept_name", "label"] # List of strings are columns to be collated
101
+ itemid: "itemid (omop_source_code)"
102
+ parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
103
+ valueuom: "valueuom"
104
+
105
+ #nw_hosp/omr:
106
+ # omr:
107
+ # code: col(result_name)
108
+ # text_value: col(result_value)
109
+ # time: col(chartdate)
110
+ # time_format: "%Y-%m-%d"
111
+
112
+ nw_hosp/patients:
113
+ gender:
114
+ code:
115
+ - GENDER
116
+ - col(gender)
117
+ time: null
118
+ dob:
119
+ code: MEDS_BIRTH # This is the MEDS official code for birth.
120
+ time: col(year_of_birth)
121
+ time_format: "%Y"
122
+ death:
123
+ code: MEDS_DEATH # This is the MEDS official code for death.
124
+ time: col(dod)
125
+ time_format:
126
+ - "%Y-%m-%d %H:%M:%S"
127
+ - "%Y-%m-%d"
128
+
129
+ #nw_hosp/pharmacy:
130
+ # medication_start:
131
+ # code:
132
+ # - MEDICATION
133
+ # - START
134
+ # - col(medication)
135
+ # time: col(starttime)
136
+ # route: route
137
+ # frequency: frequency
138
+ # doses_per_24_hrs: doses_per_24_hrs
139
+ # poe_id: poe_id
140
+ # time_format:
141
+ # - "%Y-%m-%d %H:%M:%S"
142
+ # - "%Y-%m-%d"
143
+ # medication_stop:
144
+ # code:
145
+ # - MEDICATION
146
+ # - STOP
147
+ # - col(medication)
148
+ # time: col(stoptime)
149
+ # poe_id: poe_id
150
+ # time_format:
151
+ # - "%Y-%m-%d %H:%M:%S"
152
+ # - "%Y-%m-%d"
153
+ #
154
+ #nw_hosp/procedures_icd:
155
+ # procedure:
156
+ # code:
157
+ # - PROCEDURE
158
+ # - ICD
159
+ # - col(icd_version)
160
+ # - col(icd_code)
161
+ # hadm_id: hadm_id
162
+ # time: col(chartdate)
163
+ # time_format: "%Y-%m-%d"
164
+ # _metadata:
165
+ # nw_hosp/d_icd_procedures:
166
+ # description: "long_title"
167
+ # parent_codes: # List of objects are string labels mapping to filters to be evaluated.
168
+ # - "ICD{icd_version}Proc/{norm_icd_code}": { icd_version: "9" }
169
+ # - "ICD{icd_version}PCS/{norm_icd_code}": { icd_version: "10" }
170
+ #
171
+ #nw_hosp/transfers:
172
+ # transfer:
173
+ # code:
174
+ # - TRANSFER_TO
175
+ # - col(eventtype)
176
+ # - col(careunit)
177
+ # time: col(intime)
178
+ # time_format: "%Y-%m-%d %H:%M:%S"
179
+ # hadm_id: hadm_id
180
+
181
+ nw_icu/icustays:
182
+ icu_admission:
183
+ code:
184
+ - ICU_ADMISSION
185
+ - col(first_careunit)
186
+ time: col(intime)
187
+ time_format: "%Y-%m-%d %H:%M:%S"
188
+ hadm_id: hadm_id
189
+ icustay_id: stay_id
190
+ icu_discharge:
191
+ code:
192
+ - ICU_DISCHARGE
193
+ - col(last_careunit)
194
+ time: col(outtime)
195
+ time_format: "%Y-%m-%d %H:%M:%S"
196
+ hadm_id: hadm_id
197
+ icustay_id: stay_id
198
+
199
+ nw_icu/chartevents:
200
+ event:
201
+ code:
202
+ - LAB
203
+ - col(itemid)
204
+ - col(valueuom)
205
+ time: col(charttime)
206
+ time_format: "%Y-%m-%d %H:%M:%S"
207
+ numeric_value: valuenum
208
+ text_value: value
209
+ hadm_id: hadm_id
210
+ icustay_id: stay_id
211
+ _metadata:
212
+ # TODO: make this work even with missing valueuom
213
+ #meas_chartevents_main:
214
+ # description: ["omop_concept_name", "label"] # List of strings are columns to be collated
215
+ # itemid: "itemid (omop_source_code)"
216
+ # parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
217
+ # TODO: I don't know if this is necessary...
218
+ d_labitems_to_loinc:
219
+ description: ["omop_concept_name", "label"] # List of strings are columns to be collated
220
+ itemid: "itemid (omop_source_code)"
221
+ parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
222
+ valueuom: "valueuom"
223
+
224
+ nw_icu/procedureevents:
225
+ start:
226
+ code:
227
+ - PROCEDURE
228
+ - START
229
+ - col(itemid)
230
+ time: col(starttime)
231
+ time_format: "%Y-%m-%d %H:%M:%S"
232
+ hadm_id: hadm_id
233
+ icustay_id: stay_id
234
+ _metadata:
235
+ proc_datetimeevents:
236
+ description: ["omop_concept_name", "label"] # List of strings are columns to be collated
237
+ itemid: "itemid (omop_source_code)"
238
+ parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
239
+ proc_itemid:
240
+ description: ["omop_concept_name", "label"] # List of strings are columns to be collated
241
+ itemid: "itemid (omop_source_code)"
242
+ parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
243
+ end:
244
+ code:
245
+ - PROCEDURE
246
+ - END
247
+ - col(itemid)
248
+ time: col(endtime)
249
+ time_format: "%Y-%m-%d %H:%M:%S"
250
+ hadm_id: hadm_id
251
+ icustay_id: stay_id
252
+ _metadata:
253
+ proc_datetimeevents:
254
+ description: ["omop_concept_name", "label"] # List of strings are columns to be collated
255
+ itemid: "itemid (omop_source_code)"
256
+ parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
257
+ proc_itemid:
258
+ description: ["omop_concept_name", "label"] # List of strings are columns to be collated
259
+ itemid: "itemid (omop_source_code)"
260
+ parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
261
+
262
+ #nw_icu/inputevents:
263
+ # input_start:
264
+ # code:
265
+ # - INFUSION_START
266
+ # - col(itemid)
267
+ # time: col(starttime)
268
+ # time_format: "%Y-%m-%d %H:%M:%S"
269
+ # hadm_id: hadm_id
270
+ # icustay_id: stay_id
271
+ # order_id: orderid
272
+ # link_order_id: linkorderid
273
+ # numeric_value: rate
274
+ # unit: rateuom
275
+ # ordercategorydescription: ordercategorydescription
276
+ # _metadata:
277
+ # inputevents_to_rxnorm:
278
+ # description: ["omop_concept_name", "label"] # List of strings are columns to be collated
279
+ # itemid: "itemid (omop_source_code)"
280
+ # parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
281
+ # input_end:
282
+ # code:
283
+ # - INFUSION_END
284
+ # - col(itemid)
285
+ # time: col(endtime)
286
+ # time_format: "%Y-%m-%d %H:%M:%S"
287
+ # hadm_id: hadm_id
288
+ # icustay_id: stay_id
289
+ # order_id: orderid
290
+ # link_order_id: linkorderid
291
+ # numeric_value: amount
292
+ # ordercategorydescription: ordercategorydescription
293
+ # statusdescription: statusdescription
294
+ # unit: amountuom
295
+ # _metadata:
296
+ # inputevents_to_rxnorm:
297
+ # description: ["omop_concept_name", "label"] # List of strings are columns to be collated
298
+ # itemid: "itemid (omop_source_code)"
299
+ # parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
300
+ # subject_weight:
301
+ # code:
302
+ # - SUBJECT_WEIGHT_AT_INFUSION
303
+ # - KG
304
+ # time: col(starttime)
305
+ # time_format: "%Y-%m-%d %H:%M:%S"
306
+ # numeric_value: patientweight
307
+ #
308
+ #nw_icu/outputevents:
309
+ # output:
310
+ # code:
311
+ # - SUBJECT_FLUID_OUTPUT
312
+ # - col(itemid)
313
+ # - col(valueuom)
314
+ # time: col(charttime)
315
+ # time_format: "%Y-%m-%d %H:%M:%S"
316
+ # hadm_id: hadm_id
317
+ # icustay_id: stay_id
318
+ # numeric_value: value
319
+ # _metadata:
320
+ # outputevents_to_loinc:
321
+ # description: ["omop_concept_name", "label"] # List of strings are columns to be collated
322
+ # itemid: "itemid (omop_source_code)"
323
+ # valueuom: "unitname"
324
+ # parent_codes: "{omop_vocabulary_id}/{omop_concept_code}"
@@ -0,0 +1,25 @@
1
+ defaults:
2
+ - _self_
3
+
4
+ root_output_dir: ???
5
+
6
+ raw_input_dir: ${root_output_dir}/raw_input
7
+ pre_MEDS_dir: ${root_output_dir}/pre_MEDS
8
+ MEDS_cohort_dir: ${root_output_dir}/MEDS_cohort
9
+
10
+ stage_runner_fp: null
11
+
12
+ do_download: True
13
+ do_overwrite: False
14
+ do_demo: False
15
+
16
+ log_dir: ${root_output_dir}/.logs
17
+
18
+ # Hydra
19
+ hydra:
20
+ job:
21
+ name: full_extraction_${now:%Y-%m-%d_%H-%M-%S}
22
+ run:
23
+ dir: ${log_dir}
24
+ sweep:
25
+ dir: ${log_dir}
@@ -0,0 +1,15 @@
1
+ input_dir: ???
2
+ output_dir: ???
3
+
4
+ do_overwrite: false
5
+
6
+ log_dir: ${output_dir}/.logs
7
+
8
+ # Hydra
9
+ hydra:
10
+ job:
11
+ name: pre_MEDS_${now:%Y-%m-%d_%H-%M-%S}
12
+ run:
13
+ dir: ${log_dir}
14
+ sweep:
15
+ dir: ${log_dir}
@@ -0,0 +1,7 @@
1
+ defaults:
2
+ - _runner
3
+ - _self_
4
+
5
+ parallelize:
6
+ n_workers: ${oc.env:N_WORKERS,1}
7
+ launcher: "joblib"
@@ -0,0 +1,25 @@
1
+ dataset_name: NWICU
2
+ raw_dataset_version: 0.1.0
3
+ dataset_metadata_version: v2.4.0
4
+ dataset_description: ???
5
+
6
+ metadata_url_base: https://raw.githubusercontent.com/MIT-LCP/mimic-code/${dataset_metadata_version}/mimic-iv/concepts/concept_map
7
+
8
+ urls:
9
+ dataset:
10
+ - url: https://physionet.org/files/nwicu-northwestern-icu/${raw_dataset_version}/data/
11
+ username: ${oc.env:DATASET_DOWNLOAD_USERNAME}
12
+ password: ${oc.env:DATASET_DOWNLOAD_PASSWORD}
13
+ # demo:
14
+ # - https://physionet.org/files/mimic-iv-demo/2.2/
15
+ common:
16
+ - ${metadata_url_base}/d_labitems_to_loinc.csv
17
+ - ${metadata_url_base}/inputevents_to_rxnorm.csv
18
+ - ${metadata_url_base}/lab_itemid_to_loinc.csv
19
+ - ${metadata_url_base}/meas_chartevents_main.csv
20
+ - ${metadata_url_base}/meas_chartevents_value.csv
21
+ - ${metadata_url_base}/numerics-summary.csv
22
+ - ${metadata_url_base}/outputevents_to_loinc.csv
23
+ - ${metadata_url_base}/proc_datetimeevents.csv
24
+ - ${metadata_url_base}/proc_itemid.csv
25
+ - ${metadata_url_base}/waveforms-summary.csv