meds-algorithm-retain 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meds_algorithm_retain-0.1.0.dist-info/METADATA +218 -0
- meds_algorithm_retain-0.1.0.dist-info/RECORD +16 -0
- meds_algorithm_retain-0.1.0.dist-info/WHEEL +4 -0
- meds_algorithm_retain-0.1.0.dist-info/entry_points.txt +2 -0
- meds_algorithm_retain-0.1.0.dist-info/licenses/LICENSE +21 -0
- retain_meds/__init__.py +6 -0
- retain_meds/__main__.py +65 -0
- retain_meds/_interpret.py +102 -0
- retain_meds/_model.py +171 -0
- retain_meds/_preprocess.py +270 -0
- retain_meds/_training.py +320 -0
- retain_meds/_windows.py +221 -0
- retain_meds/config.py +114 -0
- retain_meds/resources/algorithms/supervised/model.yaml +24 -0
- retain_meds/resources/runs/run.yaml +25 -0
- retain_meds/retain_meds.py +252 -0
retain_meds/_windows.py
ADDED
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Turn `(subject_id, prediction_time)` keys into RETAIN visit sequences.
|
|
2
|
+
|
|
3
|
+
Three invariants:
|
|
4
|
+
|
|
5
|
+
1. **No label leakage.** Nothing here sees a value column; `build_sequences` rejects any frame carrying
|
|
6
|
+
one, and the history is cut at the prediction time.
|
|
7
|
+
2. **Keys, never row order.** Every sequence carries its own `(subject_id, prediction_time)` key and is
|
|
8
|
+
associated with a score through that key alone.
|
|
9
|
+
3. **Every requested key produces a sequence.** A key whose subject has no history yields an empty
|
|
10
|
+
sequence rather than being dropped, so the published index can cover the requested index exactly.
|
|
11
|
+
|
|
12
|
+
Sequences are held in a CSR-style ragged layout — `row_ptr` into visits, `visit_ptr` into codes — because
|
|
13
|
+
the padded dense form is `n_keys x max_visits x |C|` and is unusable at MIMIC-IV scale.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import json
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
|
|
22
|
+
import numpy as np
|
|
23
|
+
import polars as pl
|
|
24
|
+
|
|
25
|
+
from .config import RETAINConfig
|
|
26
|
+
from ._preprocess import PAYLOAD_FORMAT, SETTINGS_FILE
|
|
27
|
+
|
|
28
|
+
KEY_COLUMNS = ("subject_id", "prediction_time")
|
|
29
|
+
|
|
30
|
+
MICROSECONDS_PER_MINUTE = 60_000_000
|
|
31
|
+
MICROSECONDS_PER_DAY = 86_400_000_000
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(frozen=True)
|
|
35
|
+
class SequenceBatch:
|
|
36
|
+
"""Visit sequences for a set of keys, in ragged CSR layout."""
|
|
37
|
+
|
|
38
|
+
#: `(n_keys, 2)` of `(subject_id, prediction_time in microseconds)`.
|
|
39
|
+
keys: np.ndarray
|
|
40
|
+
#: `(n_keys + 1,)` index into the visit arrays.
|
|
41
|
+
row_ptr: np.ndarray
|
|
42
|
+
#: `(n_visits + 1,)` index into `codes`.
|
|
43
|
+
visit_ptr: np.ndarray
|
|
44
|
+
#: `(n_visits,)` bucket start time in microseconds.
|
|
45
|
+
visit_time: np.ndarray
|
|
46
|
+
#: `(n_codes,)` vocabulary indices.
|
|
47
|
+
codes: np.ndarray
|
|
48
|
+
|
|
49
|
+
@property
|
|
50
|
+
def n_keys(self) -> int:
|
|
51
|
+
return int(self.keys.shape[0])
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def lengths(self) -> np.ndarray:
|
|
55
|
+
return (self.row_ptr[1:] - self.row_ptr[:-1]).astype(np.int64)
|
|
56
|
+
|
|
57
|
+
@property
|
|
58
|
+
def n_empty(self) -> int:
|
|
59
|
+
return int((self.lengths == 0).sum())
|
|
60
|
+
|
|
61
|
+
def visit_gap_days(self) -> np.ndarray:
|
|
62
|
+
"""Days between consecutive visits, zero at each first visit.
|
|
63
|
+
|
|
64
|
+
This is the `--time_file` quantity of the official repository, whose README defines it as "the
|
|
65
|
+
duration between consecutive visits" with "the duration for the first visit set to zero".
|
|
66
|
+
"""
|
|
67
|
+
gaps = np.zeros(self.visit_time.size, dtype=np.float32)
|
|
68
|
+
if self.visit_time.size:
|
|
69
|
+
deltas = np.diff(self.visit_time) / MICROSECONDS_PER_DAY
|
|
70
|
+
gaps[1:] = deltas
|
|
71
|
+
gaps[self.row_ptr[:-1][self.lengths > 0]] = 0.0
|
|
72
|
+
return gaps
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class PatientStore:
|
|
76
|
+
"""Read access to a published `retain-patients/v1` artifact."""
|
|
77
|
+
|
|
78
|
+
def __init__(self, data_dir: Path | str) -> None:
|
|
79
|
+
self.root = Path(data_dir)
|
|
80
|
+
self.vocabulary = pl.read_parquet(self.root / "vocabulary.parquet")
|
|
81
|
+
self.splits = pl.read_parquet(self.root / "subject_splits.parquet")
|
|
82
|
+
self.settings = json.loads((self.root / SETTINGS_FILE).read_text())
|
|
83
|
+
self.shards = sorted(
|
|
84
|
+
path for path in (self.root / "sequences").rglob("*.parquet") if not path.name.startswith(".")
|
|
85
|
+
)
|
|
86
|
+
if not self.shards:
|
|
87
|
+
raise FileNotFoundError(f"{self.root} declares {PAYLOAD_FORMAT} but holds no sequence shards")
|
|
88
|
+
|
|
89
|
+
@property
|
|
90
|
+
def vocabulary_size(self) -> int:
|
|
91
|
+
return self.vocabulary.height
|
|
92
|
+
|
|
93
|
+
def split_of(self, subject_ids: pl.Series) -> pl.Series:
|
|
94
|
+
"""Split name per subject, `None` for a subject the dataset does not assign."""
|
|
95
|
+
frame = pl.DataFrame({"subject_id": subject_ids.cast(pl.Int64)})
|
|
96
|
+
joined = frame.join(self.splits, on="subject_id", how="left")
|
|
97
|
+
if joined.height != frame.height:
|
|
98
|
+
# `_training_rows` indexes label rows by position against this result, so an expanded
|
|
99
|
+
# join would not error anywhere -- it would shift every split by one and quietly move
|
|
100
|
+
# held-out rows into the fit. `read_split_map` deduplicates; this refuses to proceed if
|
|
101
|
+
# a store was published before it did, rather than trusting that guarantee silently.
|
|
102
|
+
raise ValueError(
|
|
103
|
+
f"the split map assigns some subject more than once: the join expanded {frame.height} "
|
|
104
|
+
f"rows to {joined.height}. Callers align splits to label rows positionally, so this "
|
|
105
|
+
"would misassign splits rather than fail."
|
|
106
|
+
)
|
|
107
|
+
return joined.get_column("split")
|
|
108
|
+
|
|
109
|
+
def code_strings(self) -> list[str]:
|
|
110
|
+
ordered = self.vocabulary.sort("code_index")
|
|
111
|
+
return ordered.get_column("code").to_list()
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def build_sequences(store: PatientStore, keys: pl.DataFrame, config: RETAINConfig) -> SequenceBatch:
|
|
115
|
+
"""Build one visit sequence per row of `keys`, which must hold the index columns only."""
|
|
116
|
+
extra = set(keys.columns) - set(KEY_COLUMNS)
|
|
117
|
+
if extra:
|
|
118
|
+
raise ValueError(f"build_sequences must never see label columns; got extra column(s) {sorted(extra)}")
|
|
119
|
+
|
|
120
|
+
keys = keys.with_columns(
|
|
121
|
+
pl.col("subject_id").cast(pl.Int64), pl.col("prediction_time").cast(pl.Datetime("us"))
|
|
122
|
+
)
|
|
123
|
+
n_rows = keys.height
|
|
124
|
+
subject_of_row = keys.get_column("subject_id").to_numpy()
|
|
125
|
+
prediction_us = keys.get_column("prediction_time").to_physical().to_numpy().astype(np.int64)
|
|
126
|
+
|
|
127
|
+
rows_by_subject: dict[int, list[int]] = {}
|
|
128
|
+
for row, subject in enumerate(subject_of_row.tolist()):
|
|
129
|
+
rows_by_subject.setdefault(int(subject), []).append(row)
|
|
130
|
+
|
|
131
|
+
bucket_us = int(config.visit_bucket_minutes) * MICROSECONDS_PER_MINUTE
|
|
132
|
+
lookback_us = (
|
|
133
|
+
None if config.lookback_days is None else int(round(config.lookback_days * MICROSECONDS_PER_DAY))
|
|
134
|
+
)
|
|
135
|
+
vocabulary_size = max(store.vocabulary_size, 1)
|
|
136
|
+
side = "right" if config.include_events_at_prediction_time else "left"
|
|
137
|
+
|
|
138
|
+
per_row_visits: list[np.ndarray | None] = [None] * n_rows
|
|
139
|
+
per_row_codes: list[np.ndarray | None] = [None] * n_rows
|
|
140
|
+
seen_subjects: set[int] = set()
|
|
141
|
+
|
|
142
|
+
for shard in store.shards:
|
|
143
|
+
frame = pl.read_parquet(shard, columns=["subject_id", "time", "code_index"])
|
|
144
|
+
if frame.is_empty():
|
|
145
|
+
continue
|
|
146
|
+
shard_subjects = frame.get_column("subject_id").to_numpy()
|
|
147
|
+
shard_times = frame.get_column("time").to_physical().to_numpy().astype(np.int64)
|
|
148
|
+
shard_codes = frame.get_column("code_index").to_numpy().astype(np.int64)
|
|
149
|
+
# The shard is written sorted by (subject_id, time, code_index), so each subject occupies one
|
|
150
|
+
# contiguous range and its events are already in chronological order.
|
|
151
|
+
boundaries = np.flatnonzero(np.diff(shard_subjects)) + 1
|
|
152
|
+
starts = np.concatenate(([0], boundaries))
|
|
153
|
+
stops = np.concatenate((boundaries, [shard_subjects.size]))
|
|
154
|
+
for start, stop in zip(starts.tolist(), stops.tolist()):
|
|
155
|
+
subject = int(shard_subjects[start])
|
|
156
|
+
rows = rows_by_subject.get(subject)
|
|
157
|
+
if not rows:
|
|
158
|
+
continue
|
|
159
|
+
if subject in seen_subjects:
|
|
160
|
+
# MEDS keeps a subject's events in one shard. If that ever stops holding, a later shard
|
|
161
|
+
# would silently replace an earlier partial history instead of extending it.
|
|
162
|
+
raise ValueError(
|
|
163
|
+
f"Subject {subject} appears in more than one shard of {store.root}; RETAIN builds "
|
|
164
|
+
"each history from a single contiguous shard range"
|
|
165
|
+
)
|
|
166
|
+
seen_subjects.add(subject)
|
|
167
|
+
times = shard_times[start:stop]
|
|
168
|
+
codes = shard_codes[start:stop]
|
|
169
|
+
for row in rows:
|
|
170
|
+
prediction = int(prediction_us[row])
|
|
171
|
+
high = int(np.searchsorted(times, prediction, side=side))
|
|
172
|
+
low = 0
|
|
173
|
+
if lookback_us is not None:
|
|
174
|
+
low = int(np.searchsorted(times, prediction - lookback_us, side="left"))
|
|
175
|
+
if high <= low:
|
|
176
|
+
continue
|
|
177
|
+
window_times = times[low:high]
|
|
178
|
+
buckets = window_times if bucket_us == 0 else (window_times // bucket_us) * bucket_us
|
|
179
|
+
visit_time, inverse = np.unique(buckets, return_inverse=True)
|
|
180
|
+
if config.max_visits is not None and visit_time.size > config.max_visits:
|
|
181
|
+
# A reverse-time model attends to the most recent visits, so truncation drops the
|
|
182
|
+
# oldest ones, mirroring the paper's bounded observation window.
|
|
183
|
+
first_kept = visit_time.size - config.max_visits
|
|
184
|
+
keep = inverse >= first_kept
|
|
185
|
+
inverse = inverse[keep] - first_kept
|
|
186
|
+
visit_time = visit_time[first_kept:]
|
|
187
|
+
window_codes = codes[low:high][keep]
|
|
188
|
+
else:
|
|
189
|
+
window_codes = codes[low:high]
|
|
190
|
+
# `x` is binary, so a code seen twice inside one bucket must contribute once. Packing
|
|
191
|
+
# (visit, code) into a single key deduplicates and orders both axes in one pass.
|
|
192
|
+
packed = np.unique(inverse.astype(np.int64) * vocabulary_size + window_codes)
|
|
193
|
+
per_row_visits[row] = visit_time
|
|
194
|
+
per_row_codes[row] = packed
|
|
195
|
+
|
|
196
|
+
row_ptr = np.zeros(n_rows + 1, dtype=np.int64)
|
|
197
|
+
widths: list[np.ndarray] = []
|
|
198
|
+
times_out: list[np.ndarray] = []
|
|
199
|
+
codes_out: list[np.ndarray] = []
|
|
200
|
+
for row in range(n_rows):
|
|
201
|
+
visit_time = per_row_visits[row]
|
|
202
|
+
packed = per_row_codes[row]
|
|
203
|
+
if visit_time is None or packed is None or visit_time.size == 0:
|
|
204
|
+
row_ptr[row + 1] = row_ptr[row]
|
|
205
|
+
continue
|
|
206
|
+
row_ptr[row + 1] = row_ptr[row] + visit_time.size
|
|
207
|
+
visit_index = packed // vocabulary_size
|
|
208
|
+
widths.append(np.bincount(visit_index, minlength=visit_time.size).astype(np.int64))
|
|
209
|
+
times_out.append(visit_time.astype(np.int64))
|
|
210
|
+
codes_out.append((packed % vocabulary_size).astype(np.int32))
|
|
211
|
+
|
|
212
|
+
width = np.concatenate(widths) if widths else np.zeros(0, dtype=np.int64)
|
|
213
|
+
visit_ptr = np.zeros(width.size + 1, dtype=np.int64)
|
|
214
|
+
np.cumsum(width, out=visit_ptr[1:])
|
|
215
|
+
return SequenceBatch(
|
|
216
|
+
keys=np.column_stack((subject_of_row.astype(np.int64), prediction_us)),
|
|
217
|
+
row_ptr=row_ptr,
|
|
218
|
+
visit_ptr=visit_ptr,
|
|
219
|
+
visit_time=np.concatenate(times_out) if times_out else np.zeros(0, dtype=np.int64),
|
|
220
|
+
codes=np.concatenate(codes_out) if codes_out else np.zeros(0, dtype=np.int32),
|
|
221
|
+
)
|
retain_meds/config.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""RETAIN's configuration surface.
|
|
2
|
+
|
|
3
|
+
Every architecture and optimisation field mirrors an option of the official `retain.py` command line
|
|
4
|
+
(`mp2893/retain` @ 9fd39c46), with the same default, so a run can be described by naming the upstream flags
|
|
5
|
+
it changed. The first block has no upstream counterpart: it is the MEDS adaptation, because the official
|
|
6
|
+
code starts from an already-tokenised list-of-list-of-list of visits and never has to decide what a visit
|
|
7
|
+
is. Those fields are marked `MEDS` below and are recorded as *adapted* in the implementation report.
|
|
8
|
+
|
|
9
|
+
Core `NNConfig` is not reused: RETAIN has two dropout rates (`keep_prob_emb`, `keep_prob_context`), four
|
|
10
|
+
explicit L2 penalties instead of one weight decay, and a default optimiser (Adadelta) without a learning
|
|
11
|
+
rate to set, so `capacity`, `learning_rate`, `weight_decay` and `dropout` would each be a field with no
|
|
12
|
+
effect.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
|
|
17
|
+
from meds_algorithm import AlgorithmConfig
|
|
18
|
+
|
|
19
|
+
SOLVERS = ("adadelta", "adam")
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class RETAINConfig(AlgorithmConfig):
|
|
24
|
+
# -- MEDS adaptation: how a MEDS code stream becomes RETAIN's visit sequences -------------------
|
|
25
|
+
#: Width of the absolute time grid that groups events into one visit. 1440 = calendar day, the
|
|
26
|
+
#: closest generic analogue of the admission-level visits of `process_mimic.py`. 0 keeps every
|
|
27
|
+
#: distinct timestamp as its own visit.
|
|
28
|
+
visit_bucket_minutes: int = 1440
|
|
29
|
+
#: Whether an event stamped exactly at the prediction time belongs to the history. MEDS-DEV tasks
|
|
30
|
+
#: declare their input window with `index_timestamp: end` and `end_inclusive: True`, so the
|
|
31
|
+
#: prediction time *is* the last instant of the granted input window; excluding it would discard
|
|
32
|
+
#: data the task definition hands to the model. Set false for a task whose label-defining event can
|
|
33
|
+
#: coincide with its own prediction time.
|
|
34
|
+
include_events_at_prediction_time: bool = True
|
|
35
|
+
#: Discard events older than this many days before the prediction time. `None` uses all history,
|
|
36
|
+
#: which is what `start: null` in the demo task's input window asks for; the paper used an 18-month
|
|
37
|
+
#: window for its own heart-failure cohort.
|
|
38
|
+
lookback_days: float | None = None
|
|
39
|
+
#: Keep at most this many of the most recent visits. `None` keeps all of them.
|
|
40
|
+
max_visits: int | None = None
|
|
41
|
+
#: Vocabulary filters. The vocabulary is learned from the training split only.
|
|
42
|
+
min_code_count: int = 1
|
|
43
|
+
max_vocab_size: int | None = None
|
|
44
|
+
#: Drop the MEDS standard's two non-clinical subject markers, `MEDS_BIRTH` and `MEDS_DEATH`, from the
|
|
45
|
+
#: code stream. Both the paper (section 4.1: visits are "diagnosis, medication and procedure codes")
|
|
46
|
+
#: and `process_mimic.py` (sequences built from `DIAGNOSES_ICD` alone, `PATIENTS.csv` read only to
|
|
47
|
+
#: derive the label) exclude demographics and outcome markers from the visit sequence. Keeping
|
|
48
|
+
#: `MEDS_BIRTH` makes a birth record the first visit of essentially every sequence, decades before the
|
|
49
|
+
#: rest, where it consumes one of RETAIN's visit-attention slots; keeping `MEDS_DEATH` puts the
|
|
50
|
+
#: canonical outcome marker in the input space of any task whose window can reach it. These are
|
|
51
|
+
#: MEDS-standard codes, not dataset predicates, so excluding them keeps this port predicate-free.
|
|
52
|
+
exclude_meds_standard_markers: bool = True
|
|
53
|
+
|
|
54
|
+
# -- Official `retain.py` options ---------------------------------------------------------------
|
|
55
|
+
#: `--time_file`: upstream defaults to no time information, so this defaults to false.
|
|
56
|
+
use_time: bool = False
|
|
57
|
+
use_log_time: bool = True # --use_log_time
|
|
58
|
+
embed_size: int = 128 # --embed_size
|
|
59
|
+
alpha_hidden_dim_size: int = 128 # --alpha_hidden_dim_size
|
|
60
|
+
beta_hidden_dim_size: int = 128 # --beta_hidden_dim_size
|
|
61
|
+
batch_size: int = 100 # --batch_size
|
|
62
|
+
n_epochs: int = 10 # --n_epochs
|
|
63
|
+
L2_output: float = 0.001 # --L2_output
|
|
64
|
+
L2_emb: float = 0.001 # --L2_emb
|
|
65
|
+
L2_alpha: float = 0.001 # --L2_alpha
|
|
66
|
+
L2_beta: float = 0.001 # --L2_beta
|
|
67
|
+
keep_prob_emb: float = 0.5 # --keep_prob_emb
|
|
68
|
+
keep_prob_context: float = 0.5 # --keep_prob_context
|
|
69
|
+
log_eps: float = 1e-8 # --log_eps
|
|
70
|
+
solver: str = "adadelta" # --solver
|
|
71
|
+
|
|
72
|
+
# -- Runtime ------------------------------------------------------------------------------------
|
|
73
|
+
# The seed is not configuration: it is the `seed` argument of every step (request.execution.seed).
|
|
74
|
+
device: str = "auto"
|
|
75
|
+
#: Only used when the task supplies no `tuning` labels, standing in for the fixed hold-out
|
|
76
|
+
#: fractions of the official `load_data_simple`.
|
|
77
|
+
validation_fraction: float = 0.1
|
|
78
|
+
max_train_rows: int | None = None
|
|
79
|
+
#: Per-key code contributions of Eq. (5). 0 disables the extra pass entirely.
|
|
80
|
+
interpretation_top_k: int = 0
|
|
81
|
+
|
|
82
|
+
def validate(self):
|
|
83
|
+
super().validate()
|
|
84
|
+
if self.solver not in SOLVERS:
|
|
85
|
+
raise ValueError(f"solver must be one of {SOLVERS}, got {self.solver!r}")
|
|
86
|
+
if self.visit_bucket_minutes < 0:
|
|
87
|
+
raise ValueError("visit_bucket_minutes must be >= 0")
|
|
88
|
+
if self.lookback_days is not None and self.lookback_days <= 0:
|
|
89
|
+
raise ValueError("lookback_days must be positive or null")
|
|
90
|
+
if not 0.0 < self.keep_prob_emb <= 1.0 or not 0.0 < self.keep_prob_context <= 1.0:
|
|
91
|
+
raise ValueError("keep_prob_emb and keep_prob_context must lie in (0, 1]")
|
|
92
|
+
for name in ("embed_size", "alpha_hidden_dim_size", "beta_hidden_dim_size", "batch_size", "n_epochs"):
|
|
93
|
+
if getattr(self, name) < 1:
|
|
94
|
+
raise ValueError(f"{name} must be >= 1")
|
|
95
|
+
for name in ("L2_output", "L2_emb", "L2_alpha", "L2_beta"):
|
|
96
|
+
if getattr(self, name) < 0:
|
|
97
|
+
raise ValueError(f"{name} must be >= 0")
|
|
98
|
+
if self.min_code_count < 1:
|
|
99
|
+
raise ValueError("min_code_count must be >= 1")
|
|
100
|
+
for name in ("max_visits", "max_vocab_size", "max_train_rows"):
|
|
101
|
+
value = getattr(self, name)
|
|
102
|
+
if value is not None and value < 1:
|
|
103
|
+
raise ValueError(f"{name} must be >= 1 or null")
|
|
104
|
+
if not 0.0 <= self.validation_fraction < 1.0:
|
|
105
|
+
raise ValueError("validation_fraction must lie in [0, 1)")
|
|
106
|
+
if self.interpretation_top_k < 0:
|
|
107
|
+
raise ValueError("interpretation_top_k must be >= 0")
|
|
108
|
+
if self.log_eps <= 0:
|
|
109
|
+
raise ValueError("log_eps must be positive")
|
|
110
|
+
|
|
111
|
+
@property
|
|
112
|
+
def gru_input_size(self) -> int:
|
|
113
|
+
"""RETAIN concatenates the visit duration onto the GRU inputs only (`build_model`)."""
|
|
114
|
+
return self.embed_size + (1 if self.use_time else 0)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
## MEDS-DEV descriptor for RETAIN. This retains the established command-slot interface.
|
|
7
|
+
## Each MEDS-DEV run directory {output_dir} holds this run's artifacts as sibling subdirectories (data/,
|
|
8
|
+
## pretrained/, algorithm/, embeddings/ or trajectories/, predictions/): MEDS-DEV keeps its own cmd.sh and .done marker in
|
|
9
|
+
## {output_dir}, and artifacts are published by publishing their own destination directory.
|
|
10
|
+
metadata:
|
|
11
|
+
description: "RETAIN (Choi et al., NeurIPS 2016) reverse-time attention model for MEDS, a supervised classifier behind the MEDS algorithm contract."
|
|
12
|
+
contacts:
|
|
13
|
+
- name: "Florent Pollet"
|
|
14
|
+
email: "flo1.raymond@gmail.com"
|
|
15
|
+
|
|
16
|
+
commands:
|
|
17
|
+
supervised:
|
|
18
|
+
train: |-
|
|
19
|
+
retain-meds preprocess_data --profile supervised external_meds_dir={dataset_dir} output_data_dir={output_dir}/data do_overwrite=true
|
|
20
|
+
retain-meds train_supervised --profile supervised input_data_dir={output_dir}/data external_labels_dir={labels_dir} output_supervised_algorithm_dir={output_dir}/algorithm
|
|
21
|
+
|
|
22
|
+
predict: |-
|
|
23
|
+
retain-meds predict_supervised --profile supervised input_data_dir="$(meds-algorithm-resolve-input {model_initialization_dir}/algorithm input_data_dir)" input_supervised_algorithm_dir={model_initialization_dir}/algorithm external_index_dir={labels_dir} splits=[{split}] output_predictions_dir={output_dir}/predictions do_overwrite=true external_evaluation_labels_dir={labels_dir}
|
|
24
|
+
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# Prototype: one config for local subprocesses or dependent Submitit SLURM jobs.
|
|
2
|
+
# Shared field/resource defaults come from meds-algorithm, not a copied schema.
|
|
3
|
+
defaults:
|
|
4
|
+
- meds_run
|
|
5
|
+
- _self_
|
|
6
|
+
|
|
7
|
+
profile: supervised
|
|
8
|
+
execution: local
|
|
9
|
+
output_dir: ??? # New directory for this run; relative paths use the invocation's working directory.
|
|
10
|
+
|
|
11
|
+
inputs:
|
|
12
|
+
meds_dir: null # Required by preprocessing and materialized zero-shot prediction.
|
|
13
|
+
index_dir: null # Required by inference and prediction.
|
|
14
|
+
labels_dir: null # Required by supervised/finetune/probe training.
|
|
15
|
+
|
|
16
|
+
# Add only algorithm settings that differ from the concrete Config class's defaults.
|
|
17
|
+
algorithm: {}
|
|
18
|
+
|
|
19
|
+
# Optional overrides, e.g. resources.compute.gpus_per_node=1 and its partition/account.
|
|
20
|
+
# Both resource presets default to CPUs; choose resources appropriate for this algorithm.
|
|
21
|
+
# resources:
|
|
22
|
+
# compute:
|
|
23
|
+
# gpus_per_node: 1
|
|
24
|
+
# partition: gpu
|
|
25
|
+
# timeout_min: 240
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
"""RETAIN behind the MEDS algorithm contract: `preprocess_data -> train_supervised -> predict_supervised`.
|
|
2
|
+
|
|
3
|
+
That is RETAIN's actual lifecycle: the embedding, both attention GRUs and the risk head are trained jointly
|
|
4
|
+
against the labelled endpoint by `train_RETAIN`, and there is no task-independent pretraining or inference
|
|
5
|
+
stage to place anywhere else.
|
|
6
|
+
|
|
7
|
+
- `preprocess_data` tokenises the MEDS code stream. It reads no labels and no predicates, and fits the
|
|
8
|
+
vocabulary on the dataset's training split only.
|
|
9
|
+
- `train_supervised` fits RETAIN on the `train` labels and selects the epoch on the `tuning` labels (or on a
|
|
10
|
+
seeded hold-out of the training rows when the task supplies none).
|
|
11
|
+
- `predict_supervised` scores every requested key. It receives the index only; the runner attaches ground
|
|
12
|
+
truth after the scores are written.
|
|
13
|
+
|
|
14
|
+
Histories are cut at each prediction time before visits are formed (`_windows.build_sequences`), so nothing
|
|
15
|
+
recorded after a prediction time can reach its prediction.
|
|
16
|
+
|
|
17
|
+
This port is predicate-free: RETAIN's input is a sequence of medical codes, which the MEDS `code` column
|
|
18
|
+
already is. `external_predicates_file` is accepted, recorded by the runner and not read.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import hashlib
|
|
22
|
+
import json
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
import numpy as np
|
|
26
|
+
import polars as pl
|
|
27
|
+
from meds_algorithm import AlgorithmRepository, PayloadMetadata, SupervisedAlgorithm
|
|
28
|
+
from meds_algorithm.requests import PredictSupervisedRequest, PreprocessDataRequest, TrainSupervisedRequest
|
|
29
|
+
|
|
30
|
+
from . import _preprocess
|
|
31
|
+
from ._preprocess import HELD_OUT_SPLIT, SETTINGS_FILE, TRAIN_SPLIT, TUNING_SPLIT, VOCABULARY_SETTINGS
|
|
32
|
+
from ._windows import KEY_COLUMNS, PatientStore, build_sequences
|
|
33
|
+
from .config import RETAINConfig
|
|
34
|
+
|
|
35
|
+
_KEYS = list(KEY_COLUMNS)
|
|
36
|
+
_PREPARED = PayloadMetadata(_preprocess.PAYLOAD_FORMAT, 1)
|
|
37
|
+
_FITTED = PayloadMetadata("retain-supervised-model", 1)
|
|
38
|
+
_PREDICTIONS = PayloadMetadata("retain-predictions", 1)
|
|
39
|
+
_EXPECTED_INPUTS = {"input_data_dir": _PREPARED, "input_supervised_algorithm_dir": _FITTED}
|
|
40
|
+
|
|
41
|
+
_MODEL_FILE = "model.pt"
|
|
42
|
+
_TRAINING_FILE = "training.json"
|
|
43
|
+
_PREPROCESS_REPORT = "preprocess_report.json"
|
|
44
|
+
_PREDICTION_REPORT = "prediction_report.json"
|
|
45
|
+
_INTERPRETATION_FILE = "interpretation.ndjson"
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _format(metadata: PayloadMetadata) -> str:
|
|
49
|
+
return f"{metadata.format}/v{metadata.version}"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _write_json(path: Path, payload: dict) -> None:
|
|
53
|
+
path.write_text(json.dumps(payload, indent=2, sort_keys=True, default=str))
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _vocabulary_digest(data_dir: Path) -> str:
|
|
57
|
+
"""Identity of the vocabulary a model was fitted on: its codes in index order."""
|
|
58
|
+
vocabulary = pl.read_parquet(Path(data_dir) / "vocabulary.parquet").sort("code_index")
|
|
59
|
+
return "sha256:" + hashlib.sha256("\n".join(vocabulary["code"].to_list()).encode()).hexdigest()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class RETAIN(SupervisedAlgorithm):
|
|
63
|
+
name = "retain_meds"
|
|
64
|
+
config_type = RETAINConfig
|
|
65
|
+
|
|
66
|
+
def validate_inputs(self, request) -> None:
|
|
67
|
+
"""Reject an incompatible input artifact before any expensive work happens."""
|
|
68
|
+
super().validate_inputs(request)
|
|
69
|
+
for argument, artifact in request.artifact_inputs():
|
|
70
|
+
found = artifact.read_manifest()["artifact"]["payload_format"]
|
|
71
|
+
if found != _format(_EXPECTED_INPUTS[argument]):
|
|
72
|
+
raise ValueError(f"{argument} has payload {found}; expected {_format(_EXPECTED_INPUTS[argument])}")
|
|
73
|
+
data = getattr(request, "data", None)
|
|
74
|
+
if data is not None:
|
|
75
|
+
# The vocabulary is fixed by preprocess_data; changing its filters later would describe a
|
|
76
|
+
# vocabulary that was never built.
|
|
77
|
+
published = json.loads((data.directory / SETTINGS_FILE).read_text())
|
|
78
|
+
for name in VOCABULARY_SETTINGS:
|
|
79
|
+
if published.get(name) != getattr(self.config, name):
|
|
80
|
+
raise ValueError(
|
|
81
|
+
f"{name}={getattr(self.config, name)!r} disagrees with input_data_dir, which was "
|
|
82
|
+
f"built with {name}={published.get(name)!r}. It is a preprocess_data setting; "
|
|
83
|
+
"rebuild the prepared data to change it."
|
|
84
|
+
)
|
|
85
|
+
fitted = getattr(request, "fitted_algorithm", None)
|
|
86
|
+
if fitted is not None and data is not None:
|
|
87
|
+
recorded = json.loads((fitted.directory / _TRAINING_FILE).read_text())["vocabulary_digest"]
|
|
88
|
+
if recorded != _vocabulary_digest(data.directory):
|
|
89
|
+
raise ValueError(
|
|
90
|
+
"input_data_dir holds a different vocabulary from the one the model was fitted on; "
|
|
91
|
+
"pass the prepared data used by train_supervised"
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
def preprocess_data(self, request: PreprocessDataRequest) -> PayloadMetadata:
|
|
95
|
+
"""Tokenise a MEDS dataset into RETAIN's code stream; task- and label-independent."""
|
|
96
|
+
report = _preprocess.preprocess(request.meds_dir, request.output.directory, self.config)
|
|
97
|
+
_write_json(request.output.directory / _PREPROCESS_REPORT, report)
|
|
98
|
+
return _PREPARED
|
|
99
|
+
|
|
100
|
+
def train_supervised(self, request: TrainSupervisedRequest) -> PayloadMetadata:
|
|
101
|
+
"""Fit RETAIN on the train labels; the tuning labels select the epoch."""
|
|
102
|
+
import torch
|
|
103
|
+
|
|
104
|
+
from ._training import resolve_device, train
|
|
105
|
+
|
|
106
|
+
config = self.config
|
|
107
|
+
seed = request.execution.seed
|
|
108
|
+
store = PatientStore(request.data.directory)
|
|
109
|
+
|
|
110
|
+
# Row order of a label file carries no meaning; a canonical key order keeps the fit a function of
|
|
111
|
+
# the labels' content (it decides how equal-length ties resolve when minibatches are built).
|
|
112
|
+
fit = request.labels.read(TRAIN_SPLIT).sort(_KEYS)
|
|
113
|
+
_require_split(store, fit, TRAIN_SPLIT)
|
|
114
|
+
try:
|
|
115
|
+
validation = request.labels.read(TUNING_SPLIT).sort(_KEYS)
|
|
116
|
+
except FileNotFoundError:
|
|
117
|
+
validation = None
|
|
118
|
+
validation_source = TUNING_SPLIT
|
|
119
|
+
if validation is not None:
|
|
120
|
+
_require_split(store, validation, TUNING_SPLIT)
|
|
121
|
+
elif fit.height > 1 and config.validation_fraction > 0:
|
|
122
|
+
# No tuning labels: a seeded hold-out of the training rows, as `load_data_simple` does upstream.
|
|
123
|
+
shuffled = np.random.default_rng(seed).permutation(fit.height)
|
|
124
|
+
n_validation = min(max(1, round(config.validation_fraction * fit.height)), fit.height - 1)
|
|
125
|
+
validation = fit[np.sort(shuffled[:n_validation])]
|
|
126
|
+
fit = fit[np.sort(shuffled[n_validation:])]
|
|
127
|
+
validation_source = f"seeded {config.validation_fraction:g} hold-out of {TRAIN_SPLIT}"
|
|
128
|
+
else:
|
|
129
|
+
validation = fit.clear()
|
|
130
|
+
validation_source = "none"
|
|
131
|
+
if config.max_train_rows is not None:
|
|
132
|
+
fit = fit.head(config.max_train_rows)
|
|
133
|
+
|
|
134
|
+
# Sequences are built from the keys alone; the targets never reach sequence construction.
|
|
135
|
+
train_batch = build_sequences(store, fit.select(_KEYS), config)
|
|
136
|
+
validation_batch = build_sequences(store, validation.select(_KEYS), config)
|
|
137
|
+
device = resolve_device(config.device)
|
|
138
|
+
model, history = train(
|
|
139
|
+
vocabulary_size=store.vocabulary_size,
|
|
140
|
+
train_batch=train_batch,
|
|
141
|
+
train_labels=fit["boolean_value"].to_numpy().astype(np.float32),
|
|
142
|
+
validation_batch=validation_batch,
|
|
143
|
+
validation_labels=validation["boolean_value"].to_numpy().astype(np.float32),
|
|
144
|
+
config=config,
|
|
145
|
+
device=device,
|
|
146
|
+
seed=seed,
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
output = request.output.directory
|
|
150
|
+
torch.save(
|
|
151
|
+
{"state_dict": model.state_dict(), "vocabulary_size": store.vocabulary_size}, output / _MODEL_FILE
|
|
152
|
+
)
|
|
153
|
+
_write_json(
|
|
154
|
+
output / _TRAINING_FILE,
|
|
155
|
+
{
|
|
156
|
+
"vocabulary_size": store.vocabulary_size,
|
|
157
|
+
"vocabulary_digest": _vocabulary_digest(request.data.directory),
|
|
158
|
+
"n_parameters": model.n_parameters(),
|
|
159
|
+
"device": str(device),
|
|
160
|
+
"seed": seed,
|
|
161
|
+
"training": history.summary(),
|
|
162
|
+
"validation_source": validation_source,
|
|
163
|
+
"fit_rows": fit.height,
|
|
164
|
+
"validation_rows": validation.height,
|
|
165
|
+
"train_visits": int(train_batch.lengths.sum()),
|
|
166
|
+
"empty_history_fit_rows": train_batch.n_empty,
|
|
167
|
+
"empty_history_validation_rows": validation_batch.n_empty,
|
|
168
|
+
},
|
|
169
|
+
)
|
|
170
|
+
return _FITTED
|
|
171
|
+
|
|
172
|
+
def predict_supervised(self, request: PredictSupervisedRequest) -> PayloadMetadata:
|
|
173
|
+
"""Score every requested key; scores carry their own keys and are never matched by row order."""
|
|
174
|
+
import torch
|
|
175
|
+
|
|
176
|
+
from ._model import Retain
|
|
177
|
+
from ._training import resolve_device, score
|
|
178
|
+
|
|
179
|
+
config = self.config
|
|
180
|
+
index = request.index.read().sort(_KEYS)
|
|
181
|
+
store = PatientStore(request.data.directory)
|
|
182
|
+
fitted = request.fitted_algorithm.directory
|
|
183
|
+
device = resolve_device(config.device)
|
|
184
|
+
checkpoint = torch.load(fitted / _MODEL_FILE, map_location=device, weights_only=True)
|
|
185
|
+
model = Retain(checkpoint["vocabulary_size"], config).to(device)
|
|
186
|
+
model.load_state_dict(checkpoint["state_dict"])
|
|
187
|
+
|
|
188
|
+
# Every key yields a sequence: a key with no history is scored by the learned intercept.
|
|
189
|
+
batch = build_sequences(store, index.select(_KEYS), config)
|
|
190
|
+
probabilities = score(model, batch, config, device)
|
|
191
|
+
predictions = pl.DataFrame(
|
|
192
|
+
{
|
|
193
|
+
"subject_id": pl.Series(batch.keys[:, 0]).cast(index.schema["subject_id"]),
|
|
194
|
+
"prediction_time": pl.Series(batch.keys[:, 1]).cast(pl.Datetime("us")),
|
|
195
|
+
"predicted_boolean_probability": pl.Series(probabilities, dtype=pl.Float64),
|
|
196
|
+
}
|
|
197
|
+
).with_columns(
|
|
198
|
+
pl.col("prediction_time").cast(index.schema["prediction_time"]),
|
|
199
|
+
(pl.col("predicted_boolean_probability") > 0.5).alias("predicted_boolean_value"),
|
|
200
|
+
)
|
|
201
|
+
missing = index.join(predictions, on=_KEYS, how="anti").height
|
|
202
|
+
extra = predictions.join(index, on=_KEYS, how="anti").height
|
|
203
|
+
if missing or extra or predictions.height != index.height:
|
|
204
|
+
raise ValueError(f"RETAIN scoring missed {missing} and added {extra} requested keys")
|
|
205
|
+
|
|
206
|
+
output = request.output.directory
|
|
207
|
+
predictions.write_parquet(output / "predictions.parquet")
|
|
208
|
+
report = {
|
|
209
|
+
"scored_keys": predictions.height,
|
|
210
|
+
"unique_subjects": predictions["subject_id"].n_unique(),
|
|
211
|
+
"visits": int(batch.lengths.sum()),
|
|
212
|
+
"empty_history_keys": batch.n_empty,
|
|
213
|
+
"exclusions": "none: every requested key is scored",
|
|
214
|
+
"device": str(device),
|
|
215
|
+
}
|
|
216
|
+
if config.interpretation_top_k > 0:
|
|
217
|
+
from ._interpret import write_contributions
|
|
218
|
+
|
|
219
|
+
report["interpretation_records"] = write_contributions(
|
|
220
|
+
output / _INTERPRETATION_FILE,
|
|
221
|
+
model=model,
|
|
222
|
+
batch=batch,
|
|
223
|
+
config=config,
|
|
224
|
+
device=device,
|
|
225
|
+
code_strings=store.code_strings(),
|
|
226
|
+
top_k=config.interpretation_top_k,
|
|
227
|
+
)
|
|
228
|
+
_write_json(output / _PREDICTION_REPORT, report)
|
|
229
|
+
return _PREDICTIONS
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _require_split(store: PatientStore, labels: pl.DataFrame, split: str) -> None:
|
|
233
|
+
"""Training labels may only name subjects of their own MEDS split, so held-out subjects are never fitted."""
|
|
234
|
+
assigned = store.split_of(labels["subject_id"])
|
|
235
|
+
outside = labels.filter(assigned.fill_null("unassigned") != split)
|
|
236
|
+
if outside.height:
|
|
237
|
+
raise ValueError(
|
|
238
|
+
f"{outside['subject_id'].n_unique()} subject(s) of the {split} labels are not in the MEDS "
|
|
239
|
+
f"{split} split (e.g. {HELD_OUT_SPLIT} subjects must never be fitted)"
|
|
240
|
+
)
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
# Declarative implementation/completion metadata, never fitted algorithms or mutable runtime state.
|
|
244
|
+
IMPLEMENTATIONS = {
|
|
245
|
+
"supervised": RETAIN,
|
|
246
|
+
}
|
|
247
|
+
PROFILES = tuple(IMPLEMENTATIONS)
|
|
248
|
+
STUB_PROFILES: set[str] = set()
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def create_repository():
|
|
252
|
+
return AlgorithmRepository(IMPLEMENTATIONS, repo=Path(__file__).resolve().parents[2])
|