meds-algorithm-retain 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,221 @@
1
+ """Turn `(subject_id, prediction_time)` keys into RETAIN visit sequences.
2
+
3
+ Three invariants:
4
+
5
+ 1. **No label leakage.** Nothing here sees a value column; `build_sequences` rejects any frame carrying
6
+ one, and the history is cut at the prediction time.
7
+ 2. **Keys, never row order.** Every sequence carries its own `(subject_id, prediction_time)` key and is
8
+ associated with a score through that key alone.
9
+ 3. **Every requested key produces a sequence.** A key whose subject has no history yields an empty
10
+ sequence rather than being dropped, so the published index can cover the requested index exactly.
11
+
12
+ Sequences are held in a CSR-style ragged layout — `row_ptr` into visits, `visit_ptr` into codes — because
13
+ the padded dense form is `n_keys x max_visits x |C|` and is unusable at MIMIC-IV scale.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import json
19
+ from dataclasses import dataclass
20
+ from pathlib import Path
21
+
22
+ import numpy as np
23
+ import polars as pl
24
+
25
+ from .config import RETAINConfig
26
+ from ._preprocess import PAYLOAD_FORMAT, SETTINGS_FILE
27
+
28
+ KEY_COLUMNS = ("subject_id", "prediction_time")
29
+
30
+ MICROSECONDS_PER_MINUTE = 60_000_000
31
+ MICROSECONDS_PER_DAY = 86_400_000_000
32
+
33
+
34
+ @dataclass(frozen=True)
35
+ class SequenceBatch:
36
+ """Visit sequences for a set of keys, in ragged CSR layout."""
37
+
38
+ #: `(n_keys, 2)` of `(subject_id, prediction_time in microseconds)`.
39
+ keys: np.ndarray
40
+ #: `(n_keys + 1,)` index into the visit arrays.
41
+ row_ptr: np.ndarray
42
+ #: `(n_visits + 1,)` index into `codes`.
43
+ visit_ptr: np.ndarray
44
+ #: `(n_visits,)` bucket start time in microseconds.
45
+ visit_time: np.ndarray
46
+ #: `(n_codes,)` vocabulary indices.
47
+ codes: np.ndarray
48
+
49
+ @property
50
+ def n_keys(self) -> int:
51
+ return int(self.keys.shape[0])
52
+
53
+ @property
54
+ def lengths(self) -> np.ndarray:
55
+ return (self.row_ptr[1:] - self.row_ptr[:-1]).astype(np.int64)
56
+
57
+ @property
58
+ def n_empty(self) -> int:
59
+ return int((self.lengths == 0).sum())
60
+
61
+ def visit_gap_days(self) -> np.ndarray:
62
+ """Days between consecutive visits, zero at each first visit.
63
+
64
+ This is the `--time_file` quantity of the official repository, whose README defines it as "the
65
+ duration between consecutive visits" with "the duration for the first visit set to zero".
66
+ """
67
+ gaps = np.zeros(self.visit_time.size, dtype=np.float32)
68
+ if self.visit_time.size:
69
+ deltas = np.diff(self.visit_time) / MICROSECONDS_PER_DAY
70
+ gaps[1:] = deltas
71
+ gaps[self.row_ptr[:-1][self.lengths > 0]] = 0.0
72
+ return gaps
73
+
74
+
75
+ class PatientStore:
76
+ """Read access to a published `retain-patients/v1` artifact."""
77
+
78
+ def __init__(self, data_dir: Path | str) -> None:
79
+ self.root = Path(data_dir)
80
+ self.vocabulary = pl.read_parquet(self.root / "vocabulary.parquet")
81
+ self.splits = pl.read_parquet(self.root / "subject_splits.parquet")
82
+ self.settings = json.loads((self.root / SETTINGS_FILE).read_text())
83
+ self.shards = sorted(
84
+ path for path in (self.root / "sequences").rglob("*.parquet") if not path.name.startswith(".")
85
+ )
86
+ if not self.shards:
87
+ raise FileNotFoundError(f"{self.root} declares {PAYLOAD_FORMAT} but holds no sequence shards")
88
+
89
+ @property
90
+ def vocabulary_size(self) -> int:
91
+ return self.vocabulary.height
92
+
93
+ def split_of(self, subject_ids: pl.Series) -> pl.Series:
94
+ """Split name per subject, `None` for a subject the dataset does not assign."""
95
+ frame = pl.DataFrame({"subject_id": subject_ids.cast(pl.Int64)})
96
+ joined = frame.join(self.splits, on="subject_id", how="left")
97
+ if joined.height != frame.height:
98
+ # `_training_rows` indexes label rows by position against this result, so an expanded
99
+ # join would not error anywhere -- it would shift every split by one and quietly move
100
+ # held-out rows into the fit. `read_split_map` deduplicates; this refuses to proceed if
101
+ # a store was published before it did, rather than trusting that guarantee silently.
102
+ raise ValueError(
103
+ f"the split map assigns some subject more than once: the join expanded {frame.height} "
104
+ f"rows to {joined.height}. Callers align splits to label rows positionally, so this "
105
+ "would misassign splits rather than fail."
106
+ )
107
+ return joined.get_column("split")
108
+
109
+ def code_strings(self) -> list[str]:
110
+ ordered = self.vocabulary.sort("code_index")
111
+ return ordered.get_column("code").to_list()
112
+
113
+
114
+ def build_sequences(store: PatientStore, keys: pl.DataFrame, config: RETAINConfig) -> SequenceBatch:
115
+ """Build one visit sequence per row of `keys`, which must hold the index columns only."""
116
+ extra = set(keys.columns) - set(KEY_COLUMNS)
117
+ if extra:
118
+ raise ValueError(f"build_sequences must never see label columns; got extra column(s) {sorted(extra)}")
119
+
120
+ keys = keys.with_columns(
121
+ pl.col("subject_id").cast(pl.Int64), pl.col("prediction_time").cast(pl.Datetime("us"))
122
+ )
123
+ n_rows = keys.height
124
+ subject_of_row = keys.get_column("subject_id").to_numpy()
125
+ prediction_us = keys.get_column("prediction_time").to_physical().to_numpy().astype(np.int64)
126
+
127
+ rows_by_subject: dict[int, list[int]] = {}
128
+ for row, subject in enumerate(subject_of_row.tolist()):
129
+ rows_by_subject.setdefault(int(subject), []).append(row)
130
+
131
+ bucket_us = int(config.visit_bucket_minutes) * MICROSECONDS_PER_MINUTE
132
+ lookback_us = (
133
+ None if config.lookback_days is None else int(round(config.lookback_days * MICROSECONDS_PER_DAY))
134
+ )
135
+ vocabulary_size = max(store.vocabulary_size, 1)
136
+ side = "right" if config.include_events_at_prediction_time else "left"
137
+
138
+ per_row_visits: list[np.ndarray | None] = [None] * n_rows
139
+ per_row_codes: list[np.ndarray | None] = [None] * n_rows
140
+ seen_subjects: set[int] = set()
141
+
142
+ for shard in store.shards:
143
+ frame = pl.read_parquet(shard, columns=["subject_id", "time", "code_index"])
144
+ if frame.is_empty():
145
+ continue
146
+ shard_subjects = frame.get_column("subject_id").to_numpy()
147
+ shard_times = frame.get_column("time").to_physical().to_numpy().astype(np.int64)
148
+ shard_codes = frame.get_column("code_index").to_numpy().astype(np.int64)
149
+ # The shard is written sorted by (subject_id, time, code_index), so each subject occupies one
150
+ # contiguous range and its events are already in chronological order.
151
+ boundaries = np.flatnonzero(np.diff(shard_subjects)) + 1
152
+ starts = np.concatenate(([0], boundaries))
153
+ stops = np.concatenate((boundaries, [shard_subjects.size]))
154
+ for start, stop in zip(starts.tolist(), stops.tolist()):
155
+ subject = int(shard_subjects[start])
156
+ rows = rows_by_subject.get(subject)
157
+ if not rows:
158
+ continue
159
+ if subject in seen_subjects:
160
+ # MEDS keeps a subject's events in one shard. If that ever stops holding, a later shard
161
+ # would silently replace an earlier partial history instead of extending it.
162
+ raise ValueError(
163
+ f"Subject {subject} appears in more than one shard of {store.root}; RETAIN builds "
164
+ "each history from a single contiguous shard range"
165
+ )
166
+ seen_subjects.add(subject)
167
+ times = shard_times[start:stop]
168
+ codes = shard_codes[start:stop]
169
+ for row in rows:
170
+ prediction = int(prediction_us[row])
171
+ high = int(np.searchsorted(times, prediction, side=side))
172
+ low = 0
173
+ if lookback_us is not None:
174
+ low = int(np.searchsorted(times, prediction - lookback_us, side="left"))
175
+ if high <= low:
176
+ continue
177
+ window_times = times[low:high]
178
+ buckets = window_times if bucket_us == 0 else (window_times // bucket_us) * bucket_us
179
+ visit_time, inverse = np.unique(buckets, return_inverse=True)
180
+ if config.max_visits is not None and visit_time.size > config.max_visits:
181
+ # A reverse-time model attends to the most recent visits, so truncation drops the
182
+ # oldest ones, mirroring the paper's bounded observation window.
183
+ first_kept = visit_time.size - config.max_visits
184
+ keep = inverse >= first_kept
185
+ inverse = inverse[keep] - first_kept
186
+ visit_time = visit_time[first_kept:]
187
+ window_codes = codes[low:high][keep]
188
+ else:
189
+ window_codes = codes[low:high]
190
+ # `x` is binary, so a code seen twice inside one bucket must contribute once. Packing
191
+ # (visit, code) into a single key deduplicates and orders both axes in one pass.
192
+ packed = np.unique(inverse.astype(np.int64) * vocabulary_size + window_codes)
193
+ per_row_visits[row] = visit_time
194
+ per_row_codes[row] = packed
195
+
196
+ row_ptr = np.zeros(n_rows + 1, dtype=np.int64)
197
+ widths: list[np.ndarray] = []
198
+ times_out: list[np.ndarray] = []
199
+ codes_out: list[np.ndarray] = []
200
+ for row in range(n_rows):
201
+ visit_time = per_row_visits[row]
202
+ packed = per_row_codes[row]
203
+ if visit_time is None or packed is None or visit_time.size == 0:
204
+ row_ptr[row + 1] = row_ptr[row]
205
+ continue
206
+ row_ptr[row + 1] = row_ptr[row] + visit_time.size
207
+ visit_index = packed // vocabulary_size
208
+ widths.append(np.bincount(visit_index, minlength=visit_time.size).astype(np.int64))
209
+ times_out.append(visit_time.astype(np.int64))
210
+ codes_out.append((packed % vocabulary_size).astype(np.int32))
211
+
212
+ width = np.concatenate(widths) if widths else np.zeros(0, dtype=np.int64)
213
+ visit_ptr = np.zeros(width.size + 1, dtype=np.int64)
214
+ np.cumsum(width, out=visit_ptr[1:])
215
+ return SequenceBatch(
216
+ keys=np.column_stack((subject_of_row.astype(np.int64), prediction_us)),
217
+ row_ptr=row_ptr,
218
+ visit_ptr=visit_ptr,
219
+ visit_time=np.concatenate(times_out) if times_out else np.zeros(0, dtype=np.int64),
220
+ codes=np.concatenate(codes_out) if codes_out else np.zeros(0, dtype=np.int32),
221
+ )
retain_meds/config.py ADDED
@@ -0,0 +1,114 @@
1
+ """RETAIN's configuration surface.
2
+
3
+ Every architecture and optimisation field mirrors an option of the official `retain.py` command line
4
+ (`mp2893/retain` @ 9fd39c46), with the same default, so a run can be described by naming the upstream flags
5
+ it changed. The first block has no upstream counterpart: it is the MEDS adaptation, because the official
6
+ code starts from an already-tokenised list-of-list-of-list of visits and never has to decide what a visit
7
+ is. Those fields are marked `MEDS` below and are recorded as *adapted* in the implementation report.
8
+
9
+ Core `NNConfig` is not reused: RETAIN has two dropout rates (`keep_prob_emb`, `keep_prob_context`), four
10
+ explicit L2 penalties instead of one weight decay, and a default optimiser (Adadelta) without a learning
11
+ rate to set, so `capacity`, `learning_rate`, `weight_decay` and `dropout` would each be a field with no
12
+ effect.
13
+ """
14
+
15
+ from dataclasses import dataclass
16
+
17
+ from meds_algorithm import AlgorithmConfig
18
+
19
+ SOLVERS = ("adadelta", "adam")
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class RETAINConfig(AlgorithmConfig):
24
+ # -- MEDS adaptation: how a MEDS code stream becomes RETAIN's visit sequences -------------------
25
+ #: Width of the absolute time grid that groups events into one visit. 1440 = calendar day, the
26
+ #: closest generic analogue of the admission-level visits of `process_mimic.py`. 0 keeps every
27
+ #: distinct timestamp as its own visit.
28
+ visit_bucket_minutes: int = 1440
29
+ #: Whether an event stamped exactly at the prediction time belongs to the history. MEDS-DEV tasks
30
+ #: declare their input window with `index_timestamp: end` and `end_inclusive: True`, so the
31
+ #: prediction time *is* the last instant of the granted input window; excluding it would discard
32
+ #: data the task definition hands to the model. Set false for a task whose label-defining event can
33
+ #: coincide with its own prediction time.
34
+ include_events_at_prediction_time: bool = True
35
+ #: Discard events older than this many days before the prediction time. `None` uses all history,
36
+ #: which is what `start: null` in the demo task's input window asks for; the paper used an 18-month
37
+ #: window for its own heart-failure cohort.
38
+ lookback_days: float | None = None
39
+ #: Keep at most this many of the most recent visits. `None` keeps all of them.
40
+ max_visits: int | None = None
41
+ #: Vocabulary filters. The vocabulary is learned from the training split only.
42
+ min_code_count: int = 1
43
+ max_vocab_size: int | None = None
44
+ #: Drop the MEDS standard's two non-clinical subject markers, `MEDS_BIRTH` and `MEDS_DEATH`, from the
45
+ #: code stream. Both the paper (section 4.1: visits are "diagnosis, medication and procedure codes")
46
+ #: and `process_mimic.py` (sequences built from `DIAGNOSES_ICD` alone, `PATIENTS.csv` read only to
47
+ #: derive the label) exclude demographics and outcome markers from the visit sequence. Keeping
48
+ #: `MEDS_BIRTH` makes a birth record the first visit of essentially every sequence, decades before the
49
+ #: rest, where it consumes one of RETAIN's visit-attention slots; keeping `MEDS_DEATH` puts the
50
+ #: canonical outcome marker in the input space of any task whose window can reach it. These are
51
+ #: MEDS-standard codes, not dataset predicates, so excluding them keeps this port predicate-free.
52
+ exclude_meds_standard_markers: bool = True
53
+
54
+ # -- Official `retain.py` options ---------------------------------------------------------------
55
+ #: `--time_file`: upstream defaults to no time information, so this defaults to false.
56
+ use_time: bool = False
57
+ use_log_time: bool = True # --use_log_time
58
+ embed_size: int = 128 # --embed_size
59
+ alpha_hidden_dim_size: int = 128 # --alpha_hidden_dim_size
60
+ beta_hidden_dim_size: int = 128 # --beta_hidden_dim_size
61
+ batch_size: int = 100 # --batch_size
62
+ n_epochs: int = 10 # --n_epochs
63
+ L2_output: float = 0.001 # --L2_output
64
+ L2_emb: float = 0.001 # --L2_emb
65
+ L2_alpha: float = 0.001 # --L2_alpha
66
+ L2_beta: float = 0.001 # --L2_beta
67
+ keep_prob_emb: float = 0.5 # --keep_prob_emb
68
+ keep_prob_context: float = 0.5 # --keep_prob_context
69
+ log_eps: float = 1e-8 # --log_eps
70
+ solver: str = "adadelta" # --solver
71
+
72
+ # -- Runtime ------------------------------------------------------------------------------------
73
+ # The seed is not configuration: it is the `seed` argument of every step (request.execution.seed).
74
+ device: str = "auto"
75
+ #: Only used when the task supplies no `tuning` labels, standing in for the fixed hold-out
76
+ #: fractions of the official `load_data_simple`.
77
+ validation_fraction: float = 0.1
78
+ max_train_rows: int | None = None
79
+ #: Per-key code contributions of Eq. (5). 0 disables the extra pass entirely.
80
+ interpretation_top_k: int = 0
81
+
82
+ def validate(self):
83
+ super().validate()
84
+ if self.solver not in SOLVERS:
85
+ raise ValueError(f"solver must be one of {SOLVERS}, got {self.solver!r}")
86
+ if self.visit_bucket_minutes < 0:
87
+ raise ValueError("visit_bucket_minutes must be >= 0")
88
+ if self.lookback_days is not None and self.lookback_days <= 0:
89
+ raise ValueError("lookback_days must be positive or null")
90
+ if not 0.0 < self.keep_prob_emb <= 1.0 or not 0.0 < self.keep_prob_context <= 1.0:
91
+ raise ValueError("keep_prob_emb and keep_prob_context must lie in (0, 1]")
92
+ for name in ("embed_size", "alpha_hidden_dim_size", "beta_hidden_dim_size", "batch_size", "n_epochs"):
93
+ if getattr(self, name) < 1:
94
+ raise ValueError(f"{name} must be >= 1")
95
+ for name in ("L2_output", "L2_emb", "L2_alpha", "L2_beta"):
96
+ if getattr(self, name) < 0:
97
+ raise ValueError(f"{name} must be >= 0")
98
+ if self.min_code_count < 1:
99
+ raise ValueError("min_code_count must be >= 1")
100
+ for name in ("max_visits", "max_vocab_size", "max_train_rows"):
101
+ value = getattr(self, name)
102
+ if value is not None and value < 1:
103
+ raise ValueError(f"{name} must be >= 1 or null")
104
+ if not 0.0 <= self.validation_fraction < 1.0:
105
+ raise ValueError("validation_fraction must lie in [0, 1)")
106
+ if self.interpretation_top_k < 0:
107
+ raise ValueError("interpretation_top_k must be >= 0")
108
+ if self.log_eps <= 0:
109
+ raise ValueError("log_eps must be positive")
110
+
111
+ @property
112
+ def gru_input_size(self) -> int:
113
+ """RETAIN concatenates the visit duration onto the GRU inputs only (`build_model`)."""
114
+ return self.embed_size + (1 if self.use_time else 0)
@@ -0,0 +1,24 @@
1
+
2
+
3
+
4
+
5
+
6
+ ## MEDS-DEV descriptor for RETAIN. This retains the established command-slot interface.
7
+ ## Each MEDS-DEV run directory {output_dir} holds this run's artifacts as sibling subdirectories (data/,
8
+ ## pretrained/, algorithm/, embeddings/ or trajectories/, predictions/): MEDS-DEV keeps its own cmd.sh and .done marker in
9
+ ## {output_dir}, and artifacts are published by publishing their own destination directory.
10
+ metadata:
11
+ description: "RETAIN (Choi et al., NeurIPS 2016) reverse-time attention model for MEDS, a supervised classifier behind the MEDS algorithm contract."
12
+ contacts:
13
+ - name: "Florent Pollet"
14
+ email: "flo1.raymond@gmail.com"
15
+
16
+ commands:
17
+ supervised:
18
+ train: |-
19
+ retain-meds preprocess_data --profile supervised external_meds_dir={dataset_dir} output_data_dir={output_dir}/data do_overwrite=true
20
+ retain-meds train_supervised --profile supervised input_data_dir={output_dir}/data external_labels_dir={labels_dir} output_supervised_algorithm_dir={output_dir}/algorithm
21
+
22
+ predict: |-
23
+ retain-meds predict_supervised --profile supervised input_data_dir="$(meds-algorithm-resolve-input {model_initialization_dir}/algorithm input_data_dir)" input_supervised_algorithm_dir={model_initialization_dir}/algorithm external_index_dir={labels_dir} splits=[{split}] output_predictions_dir={output_dir}/predictions do_overwrite=true external_evaluation_labels_dir={labels_dir}
24
+
@@ -0,0 +1,25 @@
1
+ # Prototype: one config for local subprocesses or dependent Submitit SLURM jobs.
2
+ # Shared field/resource defaults come from meds-algorithm, not a copied schema.
3
+ defaults:
4
+ - meds_run
5
+ - _self_
6
+
7
+ profile: supervised
8
+ execution: local
9
+ output_dir: ??? # New directory for this run; relative paths use the invocation's working directory.
10
+
11
+ inputs:
12
+ meds_dir: null # Required by preprocessing and materialized zero-shot prediction.
13
+ index_dir: null # Required by inference and prediction.
14
+ labels_dir: null # Required by supervised/finetune/probe training.
15
+
16
+ # Add only algorithm settings that differ from the concrete Config class's defaults.
17
+ algorithm: {}
18
+
19
+ # Optional overrides, e.g. resources.compute.gpus_per_node=1 and its partition/account.
20
+ # Both resource presets default to CPUs; choose resources appropriate for this algorithm.
21
+ # resources:
22
+ # compute:
23
+ # gpus_per_node: 1
24
+ # partition: gpu
25
+ # timeout_min: 240
@@ -0,0 +1,252 @@
1
+ """RETAIN behind the MEDS algorithm contract: `preprocess_data -> train_supervised -> predict_supervised`.
2
+
3
+ That is RETAIN's actual lifecycle: the embedding, both attention GRUs and the risk head are trained jointly
4
+ against the labelled endpoint by `train_RETAIN`, and there is no task-independent pretraining or inference
5
+ stage to place anywhere else.
6
+
7
+ - `preprocess_data` tokenises the MEDS code stream. It reads no labels and no predicates, and fits the
8
+ vocabulary on the dataset's training split only.
9
+ - `train_supervised` fits RETAIN on the `train` labels and selects the epoch on the `tuning` labels (or on a
10
+ seeded hold-out of the training rows when the task supplies none).
11
+ - `predict_supervised` scores every requested key. It receives the index only; the runner attaches ground
12
+ truth after the scores are written.
13
+
14
+ Histories are cut at each prediction time before visits are formed (`_windows.build_sequences`), so nothing
15
+ recorded after a prediction time can reach its prediction.
16
+
17
+ This port is predicate-free: RETAIN's input is a sequence of medical codes, which the MEDS `code` column
18
+ already is. `external_predicates_file` is accepted, recorded by the runner and not read.
19
+ """
20
+
21
+ import hashlib
22
+ import json
23
+ from pathlib import Path
24
+
25
+ import numpy as np
26
+ import polars as pl
27
+ from meds_algorithm import AlgorithmRepository, PayloadMetadata, SupervisedAlgorithm
28
+ from meds_algorithm.requests import PredictSupervisedRequest, PreprocessDataRequest, TrainSupervisedRequest
29
+
30
+ from . import _preprocess
31
+ from ._preprocess import HELD_OUT_SPLIT, SETTINGS_FILE, TRAIN_SPLIT, TUNING_SPLIT, VOCABULARY_SETTINGS
32
+ from ._windows import KEY_COLUMNS, PatientStore, build_sequences
33
+ from .config import RETAINConfig
34
+
35
+ _KEYS = list(KEY_COLUMNS)
36
+ _PREPARED = PayloadMetadata(_preprocess.PAYLOAD_FORMAT, 1)
37
+ _FITTED = PayloadMetadata("retain-supervised-model", 1)
38
+ _PREDICTIONS = PayloadMetadata("retain-predictions", 1)
39
+ _EXPECTED_INPUTS = {"input_data_dir": _PREPARED, "input_supervised_algorithm_dir": _FITTED}
40
+
41
+ _MODEL_FILE = "model.pt"
42
+ _TRAINING_FILE = "training.json"
43
+ _PREPROCESS_REPORT = "preprocess_report.json"
44
+ _PREDICTION_REPORT = "prediction_report.json"
45
+ _INTERPRETATION_FILE = "interpretation.ndjson"
46
+
47
+
48
+ def _format(metadata: PayloadMetadata) -> str:
49
+ return f"{metadata.format}/v{metadata.version}"
50
+
51
+
52
+ def _write_json(path: Path, payload: dict) -> None:
53
+ path.write_text(json.dumps(payload, indent=2, sort_keys=True, default=str))
54
+
55
+
56
+ def _vocabulary_digest(data_dir: Path) -> str:
57
+ """Identity of the vocabulary a model was fitted on: its codes in index order."""
58
+ vocabulary = pl.read_parquet(Path(data_dir) / "vocabulary.parquet").sort("code_index")
59
+ return "sha256:" + hashlib.sha256("\n".join(vocabulary["code"].to_list()).encode()).hexdigest()
60
+
61
+
62
+ class RETAIN(SupervisedAlgorithm):
63
+ name = "retain_meds"
64
+ config_type = RETAINConfig
65
+
66
+ def validate_inputs(self, request) -> None:
67
+ """Reject an incompatible input artifact before any expensive work happens."""
68
+ super().validate_inputs(request)
69
+ for argument, artifact in request.artifact_inputs():
70
+ found = artifact.read_manifest()["artifact"]["payload_format"]
71
+ if found != _format(_EXPECTED_INPUTS[argument]):
72
+ raise ValueError(f"{argument} has payload {found}; expected {_format(_EXPECTED_INPUTS[argument])}")
73
+ data = getattr(request, "data", None)
74
+ if data is not None:
75
+ # The vocabulary is fixed by preprocess_data; changing its filters later would describe a
76
+ # vocabulary that was never built.
77
+ published = json.loads((data.directory / SETTINGS_FILE).read_text())
78
+ for name in VOCABULARY_SETTINGS:
79
+ if published.get(name) != getattr(self.config, name):
80
+ raise ValueError(
81
+ f"{name}={getattr(self.config, name)!r} disagrees with input_data_dir, which was "
82
+ f"built with {name}={published.get(name)!r}. It is a preprocess_data setting; "
83
+ "rebuild the prepared data to change it."
84
+ )
85
+ fitted = getattr(request, "fitted_algorithm", None)
86
+ if fitted is not None and data is not None:
87
+ recorded = json.loads((fitted.directory / _TRAINING_FILE).read_text())["vocabulary_digest"]
88
+ if recorded != _vocabulary_digest(data.directory):
89
+ raise ValueError(
90
+ "input_data_dir holds a different vocabulary from the one the model was fitted on; "
91
+ "pass the prepared data used by train_supervised"
92
+ )
93
+
94
+ def preprocess_data(self, request: PreprocessDataRequest) -> PayloadMetadata:
95
+ """Tokenise a MEDS dataset into RETAIN's code stream; task- and label-independent."""
96
+ report = _preprocess.preprocess(request.meds_dir, request.output.directory, self.config)
97
+ _write_json(request.output.directory / _PREPROCESS_REPORT, report)
98
+ return _PREPARED
99
+
100
+ def train_supervised(self, request: TrainSupervisedRequest) -> PayloadMetadata:
101
+ """Fit RETAIN on the train labels; the tuning labels select the epoch."""
102
+ import torch
103
+
104
+ from ._training import resolve_device, train
105
+
106
+ config = self.config
107
+ seed = request.execution.seed
108
+ store = PatientStore(request.data.directory)
109
+
110
+ # Row order of a label file carries no meaning; a canonical key order keeps the fit a function of
111
+ # the labels' content (it decides how equal-length ties resolve when minibatches are built).
112
+ fit = request.labels.read(TRAIN_SPLIT).sort(_KEYS)
113
+ _require_split(store, fit, TRAIN_SPLIT)
114
+ try:
115
+ validation = request.labels.read(TUNING_SPLIT).sort(_KEYS)
116
+ except FileNotFoundError:
117
+ validation = None
118
+ validation_source = TUNING_SPLIT
119
+ if validation is not None:
120
+ _require_split(store, validation, TUNING_SPLIT)
121
+ elif fit.height > 1 and config.validation_fraction > 0:
122
+ # No tuning labels: a seeded hold-out of the training rows, as `load_data_simple` does upstream.
123
+ shuffled = np.random.default_rng(seed).permutation(fit.height)
124
+ n_validation = min(max(1, round(config.validation_fraction * fit.height)), fit.height - 1)
125
+ validation = fit[np.sort(shuffled[:n_validation])]
126
+ fit = fit[np.sort(shuffled[n_validation:])]
127
+ validation_source = f"seeded {config.validation_fraction:g} hold-out of {TRAIN_SPLIT}"
128
+ else:
129
+ validation = fit.clear()
130
+ validation_source = "none"
131
+ if config.max_train_rows is not None:
132
+ fit = fit.head(config.max_train_rows)
133
+
134
+ # Sequences are built from the keys alone; the targets never reach sequence construction.
135
+ train_batch = build_sequences(store, fit.select(_KEYS), config)
136
+ validation_batch = build_sequences(store, validation.select(_KEYS), config)
137
+ device = resolve_device(config.device)
138
+ model, history = train(
139
+ vocabulary_size=store.vocabulary_size,
140
+ train_batch=train_batch,
141
+ train_labels=fit["boolean_value"].to_numpy().astype(np.float32),
142
+ validation_batch=validation_batch,
143
+ validation_labels=validation["boolean_value"].to_numpy().astype(np.float32),
144
+ config=config,
145
+ device=device,
146
+ seed=seed,
147
+ )
148
+
149
+ output = request.output.directory
150
+ torch.save(
151
+ {"state_dict": model.state_dict(), "vocabulary_size": store.vocabulary_size}, output / _MODEL_FILE
152
+ )
153
+ _write_json(
154
+ output / _TRAINING_FILE,
155
+ {
156
+ "vocabulary_size": store.vocabulary_size,
157
+ "vocabulary_digest": _vocabulary_digest(request.data.directory),
158
+ "n_parameters": model.n_parameters(),
159
+ "device": str(device),
160
+ "seed": seed,
161
+ "training": history.summary(),
162
+ "validation_source": validation_source,
163
+ "fit_rows": fit.height,
164
+ "validation_rows": validation.height,
165
+ "train_visits": int(train_batch.lengths.sum()),
166
+ "empty_history_fit_rows": train_batch.n_empty,
167
+ "empty_history_validation_rows": validation_batch.n_empty,
168
+ },
169
+ )
170
+ return _FITTED
171
+
172
+ def predict_supervised(self, request: PredictSupervisedRequest) -> PayloadMetadata:
173
+ """Score every requested key; scores carry their own keys and are never matched by row order."""
174
+ import torch
175
+
176
+ from ._model import Retain
177
+ from ._training import resolve_device, score
178
+
179
+ config = self.config
180
+ index = request.index.read().sort(_KEYS)
181
+ store = PatientStore(request.data.directory)
182
+ fitted = request.fitted_algorithm.directory
183
+ device = resolve_device(config.device)
184
+ checkpoint = torch.load(fitted / _MODEL_FILE, map_location=device, weights_only=True)
185
+ model = Retain(checkpoint["vocabulary_size"], config).to(device)
186
+ model.load_state_dict(checkpoint["state_dict"])
187
+
188
+ # Every key yields a sequence: a key with no history is scored by the learned intercept.
189
+ batch = build_sequences(store, index.select(_KEYS), config)
190
+ probabilities = score(model, batch, config, device)
191
+ predictions = pl.DataFrame(
192
+ {
193
+ "subject_id": pl.Series(batch.keys[:, 0]).cast(index.schema["subject_id"]),
194
+ "prediction_time": pl.Series(batch.keys[:, 1]).cast(pl.Datetime("us")),
195
+ "predicted_boolean_probability": pl.Series(probabilities, dtype=pl.Float64),
196
+ }
197
+ ).with_columns(
198
+ pl.col("prediction_time").cast(index.schema["prediction_time"]),
199
+ (pl.col("predicted_boolean_probability") > 0.5).alias("predicted_boolean_value"),
200
+ )
201
+ missing = index.join(predictions, on=_KEYS, how="anti").height
202
+ extra = predictions.join(index, on=_KEYS, how="anti").height
203
+ if missing or extra or predictions.height != index.height:
204
+ raise ValueError(f"RETAIN scoring missed {missing} and added {extra} requested keys")
205
+
206
+ output = request.output.directory
207
+ predictions.write_parquet(output / "predictions.parquet")
208
+ report = {
209
+ "scored_keys": predictions.height,
210
+ "unique_subjects": predictions["subject_id"].n_unique(),
211
+ "visits": int(batch.lengths.sum()),
212
+ "empty_history_keys": batch.n_empty,
213
+ "exclusions": "none: every requested key is scored",
214
+ "device": str(device),
215
+ }
216
+ if config.interpretation_top_k > 0:
217
+ from ._interpret import write_contributions
218
+
219
+ report["interpretation_records"] = write_contributions(
220
+ output / _INTERPRETATION_FILE,
221
+ model=model,
222
+ batch=batch,
223
+ config=config,
224
+ device=device,
225
+ code_strings=store.code_strings(),
226
+ top_k=config.interpretation_top_k,
227
+ )
228
+ _write_json(output / _PREDICTION_REPORT, report)
229
+ return _PREDICTIONS
230
+
231
+
232
+ def _require_split(store: PatientStore, labels: pl.DataFrame, split: str) -> None:
233
+ """Training labels may only name subjects of their own MEDS split, so held-out subjects are never fitted."""
234
+ assigned = store.split_of(labels["subject_id"])
235
+ outside = labels.filter(assigned.fill_null("unassigned") != split)
236
+ if outside.height:
237
+ raise ValueError(
238
+ f"{outside['subject_id'].n_unique()} subject(s) of the {split} labels are not in the MEDS "
239
+ f"{split} split (e.g. {HELD_OUT_SPLIT} subjects must never be fitted)"
240
+ )
241
+
242
+
243
+ # Declarative implementation/completion metadata, never fitted algorithms or mutable runtime state.
244
+ IMPLEMENTATIONS = {
245
+ "supervised": RETAIN,
246
+ }
247
+ PROFILES = tuple(IMPLEMENTATIONS)
248
+ STUB_PROFILES: set[str] = set()
249
+
250
+
251
+ def create_repository():
252
+ return AlgorithmRepository(IMPLEMENTATIONS, repo=Path(__file__).resolve().parents[2])