meds-algorithm-retain 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,270 @@
1
+ """Turn a MEDS dataset into RETAIN's tokenised code stream.
2
+
3
+ The official pipeline (`process_mimic.py`) starts from ICD-9 strings, maps each distinct string to an
4
+ integer, and emits a list-of-list-of-list of visits. This module does the same thing from the generic MEDS
5
+ `code` column: no dataset predicates, no variable map, no charted-vital semantics, nothing beyond "a code
6
+ observed at a time for a subject".
7
+
8
+ Two properties matter and are enforced here:
9
+
10
+ * The vocabulary is counted over the **training split only**. A vocabulary fitted on held-out subjects
11
+ would leak information about the evaluation cohort into the model's input space.
12
+ * Events are stored per `(subject_id, time)`, not per visit. Bucketing into visits happens later, after
13
+ the prediction-time cut, so that no visit can straddle a prediction time and carry future events into
14
+ its own history.
15
+ """
16
+
17
+ from __future__ import annotations
18
+
19
+ import json
20
+ from pathlib import Path
21
+ from typing import Any
22
+
23
+ import polars as pl
24
+
25
+ from .config import RETAINConfig
26
+
27
+ PAYLOAD_FORMAT = "retain-patients"
28
+ #: The settings that fixed this payload's vocabulary. `config.json` is reserved for the runner.
29
+ SETTINGS_FILE = "preprocess.json"
30
+ #: Settings that describe the published vocabulary and cannot change after preprocessing.
31
+ VOCABULARY_SETTINGS = ("min_code_count", "max_vocab_size", "exclude_meds_standard_markers")
32
+
33
+ TRAIN_SPLIT = "train"
34
+ TUNING_SPLIT = "tuning"
35
+ HELD_OUT_SPLIT = "held_out"
36
+
37
+ EVENT_COLUMNS = ("subject_id", "time", "code")
38
+
39
+ #: The MEDS standard's two non-clinical subject markers. `MEDS_BIRTH` is a demographic anchor and
40
+ #: `MEDS_DEATH` is the canonical outcome marker; neither is a clinical event of the kind RETAIN's visits
41
+ #: are made of. Defined by the MEDS schema itself, so naming them here is not dataset knowledge.
42
+ MEDS_STANDARD_MARKERS = ("MEDS_BIRTH", "MEDS_DEATH")
43
+
44
+
45
+ def data_files(meds_dir: Path) -> list[Path]:
46
+ root = Path(meds_dir) / "data"
47
+ if not root.is_dir():
48
+ raise FileNotFoundError(f"{meds_dir} is not a MEDS dataset: no data/ directory")
49
+ files = sorted(path for path in root.rglob("*.parquet") if not path.name.startswith("."))
50
+ if not files:
51
+ raise FileNotFoundError(f"{root} contains no parquet shards")
52
+ return files
53
+
54
+
55
+ #: Names for the two places split membership can come from. This port treats the declared map as
56
+ #: authoritative and consults shard placement only in its absence; a sibling port made the opposite
57
+ #: choice, so the resolved value is recorded per run rather than left implicit.
58
+ DECLARED_MAP_SOURCE = "metadata/subject_splits.parquet"
59
+ PLACEMENT_SOURCE = "data/<split>/ placement"
60
+
61
+
62
+ def read_split_map(meds_dir: Path) -> pl.DataFrame:
63
+ """`(subject_id, split)` from MEDS metadata, falling back to the split-sharded directory layout."""
64
+ return read_split_map_with_source(meds_dir)[0]
65
+
66
+
67
+ def read_split_map_with_source(meds_dir: Path) -> tuple[pl.DataFrame, str]:
68
+ """The split map and the name of the source it came from."""
69
+ meds_dir = Path(meds_dir)
70
+ metadata = meds_dir / "metadata" / "subject_splits.parquet"
71
+ if metadata.is_file():
72
+ frame = pl.read_parquet(metadata)
73
+ if {"subject_id", "split"} <= set(frame.columns):
74
+ return (
75
+ frame.select(
76
+ pl.col("subject_id").cast(pl.Int64), pl.col("split").cast(pl.Utf8)
77
+ ).unique(subset=["subject_id"], keep="first"),
78
+ DECLARED_MAP_SOURCE,
79
+ )
80
+ root = meds_dir / "data"
81
+ frames = []
82
+ placed = False
83
+ for path in data_files(meds_dir):
84
+ parts = path.relative_to(root).parts
85
+ placed = placed or len(parts) > 1
86
+ split = parts[0] if len(parts) > 1 else TRAIN_SPLIT
87
+ subjects = pl.read_parquet(path, columns=["subject_id"]).unique()
88
+ frames.append(subjects.with_columns(pl.lit(split).alias("split")))
89
+ if not placed:
90
+ # A flat `data/*.parquet` layout carries no split information, and neither does this
91
+ # dataset's metadata. Defaulting every subject to `train` -- which this loop would
92
+ # otherwise do -- fits the vocabulary over held-out patients and trains on them, with
93
+ # nothing anywhere reporting an error. Measured: on a flat three-subject extract the map
94
+ # came back `[(1, 'train'), (2, 'train'), (3, 'train')]`, one distinct split. Refuse instead.
95
+ raise ValueError(
96
+ f"{meds_dir} has a flat data/ layout and no metadata/subject_splits.parquet, so subject "
97
+ "splits cannot be determined. Treating every subject as 'train' would make the "
98
+ "vocabulary transductive and fit on held-out patients silently. Supply "
99
+ "metadata/subject_splits.parquet, or shard the extract into data/<split>/ directories."
100
+ )
101
+ return (
102
+ pl.concat(frames)
103
+ .select(pl.col("subject_id").cast(pl.Int64), pl.col("split").cast(pl.Utf8))
104
+ .unique(subset=["subject_id"], keep="first"),
105
+ PLACEMENT_SOURCE,
106
+ )
107
+
108
+
109
+ def _keeps_code(config: RETAINConfig) -> pl.Expr:
110
+ """Row filter dropping the MEDS standard markers when configured to."""
111
+ if not config.exclude_meds_standard_markers:
112
+ return pl.lit(True)
113
+ return ~pl.col("code").is_in(list(MEDS_STANDARD_MARKERS))
114
+
115
+
116
+ def _build_vocabulary(files: list[Path], train_subjects: pl.DataFrame, config: RETAINConfig) -> pl.DataFrame:
117
+ """Count codes over training subjects and assign integer indices, most frequent first."""
118
+ counts: pl.DataFrame | None = None
119
+ for path in files:
120
+ shard = (
121
+ pl.scan_parquet(path)
122
+ .select("subject_id", "time", "code")
123
+ .filter(pl.col("time").is_not_null())
124
+ .filter(_keeps_code(config))
125
+ .with_columns(pl.col("subject_id").cast(pl.Int64))
126
+ .join(train_subjects.lazy(), on="subject_id", how="semi")
127
+ .group_by("code")
128
+ .agg(pl.len().alias("count"))
129
+ .collect()
130
+ )
131
+ counts = shard if counts is None else pl.concat([counts, shard])
132
+ if counts is None or counts.is_empty():
133
+ raise ValueError(
134
+ "No timestamped events belong to the training split, so RETAIN has no vocabulary to learn. "
135
+ "Check that the MEDS dataset declares a 'train' split."
136
+ )
137
+ counts = counts.group_by("code").agg(pl.col("count").sum())
138
+ counts = counts.filter(pl.col("count") >= config.min_code_count)
139
+ # Deterministic order: frequency first, then the code string, so the same dataset always produces
140
+ # the same indices regardless of shard iteration order.
141
+ counts = counts.sort(["count", "code"], descending=[True, False])
142
+ if config.max_vocab_size is not None:
143
+ counts = counts.head(config.max_vocab_size)
144
+ if counts.is_empty():
145
+ raise ValueError(
146
+ f"min_code_count={config.min_code_count} removed every code from the training split"
147
+ )
148
+ return counts.with_row_index("code_index").select(
149
+ pl.col("code"), pl.col("code_index").cast(pl.UInt32), pl.col("count").cast(pl.Int64)
150
+ )
151
+
152
+
153
+ def _placement_agreement(
154
+ placement_frames: list[pl.DataFrame], splits: pl.DataFrame, meds_dir: Path
155
+ ) -> dict[str, Any]:
156
+ """Compare shard placement against the declared split map, when placement encodes splits at all.
157
+
158
+ Both sources exist on a normal MEDS extract and this port resolves membership from the map, so a
159
+ divergence would move subjects between cohorts here **and nowhere else** -- a sibling port resolves
160
+ from placement. Measured on the pilot extract: 0 disagreements over 364,627 subjects, reported by
161
+ the cluster session. That agreement is what made the precedence unobservable in the first place, so
162
+ it is checked per run rather than assumed to hold.
163
+
164
+ Only compared when the directory names are split names. `data/shard_00/` is a legal layout that
165
+ encodes no split, and comparing against it would refuse every such extract.
166
+ """
167
+ if not placement_frames:
168
+ return {"placement_encodes_splits": False, "placement_disagreements": None}
169
+ placement = pl.concat(placement_frames).unique(subset=["subject_id"], keep="first")
170
+ declared = set(splits.get_column("split").unique().to_list())
171
+ observed = set(placement.get_column("placement").unique().to_list())
172
+ if not observed <= declared:
173
+ return {"placement_encodes_splits": False, "placement_disagreements": None}
174
+
175
+ joined = placement.join(splits, on="subject_id", how="inner")
176
+ disagreements = joined.filter(pl.col("placement") != pl.col("split"))
177
+ if disagreements.height:
178
+ sample = disagreements.head(3).rows()
179
+ raise ValueError(
180
+ f"{meds_dir} contradicts itself about {disagreements.height} subject(s): shard placement "
181
+ f"and {DECLARED_MAP_SOURCE} assign them to different splits, e.g. {sample} as "
182
+ "(subject_id, placement, declared). This port resolves membership from the declared map, "
183
+ "so it would train on a different cohort than a port resolving from placement, with no "
184
+ "error on either side. Reconcile the extract before comparing models across it."
185
+ )
186
+ return {"placement_encodes_splits": True, "placement_disagreements": 0}
187
+
188
+
189
+ def preprocess(meds_dir: Path, staging: Path, config: RETAINConfig) -> dict[str, Any]:
190
+ """Write the patients payload into `staging` and return its work units."""
191
+ meds_dir = Path(meds_dir)
192
+ files = data_files(meds_dir)
193
+ splits, split_source = read_split_map_with_source(meds_dir)
194
+ train_subjects = splits.filter(pl.col("split") == TRAIN_SPLIT).select("subject_id")
195
+ if train_subjects.is_empty():
196
+ raise ValueError(
197
+ f"{meds_dir} declares no '{TRAIN_SPLIT}' split; RETAIN fits its vocabulary and its "
198
+ "parameters on the training split only."
199
+ )
200
+
201
+ vocabulary = _build_vocabulary(files, train_subjects, config)
202
+ vocabulary.write_parquet(staging / "vocabulary.parquet")
203
+ splits.write_parquet(staging / "subject_splits.parquet")
204
+ lookup = vocabulary.select("code", "code_index")
205
+
206
+ sequences = staging / "sequences"
207
+ totals = {
208
+ "events": 0,
209
+ "static_events": 0,
210
+ "meds_standard_marker_events": 0,
211
+ "out_of_vocabulary_events": 0,
212
+ "same_instant_duplicate_events": 0,
213
+ "kept_events": 0,
214
+ }
215
+ subjects: set[int] = set()
216
+ event_groups = 0
217
+ placement_frames: list[pl.DataFrame] = []
218
+ for path in files:
219
+ parts = path.relative_to(meds_dir / "data").parts
220
+ frame = pl.read_parquet(path, columns=list(EVENT_COLUMNS)).with_columns(
221
+ pl.col("subject_id").cast(pl.Int64),
222
+ pl.col("time").cast(pl.Datetime("us")),
223
+ pl.col("code").cast(pl.Utf8),
224
+ )
225
+ totals["events"] += frame.height
226
+ if len(parts) > 1:
227
+ placement_frames.append(
228
+ frame.select(pl.col("subject_id").unique()).with_columns(
229
+ pl.lit(parts[0]).alias("placement")
230
+ )
231
+ )
232
+ timestamped = frame.filter(pl.col("time").is_not_null())
233
+ totals["static_events"] += frame.height - timestamped.height
234
+ clinical = timestamped.filter(_keeps_code(config))
235
+ totals["meds_standard_marker_events"] += timestamped.height - clinical.height
236
+ mapped = clinical.join(lookup, on="code", how="inner")
237
+ totals["out_of_vocabulary_events"] += clinical.height - mapped.height
238
+ # One row per (subject, timestamp, code): RETAIN's input is binary, so a code repeated at the
239
+ # same instant must contribute once, exactly as `xvec[subseq] = 1.` does upstream.
240
+ events = (
241
+ mapped.select("subject_id", "time", "code_index")
242
+ .unique(subset=["subject_id", "time", "code_index"])
243
+ .sort(["subject_id", "time", "code_index"])
244
+ )
245
+ # Every input row must land in exactly one reported bucket, or a reader who sees a tiny
246
+ # out_of_vocabulary_events will conclude that little was discarded and be wrong by however
247
+ # many same-instant duplicates the extract happens to contain -- a quantity that differs
248
+ # between datasets and was, until this counter, reported by nothing.
249
+ totals["same_instant_duplicate_events"] += mapped.height - events.height
250
+ totals["kept_events"] += events.height
251
+ subjects.update(events.get_column("subject_id").unique().to_list())
252
+ event_groups += events.select("subject_id", "time").unique().height
253
+ destination = sequences / path.relative_to(meds_dir / "data")
254
+ destination.parent.mkdir(parents=True, exist_ok=True)
255
+ events.write_parquet(destination)
256
+
257
+ agreement = _placement_agreement(placement_frames, splits, meds_dir)
258
+
259
+ settings = {name: getattr(config, name) for name in VOCABULARY_SETTINGS}
260
+ (staging / SETTINGS_FILE).write_text(json.dumps(settings, indent=2, sort_keys=True))
261
+ return {
262
+ "subjects": len(subjects),
263
+ "subjects_in_split_map": splits.height,
264
+ "split_source": split_source,
265
+ **agreement,
266
+ "shards": len(files),
267
+ "vocabulary_size": vocabulary.height,
268
+ "event_groups": event_groups,
269
+ **totals,
270
+ }
@@ -0,0 +1,320 @@
1
+ """Batching, the optimisation loop, and scoring.
2
+
3
+ The loop follows `train_RETAIN` in `mp2893/retain` @ 9fd39c46: length-sorted minibatches whose *order* is
4
+ reshuffled every epoch, Adadelta by default, the four explicit L2 penalties added to a binary cross-entropy
5
+ guarded by `log_eps`, and the parameters of the epoch with the best validation AUC.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import copy
11
+ import random
12
+ import sys
13
+ from dataclasses import dataclass, field
14
+ from typing import Any
15
+
16
+ import numpy as np
17
+ import torch
18
+
19
+ from .config import RETAINConfig
20
+ from ._model import Retain
21
+ from ._windows import SequenceBatch
22
+
23
+
24
+ def resolve_device(selector: str) -> torch.device:
25
+ if selector == "auto":
26
+ return torch.device("cuda" if torch.cuda.is_available() else "cpu")
27
+ return torch.device(selector)
28
+
29
+
30
+ def gpu_metrics(device: torch.device) -> dict[str, Any] | None:
31
+ """SPEC section 6's GPU block, or `None` when no accelerator was used."""
32
+ if device.type != "cuda" or not torch.cuda.is_available():
33
+ return None
34
+ return {
35
+ "peak_allocated_bytes": int(torch.cuda.max_memory_allocated()),
36
+ "peak_reserved_bytes": int(torch.cuda.max_memory_reserved()),
37
+ "source": "torch.cuda.max_memory_allocated",
38
+ }
39
+
40
+
41
+ def roc_auc(labels: np.ndarray, scores: np.ndarray) -> float | None:
42
+ """Rank-based ROC AUC with tie correction. `None` when only one class is present.
43
+
44
+ Equivalent to `sklearn.metrics.roc_auc_score`, which the official code calls, without taking on
45
+ scikit-learn as a runtime dependency for a single function.
46
+ """
47
+ labels = np.asarray(labels).astype(bool)
48
+ scores = np.asarray(scores, dtype=np.float64)
49
+ n_positive = int(labels.sum())
50
+ n_negative = int(labels.size - n_positive)
51
+ if n_positive == 0 or n_negative == 0:
52
+ return None
53
+ order = np.argsort(scores, kind="mergesort")
54
+ ranks = np.empty(scores.size, dtype=np.float64)
55
+ ranks[order] = np.arange(1, scores.size + 1, dtype=np.float64)
56
+ ordered = scores[order]
57
+ start = 0
58
+ while start < ordered.size:
59
+ stop = start + 1
60
+ while stop < ordered.size and ordered[stop] == ordered[start]:
61
+ stop += 1
62
+ if stop - start > 1:
63
+ ranks[order[start:stop]] = (start + 1 + stop) / 2.0
64
+ start = stop
65
+ return float((ranks[labels].sum() - n_positive * (n_positive + 1) / 2.0) / (n_positive * n_negative))
66
+
67
+
68
+ @dataclass
69
+ class TensorBatch:
70
+ codes: torch.Tensor
71
+ offsets: torch.Tensor
72
+ slots: torch.Tensor
73
+ lengths: torch.Tensor
74
+ gaps: torch.Tensor | None
75
+ n_rows: int
76
+ n_visits: int
77
+
78
+
79
+ def collate(batch: SequenceBatch, rows: np.ndarray, config: RETAINConfig, device: torch.device) -> TensorBatch:
80
+ """Build the `EmbeddingBag` view of `rows` and the padded grid they scatter into.
81
+
82
+ A row with no history keeps a length of one so the attention softmax has a position to normalise
83
+ over. Its single visit carries no bag, so `v = 0`, `c = 0`, and the score is the model's learned
84
+ intercept. That is what lets every requested key be scored instead of excluded.
85
+ """
86
+ rows = np.asarray(rows, dtype=np.int64)
87
+ lengths = batch.lengths[rows]
88
+ n_visits = int(max(lengths.max(initial=0), 1))
89
+ gaps_source = batch.visit_gap_days() if config.use_time else None
90
+
91
+ code_chunks: list[np.ndarray] = []
92
+ width_chunks: list[np.ndarray] = []
93
+ slot_chunks: list[np.ndarray] = []
94
+ gap_chunks: list[np.ndarray] = []
95
+ for position, row in enumerate(rows.tolist()):
96
+ start, stop = int(batch.row_ptr[row]), int(batch.row_ptr[row + 1])
97
+ if stop == start:
98
+ continue
99
+ code_chunks.append(batch.codes[int(batch.visit_ptr[start]) : int(batch.visit_ptr[stop])])
100
+ width_chunks.append(batch.visit_ptr[start + 1 : stop + 1] - batch.visit_ptr[start:stop])
101
+ slot_chunks.append(position * n_visits + np.arange(stop - start, dtype=np.int64))
102
+ if gaps_source is not None:
103
+ gap_chunks.append(gaps_source[start:stop])
104
+
105
+ if code_chunks:
106
+ codes = np.concatenate(code_chunks).astype(np.int64)
107
+ widths = np.concatenate(width_chunks).astype(np.int64)
108
+ offsets = np.zeros(widths.size + 1, dtype=np.int64)
109
+ np.cumsum(widths, out=offsets[1:])
110
+ slots = np.concatenate(slot_chunks)
111
+ else:
112
+ codes = np.zeros(0, dtype=np.int64)
113
+ offsets = np.zeros(1, dtype=np.int64)
114
+ slots = np.zeros(0, dtype=np.int64)
115
+
116
+ gaps = None
117
+ if gaps_source is not None:
118
+ stacked = np.concatenate(gap_chunks) if gap_chunks else np.zeros(0, dtype=np.float32)
119
+ gaps = torch.from_numpy(np.ascontiguousarray(stacked)).to(device)
120
+ return TensorBatch(
121
+ codes=torch.from_numpy(codes).to(device),
122
+ offsets=torch.from_numpy(offsets).to(device),
123
+ slots=torch.from_numpy(slots).to(device),
124
+ lengths=torch.from_numpy(np.maximum(lengths, 1)),
125
+ gaps=gaps,
126
+ n_rows=int(rows.size),
127
+ n_visits=n_visits,
128
+ )
129
+
130
+
131
+ def _cross_entropy(logits: torch.Tensor, targets: torch.Tensor, log_eps: float) -> torch.Tensor:
132
+ """`-(y log(y_hat + eps) + (1-y) log(1 - y_hat + eps))`, exactly as in `build_model`.
133
+
134
+ Written out rather than delegated to `binary_cross_entropy_with_logits` because upstream adds
135
+ `log_eps` inside both logarithms, which changes the loss surface near saturation.
136
+ """
137
+ probabilities = torch.sigmoid(logits)
138
+ return -(
139
+ targets * torch.log(probabilities + log_eps)
140
+ + (1.0 - targets) * torch.log(1.0 - probabilities + log_eps)
141
+ ).mean()
142
+
143
+
144
+ def _optimizer(model: Retain, config: RETAINConfig) -> torch.optim.Optimizer:
145
+ parameters = model.trainable_parameters()
146
+ if config.solver == "adam":
147
+ # `adam()` upstream parameterises b1/b2 as 1 - beta, so b1=0.1, b2=0.001 is the usual
148
+ # (0.9, 0.999), and its fix1/fix2 terms are standard bias correction.
149
+ return torch.optim.Adam(parameters, lr=0.0002, betas=(0.9, 0.999), eps=1e-8)
150
+ # The Theano `adadelta()` block is rho=0.95 with eps=1e-6 and an implicit unit learning rate.
151
+ return torch.optim.Adadelta(parameters, lr=1.0, rho=0.95, eps=1e-6)
152
+
153
+
154
+ @dataclass
155
+ class TrainingHistory:
156
+ """A commit-safe record of how the fit actually went."""
157
+
158
+ epochs_run: int = 0
159
+ best_epoch: int = 0
160
+ best_validation_auc: float | None = None
161
+ train_losses: list[float] = field(default_factory=list)
162
+ validation_aucs: list[float | None] = field(default_factory=list)
163
+ n_train_rows: int = 0
164
+ n_validation_rows: int = 0
165
+ n_positive_train_rows: int = 0
166
+ n_positive_validation_rows: int = 0
167
+ n_empty_train_sequences: int = 0
168
+ n_empty_validation_sequences: int = 0
169
+ optimizer_steps: int = 0
170
+ selected_by: str = "validation_auc"
171
+
172
+ def summary(self) -> dict[str, Any]:
173
+ return {
174
+ "epochs_run": self.epochs_run,
175
+ "best_epoch": self.best_epoch,
176
+ "best_validation_auc": self.best_validation_auc,
177
+ "train_losses": [round(value, 6) for value in self.train_losses],
178
+ "validation_aucs": [None if v is None else round(v, 6) for v in self.validation_aucs],
179
+ "n_train_rows": self.n_train_rows,
180
+ "n_validation_rows": self.n_validation_rows,
181
+ "n_positive_train_rows": self.n_positive_train_rows,
182
+ "n_positive_validation_rows": self.n_positive_validation_rows,
183
+ "n_empty_train_sequences": self.n_empty_train_sequences,
184
+ "n_empty_validation_sequences": self.n_empty_validation_sequences,
185
+ "optimizer_steps": self.optimizer_steps,
186
+ "selected_by": self.selected_by,
187
+ }
188
+
189
+
190
+ def _length_sorted(batch: SequenceBatch) -> np.ndarray:
191
+ """`len_argsort`: order rows by sequence length so a minibatch pads as little as possible.
192
+
193
+ Ties break on `(subject_id, prediction_time)` rather than on row position. Upstream's `len_argsort`
194
+ is a stable sort on a list whose order is fixed by a pickled file, so position is a stable tiebreak
195
+ there; here the rows come from a task index whose row order carries no meaning and is not guaranteed
196
+ to be reproduced. Leaving the tie on position would let two byte-different but content-identical label
197
+ files produce different minibatches, and so a different fit, at the same seed.
198
+ """
199
+ return np.lexsort((batch.keys[:, 1], batch.keys[:, 0], batch.lengths))
200
+
201
+
202
+ def score(
203
+ model: Retain, batch: SequenceBatch, config: RETAINConfig, device: torch.device
204
+ ) -> np.ndarray:
205
+ """Probabilities for every row of `batch`, in `batch`'s own row order."""
206
+ model.eval()
207
+ scores = np.zeros(batch.n_keys, dtype=np.float64)
208
+ order = _length_sorted(batch)
209
+ with torch.no_grad():
210
+ for start in range(0, order.size, config.batch_size):
211
+ rows = order[start : start + config.batch_size]
212
+ tensors = collate(batch, rows, config, device)
213
+ output = model(
214
+ tensors.codes,
215
+ tensors.offsets,
216
+ tensors.slots,
217
+ tensors.lengths,
218
+ tensors.gaps,
219
+ n_rows=tensors.n_rows,
220
+ n_visits=tensors.n_visits,
221
+ )
222
+ scores[rows] = torch.sigmoid(output.logits).detach().cpu().numpy().astype(np.float64)
223
+ return scores
224
+
225
+
226
+ def train(
227
+ *,
228
+ vocabulary_size: int,
229
+ train_batch: SequenceBatch,
230
+ train_labels: np.ndarray,
231
+ validation_batch: SequenceBatch,
232
+ validation_labels: np.ndarray,
233
+ config: RETAINConfig,
234
+ device: torch.device,
235
+ seed: int,
236
+ ) -> tuple[Retain, TrainingHistory]:
237
+ torch.manual_seed(seed)
238
+ np.random.seed(seed)
239
+ shuffler = random.Random(seed)
240
+
241
+ model = Retain(vocabulary_size, config).to(device)
242
+ optimizer = _optimizer(model, config)
243
+ targets = torch.from_numpy(train_labels.astype(np.float32)).to(device)
244
+
245
+ order = _length_sorted(train_batch)
246
+ n_batches = max(1, int(np.ceil(order.size / config.batch_size)))
247
+ history = TrainingHistory(
248
+ n_train_rows=train_batch.n_keys,
249
+ n_validation_rows=validation_batch.n_keys,
250
+ n_positive_train_rows=int(train_labels.sum()),
251
+ n_positive_validation_rows=int(validation_labels.sum()),
252
+ n_empty_train_sequences=train_batch.n_empty,
253
+ n_empty_validation_sequences=validation_batch.n_empty,
254
+ )
255
+ best_state = copy.deepcopy(model.state_dict())
256
+
257
+ for epoch in range(config.n_epochs):
258
+ model.train()
259
+ losses: list[float] = []
260
+ for index in shuffler.sample(range(n_batches), n_batches):
261
+ rows = order[index * config.batch_size : (index + 1) * config.batch_size]
262
+ if not rows.size:
263
+ continue
264
+ tensors = collate(train_batch, rows, config, device)
265
+ output = model(
266
+ tensors.codes,
267
+ tensors.offsets,
268
+ tensors.slots,
269
+ tensors.lengths,
270
+ tensors.gaps,
271
+ n_rows=tensors.n_rows,
272
+ n_visits=tensors.n_visits,
273
+ )
274
+ loss = _cross_entropy(output.logits, targets[rows], config.log_eps)
275
+ cost = loss + model.regularization()
276
+ optimizer.zero_grad(set_to_none=True)
277
+ cost.backward()
278
+ optimizer.step()
279
+ history.optimizer_steps += 1
280
+ losses.append(float(loss.detach().cpu()))
281
+ history.epochs_run = epoch + 1
282
+ history.train_losses.append(float(np.mean(losses)) if losses else float("nan"))
283
+
284
+ auc = None
285
+ if validation_batch.n_keys:
286
+ auc = roc_auc(validation_labels, score(model, validation_batch, config, device))
287
+ history.validation_aucs.append(auc)
288
+ improved = auc is not None and (
289
+ history.best_validation_auc is None or auc > history.best_validation_auc
290
+ )
291
+ if improved:
292
+ history.best_validation_auc = auc
293
+ history.best_epoch = epoch
294
+ best_state = copy.deepcopy(model.state_dict())
295
+
296
+ # Emitted every epoch, unconditionally, to stderr. A full-data fit runs for hours and this
297
+ # command writes its artifact and manifest only at the very end, so without this line a healthy
298
+ # run and a hung one are indistinguishable from outside and there is nothing on disk to inspect
299
+ # mid-run. `epochs_since_best` is the quantity that says whether the fit is still going anywhere.
300
+ print(
301
+ f"[retain] epoch {epoch + 1}/{config.n_epochs} "
302
+ f"train_loss={history.train_losses[-1]:.6f} "
303
+ f"validation_auc={'n/a' if auc is None else f'{auc:.6f}'} "
304
+ f"best_epoch={history.best_epoch + 1} "
305
+ f"epochs_since_best={epoch - history.best_epoch} "
306
+ f"steps={history.optimizer_steps}"
307
+ f"{' *' if improved else ''}",
308
+ file=sys.stderr,
309
+ flush=True,
310
+ )
311
+
312
+ if history.best_validation_auc is None:
313
+ # No usable validation signal: either no tuning rows, or one class only. Upstream would have
314
+ # saved nothing at all here; keeping the final epoch is the closest usable behaviour and is
315
+ # recorded so the report cannot claim a selection that never happened.
316
+ history.selected_by = "final_epoch"
317
+ history.best_epoch = history.epochs_run - 1
318
+ else:
319
+ model.load_state_dict(best_state)
320
+ return model, history