meds-algorithm-retain 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meds_algorithm_retain-0.1.0.dist-info/METADATA +218 -0
- meds_algorithm_retain-0.1.0.dist-info/RECORD +16 -0
- meds_algorithm_retain-0.1.0.dist-info/WHEEL +4 -0
- meds_algorithm_retain-0.1.0.dist-info/entry_points.txt +2 -0
- meds_algorithm_retain-0.1.0.dist-info/licenses/LICENSE +21 -0
- retain_meds/__init__.py +6 -0
- retain_meds/__main__.py +65 -0
- retain_meds/_interpret.py +102 -0
- retain_meds/_model.py +171 -0
- retain_meds/_preprocess.py +270 -0
- retain_meds/_training.py +320 -0
- retain_meds/_windows.py +221 -0
- retain_meds/config.py +114 -0
- retain_meds/resources/algorithms/supervised/model.yaml +24 -0
- retain_meds/resources/runs/run.yaml +25 -0
- retain_meds/retain_meds.py +252 -0
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"""Turn a MEDS dataset into RETAIN's tokenised code stream.
|
|
2
|
+
|
|
3
|
+
The official pipeline (`process_mimic.py`) starts from ICD-9 strings, maps each distinct string to an
|
|
4
|
+
integer, and emits a list-of-list-of-list of visits. This module does the same thing from the generic MEDS
|
|
5
|
+
`code` column: no dataset predicates, no variable map, no charted-vital semantics, nothing beyond "a code
|
|
6
|
+
observed at a time for a subject".
|
|
7
|
+
|
|
8
|
+
Two properties matter and are enforced here:
|
|
9
|
+
|
|
10
|
+
* The vocabulary is counted over the **training split only**. A vocabulary fitted on held-out subjects
|
|
11
|
+
would leak information about the evaluation cohort into the model's input space.
|
|
12
|
+
* Events are stored per `(subject_id, time)`, not per visit. Bucketing into visits happens later, after
|
|
13
|
+
the prediction-time cut, so that no visit can straddle a prediction time and carry future events into
|
|
14
|
+
its own history.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import json
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
import polars as pl
|
|
24
|
+
|
|
25
|
+
from .config import RETAINConfig
|
|
26
|
+
|
|
27
|
+
PAYLOAD_FORMAT = "retain-patients"
|
|
28
|
+
#: The settings that fixed this payload's vocabulary. `config.json` is reserved for the runner.
|
|
29
|
+
SETTINGS_FILE = "preprocess.json"
|
|
30
|
+
#: Settings that describe the published vocabulary and cannot change after preprocessing.
|
|
31
|
+
VOCABULARY_SETTINGS = ("min_code_count", "max_vocab_size", "exclude_meds_standard_markers")
|
|
32
|
+
|
|
33
|
+
TRAIN_SPLIT = "train"
|
|
34
|
+
TUNING_SPLIT = "tuning"
|
|
35
|
+
HELD_OUT_SPLIT = "held_out"
|
|
36
|
+
|
|
37
|
+
EVENT_COLUMNS = ("subject_id", "time", "code")
|
|
38
|
+
|
|
39
|
+
#: The MEDS standard's two non-clinical subject markers. `MEDS_BIRTH` is a demographic anchor and
|
|
40
|
+
#: `MEDS_DEATH` is the canonical outcome marker; neither is a clinical event of the kind RETAIN's visits
|
|
41
|
+
#: are made of. Defined by the MEDS schema itself, so naming them here is not dataset knowledge.
|
|
42
|
+
MEDS_STANDARD_MARKERS = ("MEDS_BIRTH", "MEDS_DEATH")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def data_files(meds_dir: Path) -> list[Path]:
|
|
46
|
+
root = Path(meds_dir) / "data"
|
|
47
|
+
if not root.is_dir():
|
|
48
|
+
raise FileNotFoundError(f"{meds_dir} is not a MEDS dataset: no data/ directory")
|
|
49
|
+
files = sorted(path for path in root.rglob("*.parquet") if not path.name.startswith("."))
|
|
50
|
+
if not files:
|
|
51
|
+
raise FileNotFoundError(f"{root} contains no parquet shards")
|
|
52
|
+
return files
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
#: Names for the two places split membership can come from. This port treats the declared map as
|
|
56
|
+
#: authoritative and consults shard placement only in its absence; a sibling port made the opposite
|
|
57
|
+
#: choice, so the resolved value is recorded per run rather than left implicit.
|
|
58
|
+
DECLARED_MAP_SOURCE = "metadata/subject_splits.parquet"
|
|
59
|
+
PLACEMENT_SOURCE = "data/<split>/ placement"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def read_split_map(meds_dir: Path) -> pl.DataFrame:
|
|
63
|
+
"""`(subject_id, split)` from MEDS metadata, falling back to the split-sharded directory layout."""
|
|
64
|
+
return read_split_map_with_source(meds_dir)[0]
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def read_split_map_with_source(meds_dir: Path) -> tuple[pl.DataFrame, str]:
|
|
68
|
+
"""The split map and the name of the source it came from."""
|
|
69
|
+
meds_dir = Path(meds_dir)
|
|
70
|
+
metadata = meds_dir / "metadata" / "subject_splits.parquet"
|
|
71
|
+
if metadata.is_file():
|
|
72
|
+
frame = pl.read_parquet(metadata)
|
|
73
|
+
if {"subject_id", "split"} <= set(frame.columns):
|
|
74
|
+
return (
|
|
75
|
+
frame.select(
|
|
76
|
+
pl.col("subject_id").cast(pl.Int64), pl.col("split").cast(pl.Utf8)
|
|
77
|
+
).unique(subset=["subject_id"], keep="first"),
|
|
78
|
+
DECLARED_MAP_SOURCE,
|
|
79
|
+
)
|
|
80
|
+
root = meds_dir / "data"
|
|
81
|
+
frames = []
|
|
82
|
+
placed = False
|
|
83
|
+
for path in data_files(meds_dir):
|
|
84
|
+
parts = path.relative_to(root).parts
|
|
85
|
+
placed = placed or len(parts) > 1
|
|
86
|
+
split = parts[0] if len(parts) > 1 else TRAIN_SPLIT
|
|
87
|
+
subjects = pl.read_parquet(path, columns=["subject_id"]).unique()
|
|
88
|
+
frames.append(subjects.with_columns(pl.lit(split).alias("split")))
|
|
89
|
+
if not placed:
|
|
90
|
+
# A flat `data/*.parquet` layout carries no split information, and neither does this
|
|
91
|
+
# dataset's metadata. Defaulting every subject to `train` -- which this loop would
|
|
92
|
+
# otherwise do -- fits the vocabulary over held-out patients and trains on them, with
|
|
93
|
+
# nothing anywhere reporting an error. Measured: on a flat three-subject extract the map
|
|
94
|
+
# came back `[(1, 'train'), (2, 'train'), (3, 'train')]`, one distinct split. Refuse instead.
|
|
95
|
+
raise ValueError(
|
|
96
|
+
f"{meds_dir} has a flat data/ layout and no metadata/subject_splits.parquet, so subject "
|
|
97
|
+
"splits cannot be determined. Treating every subject as 'train' would make the "
|
|
98
|
+
"vocabulary transductive and fit on held-out patients silently. Supply "
|
|
99
|
+
"metadata/subject_splits.parquet, or shard the extract into data/<split>/ directories."
|
|
100
|
+
)
|
|
101
|
+
return (
|
|
102
|
+
pl.concat(frames)
|
|
103
|
+
.select(pl.col("subject_id").cast(pl.Int64), pl.col("split").cast(pl.Utf8))
|
|
104
|
+
.unique(subset=["subject_id"], keep="first"),
|
|
105
|
+
PLACEMENT_SOURCE,
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def _keeps_code(config: RETAINConfig) -> pl.Expr:
|
|
110
|
+
"""Row filter dropping the MEDS standard markers when configured to."""
|
|
111
|
+
if not config.exclude_meds_standard_markers:
|
|
112
|
+
return pl.lit(True)
|
|
113
|
+
return ~pl.col("code").is_in(list(MEDS_STANDARD_MARKERS))
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def _build_vocabulary(files: list[Path], train_subjects: pl.DataFrame, config: RETAINConfig) -> pl.DataFrame:
|
|
117
|
+
"""Count codes over training subjects and assign integer indices, most frequent first."""
|
|
118
|
+
counts: pl.DataFrame | None = None
|
|
119
|
+
for path in files:
|
|
120
|
+
shard = (
|
|
121
|
+
pl.scan_parquet(path)
|
|
122
|
+
.select("subject_id", "time", "code")
|
|
123
|
+
.filter(pl.col("time").is_not_null())
|
|
124
|
+
.filter(_keeps_code(config))
|
|
125
|
+
.with_columns(pl.col("subject_id").cast(pl.Int64))
|
|
126
|
+
.join(train_subjects.lazy(), on="subject_id", how="semi")
|
|
127
|
+
.group_by("code")
|
|
128
|
+
.agg(pl.len().alias("count"))
|
|
129
|
+
.collect()
|
|
130
|
+
)
|
|
131
|
+
counts = shard if counts is None else pl.concat([counts, shard])
|
|
132
|
+
if counts is None or counts.is_empty():
|
|
133
|
+
raise ValueError(
|
|
134
|
+
"No timestamped events belong to the training split, so RETAIN has no vocabulary to learn. "
|
|
135
|
+
"Check that the MEDS dataset declares a 'train' split."
|
|
136
|
+
)
|
|
137
|
+
counts = counts.group_by("code").agg(pl.col("count").sum())
|
|
138
|
+
counts = counts.filter(pl.col("count") >= config.min_code_count)
|
|
139
|
+
# Deterministic order: frequency first, then the code string, so the same dataset always produces
|
|
140
|
+
# the same indices regardless of shard iteration order.
|
|
141
|
+
counts = counts.sort(["count", "code"], descending=[True, False])
|
|
142
|
+
if config.max_vocab_size is not None:
|
|
143
|
+
counts = counts.head(config.max_vocab_size)
|
|
144
|
+
if counts.is_empty():
|
|
145
|
+
raise ValueError(
|
|
146
|
+
f"min_code_count={config.min_code_count} removed every code from the training split"
|
|
147
|
+
)
|
|
148
|
+
return counts.with_row_index("code_index").select(
|
|
149
|
+
pl.col("code"), pl.col("code_index").cast(pl.UInt32), pl.col("count").cast(pl.Int64)
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _placement_agreement(
|
|
154
|
+
placement_frames: list[pl.DataFrame], splits: pl.DataFrame, meds_dir: Path
|
|
155
|
+
) -> dict[str, Any]:
|
|
156
|
+
"""Compare shard placement against the declared split map, when placement encodes splits at all.
|
|
157
|
+
|
|
158
|
+
Both sources exist on a normal MEDS extract and this port resolves membership from the map, so a
|
|
159
|
+
divergence would move subjects between cohorts here **and nowhere else** -- a sibling port resolves
|
|
160
|
+
from placement. Measured on the pilot extract: 0 disagreements over 364,627 subjects, reported by
|
|
161
|
+
the cluster session. That agreement is what made the precedence unobservable in the first place, so
|
|
162
|
+
it is checked per run rather than assumed to hold.
|
|
163
|
+
|
|
164
|
+
Only compared when the directory names are split names. `data/shard_00/` is a legal layout that
|
|
165
|
+
encodes no split, and comparing against it would refuse every such extract.
|
|
166
|
+
"""
|
|
167
|
+
if not placement_frames:
|
|
168
|
+
return {"placement_encodes_splits": False, "placement_disagreements": None}
|
|
169
|
+
placement = pl.concat(placement_frames).unique(subset=["subject_id"], keep="first")
|
|
170
|
+
declared = set(splits.get_column("split").unique().to_list())
|
|
171
|
+
observed = set(placement.get_column("placement").unique().to_list())
|
|
172
|
+
if not observed <= declared:
|
|
173
|
+
return {"placement_encodes_splits": False, "placement_disagreements": None}
|
|
174
|
+
|
|
175
|
+
joined = placement.join(splits, on="subject_id", how="inner")
|
|
176
|
+
disagreements = joined.filter(pl.col("placement") != pl.col("split"))
|
|
177
|
+
if disagreements.height:
|
|
178
|
+
sample = disagreements.head(3).rows()
|
|
179
|
+
raise ValueError(
|
|
180
|
+
f"{meds_dir} contradicts itself about {disagreements.height} subject(s): shard placement "
|
|
181
|
+
f"and {DECLARED_MAP_SOURCE} assign them to different splits, e.g. {sample} as "
|
|
182
|
+
"(subject_id, placement, declared). This port resolves membership from the declared map, "
|
|
183
|
+
"so it would train on a different cohort than a port resolving from placement, with no "
|
|
184
|
+
"error on either side. Reconcile the extract before comparing models across it."
|
|
185
|
+
)
|
|
186
|
+
return {"placement_encodes_splits": True, "placement_disagreements": 0}
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def preprocess(meds_dir: Path, staging: Path, config: RETAINConfig) -> dict[str, Any]:
|
|
190
|
+
"""Write the patients payload into `staging` and return its work units."""
|
|
191
|
+
meds_dir = Path(meds_dir)
|
|
192
|
+
files = data_files(meds_dir)
|
|
193
|
+
splits, split_source = read_split_map_with_source(meds_dir)
|
|
194
|
+
train_subjects = splits.filter(pl.col("split") == TRAIN_SPLIT).select("subject_id")
|
|
195
|
+
if train_subjects.is_empty():
|
|
196
|
+
raise ValueError(
|
|
197
|
+
f"{meds_dir} declares no '{TRAIN_SPLIT}' split; RETAIN fits its vocabulary and its "
|
|
198
|
+
"parameters on the training split only."
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
vocabulary = _build_vocabulary(files, train_subjects, config)
|
|
202
|
+
vocabulary.write_parquet(staging / "vocabulary.parquet")
|
|
203
|
+
splits.write_parquet(staging / "subject_splits.parquet")
|
|
204
|
+
lookup = vocabulary.select("code", "code_index")
|
|
205
|
+
|
|
206
|
+
sequences = staging / "sequences"
|
|
207
|
+
totals = {
|
|
208
|
+
"events": 0,
|
|
209
|
+
"static_events": 0,
|
|
210
|
+
"meds_standard_marker_events": 0,
|
|
211
|
+
"out_of_vocabulary_events": 0,
|
|
212
|
+
"same_instant_duplicate_events": 0,
|
|
213
|
+
"kept_events": 0,
|
|
214
|
+
}
|
|
215
|
+
subjects: set[int] = set()
|
|
216
|
+
event_groups = 0
|
|
217
|
+
placement_frames: list[pl.DataFrame] = []
|
|
218
|
+
for path in files:
|
|
219
|
+
parts = path.relative_to(meds_dir / "data").parts
|
|
220
|
+
frame = pl.read_parquet(path, columns=list(EVENT_COLUMNS)).with_columns(
|
|
221
|
+
pl.col("subject_id").cast(pl.Int64),
|
|
222
|
+
pl.col("time").cast(pl.Datetime("us")),
|
|
223
|
+
pl.col("code").cast(pl.Utf8),
|
|
224
|
+
)
|
|
225
|
+
totals["events"] += frame.height
|
|
226
|
+
if len(parts) > 1:
|
|
227
|
+
placement_frames.append(
|
|
228
|
+
frame.select(pl.col("subject_id").unique()).with_columns(
|
|
229
|
+
pl.lit(parts[0]).alias("placement")
|
|
230
|
+
)
|
|
231
|
+
)
|
|
232
|
+
timestamped = frame.filter(pl.col("time").is_not_null())
|
|
233
|
+
totals["static_events"] += frame.height - timestamped.height
|
|
234
|
+
clinical = timestamped.filter(_keeps_code(config))
|
|
235
|
+
totals["meds_standard_marker_events"] += timestamped.height - clinical.height
|
|
236
|
+
mapped = clinical.join(lookup, on="code", how="inner")
|
|
237
|
+
totals["out_of_vocabulary_events"] += clinical.height - mapped.height
|
|
238
|
+
# One row per (subject, timestamp, code): RETAIN's input is binary, so a code repeated at the
|
|
239
|
+
# same instant must contribute once, exactly as `xvec[subseq] = 1.` does upstream.
|
|
240
|
+
events = (
|
|
241
|
+
mapped.select("subject_id", "time", "code_index")
|
|
242
|
+
.unique(subset=["subject_id", "time", "code_index"])
|
|
243
|
+
.sort(["subject_id", "time", "code_index"])
|
|
244
|
+
)
|
|
245
|
+
# Every input row must land in exactly one reported bucket, or a reader who sees a tiny
|
|
246
|
+
# out_of_vocabulary_events will conclude that little was discarded and be wrong by however
|
|
247
|
+
# many same-instant duplicates the extract happens to contain -- a quantity that differs
|
|
248
|
+
# between datasets and was, until this counter, reported by nothing.
|
|
249
|
+
totals["same_instant_duplicate_events"] += mapped.height - events.height
|
|
250
|
+
totals["kept_events"] += events.height
|
|
251
|
+
subjects.update(events.get_column("subject_id").unique().to_list())
|
|
252
|
+
event_groups += events.select("subject_id", "time").unique().height
|
|
253
|
+
destination = sequences / path.relative_to(meds_dir / "data")
|
|
254
|
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
|
255
|
+
events.write_parquet(destination)
|
|
256
|
+
|
|
257
|
+
agreement = _placement_agreement(placement_frames, splits, meds_dir)
|
|
258
|
+
|
|
259
|
+
settings = {name: getattr(config, name) for name in VOCABULARY_SETTINGS}
|
|
260
|
+
(staging / SETTINGS_FILE).write_text(json.dumps(settings, indent=2, sort_keys=True))
|
|
261
|
+
return {
|
|
262
|
+
"subjects": len(subjects),
|
|
263
|
+
"subjects_in_split_map": splits.height,
|
|
264
|
+
"split_source": split_source,
|
|
265
|
+
**agreement,
|
|
266
|
+
"shards": len(files),
|
|
267
|
+
"vocabulary_size": vocabulary.height,
|
|
268
|
+
"event_groups": event_groups,
|
|
269
|
+
**totals,
|
|
270
|
+
}
|
retain_meds/_training.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
"""Batching, the optimisation loop, and scoring.
|
|
2
|
+
|
|
3
|
+
The loop follows `train_RETAIN` in `mp2893/retain` @ 9fd39c46: length-sorted minibatches whose *order* is
|
|
4
|
+
reshuffled every epoch, Adadelta by default, the four explicit L2 penalties added to a binary cross-entropy
|
|
5
|
+
guarded by `log_eps`, and the parameters of the epoch with the best validation AUC.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import copy
|
|
11
|
+
import random
|
|
12
|
+
import sys
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import numpy as np
|
|
17
|
+
import torch
|
|
18
|
+
|
|
19
|
+
from .config import RETAINConfig
|
|
20
|
+
from ._model import Retain
|
|
21
|
+
from ._windows import SequenceBatch
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def resolve_device(selector: str) -> torch.device:
|
|
25
|
+
if selector == "auto":
|
|
26
|
+
return torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
|
27
|
+
return torch.device(selector)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def gpu_metrics(device: torch.device) -> dict[str, Any] | None:
|
|
31
|
+
"""SPEC section 6's GPU block, or `None` when no accelerator was used."""
|
|
32
|
+
if device.type != "cuda" or not torch.cuda.is_available():
|
|
33
|
+
return None
|
|
34
|
+
return {
|
|
35
|
+
"peak_allocated_bytes": int(torch.cuda.max_memory_allocated()),
|
|
36
|
+
"peak_reserved_bytes": int(torch.cuda.max_memory_reserved()),
|
|
37
|
+
"source": "torch.cuda.max_memory_allocated",
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def roc_auc(labels: np.ndarray, scores: np.ndarray) -> float | None:
|
|
42
|
+
"""Rank-based ROC AUC with tie correction. `None` when only one class is present.
|
|
43
|
+
|
|
44
|
+
Equivalent to `sklearn.metrics.roc_auc_score`, which the official code calls, without taking on
|
|
45
|
+
scikit-learn as a runtime dependency for a single function.
|
|
46
|
+
"""
|
|
47
|
+
labels = np.asarray(labels).astype(bool)
|
|
48
|
+
scores = np.asarray(scores, dtype=np.float64)
|
|
49
|
+
n_positive = int(labels.sum())
|
|
50
|
+
n_negative = int(labels.size - n_positive)
|
|
51
|
+
if n_positive == 0 or n_negative == 0:
|
|
52
|
+
return None
|
|
53
|
+
order = np.argsort(scores, kind="mergesort")
|
|
54
|
+
ranks = np.empty(scores.size, dtype=np.float64)
|
|
55
|
+
ranks[order] = np.arange(1, scores.size + 1, dtype=np.float64)
|
|
56
|
+
ordered = scores[order]
|
|
57
|
+
start = 0
|
|
58
|
+
while start < ordered.size:
|
|
59
|
+
stop = start + 1
|
|
60
|
+
while stop < ordered.size and ordered[stop] == ordered[start]:
|
|
61
|
+
stop += 1
|
|
62
|
+
if stop - start > 1:
|
|
63
|
+
ranks[order[start:stop]] = (start + 1 + stop) / 2.0
|
|
64
|
+
start = stop
|
|
65
|
+
return float((ranks[labels].sum() - n_positive * (n_positive + 1) / 2.0) / (n_positive * n_negative))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@dataclass
|
|
69
|
+
class TensorBatch:
|
|
70
|
+
codes: torch.Tensor
|
|
71
|
+
offsets: torch.Tensor
|
|
72
|
+
slots: torch.Tensor
|
|
73
|
+
lengths: torch.Tensor
|
|
74
|
+
gaps: torch.Tensor | None
|
|
75
|
+
n_rows: int
|
|
76
|
+
n_visits: int
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def collate(batch: SequenceBatch, rows: np.ndarray, config: RETAINConfig, device: torch.device) -> TensorBatch:
|
|
80
|
+
"""Build the `EmbeddingBag` view of `rows` and the padded grid they scatter into.
|
|
81
|
+
|
|
82
|
+
A row with no history keeps a length of one so the attention softmax has a position to normalise
|
|
83
|
+
over. Its single visit carries no bag, so `v = 0`, `c = 0`, and the score is the model's learned
|
|
84
|
+
intercept. That is what lets every requested key be scored instead of excluded.
|
|
85
|
+
"""
|
|
86
|
+
rows = np.asarray(rows, dtype=np.int64)
|
|
87
|
+
lengths = batch.lengths[rows]
|
|
88
|
+
n_visits = int(max(lengths.max(initial=0), 1))
|
|
89
|
+
gaps_source = batch.visit_gap_days() if config.use_time else None
|
|
90
|
+
|
|
91
|
+
code_chunks: list[np.ndarray] = []
|
|
92
|
+
width_chunks: list[np.ndarray] = []
|
|
93
|
+
slot_chunks: list[np.ndarray] = []
|
|
94
|
+
gap_chunks: list[np.ndarray] = []
|
|
95
|
+
for position, row in enumerate(rows.tolist()):
|
|
96
|
+
start, stop = int(batch.row_ptr[row]), int(batch.row_ptr[row + 1])
|
|
97
|
+
if stop == start:
|
|
98
|
+
continue
|
|
99
|
+
code_chunks.append(batch.codes[int(batch.visit_ptr[start]) : int(batch.visit_ptr[stop])])
|
|
100
|
+
width_chunks.append(batch.visit_ptr[start + 1 : stop + 1] - batch.visit_ptr[start:stop])
|
|
101
|
+
slot_chunks.append(position * n_visits + np.arange(stop - start, dtype=np.int64))
|
|
102
|
+
if gaps_source is not None:
|
|
103
|
+
gap_chunks.append(gaps_source[start:stop])
|
|
104
|
+
|
|
105
|
+
if code_chunks:
|
|
106
|
+
codes = np.concatenate(code_chunks).astype(np.int64)
|
|
107
|
+
widths = np.concatenate(width_chunks).astype(np.int64)
|
|
108
|
+
offsets = np.zeros(widths.size + 1, dtype=np.int64)
|
|
109
|
+
np.cumsum(widths, out=offsets[1:])
|
|
110
|
+
slots = np.concatenate(slot_chunks)
|
|
111
|
+
else:
|
|
112
|
+
codes = np.zeros(0, dtype=np.int64)
|
|
113
|
+
offsets = np.zeros(1, dtype=np.int64)
|
|
114
|
+
slots = np.zeros(0, dtype=np.int64)
|
|
115
|
+
|
|
116
|
+
gaps = None
|
|
117
|
+
if gaps_source is not None:
|
|
118
|
+
stacked = np.concatenate(gap_chunks) if gap_chunks else np.zeros(0, dtype=np.float32)
|
|
119
|
+
gaps = torch.from_numpy(np.ascontiguousarray(stacked)).to(device)
|
|
120
|
+
return TensorBatch(
|
|
121
|
+
codes=torch.from_numpy(codes).to(device),
|
|
122
|
+
offsets=torch.from_numpy(offsets).to(device),
|
|
123
|
+
slots=torch.from_numpy(slots).to(device),
|
|
124
|
+
lengths=torch.from_numpy(np.maximum(lengths, 1)),
|
|
125
|
+
gaps=gaps,
|
|
126
|
+
n_rows=int(rows.size),
|
|
127
|
+
n_visits=n_visits,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _cross_entropy(logits: torch.Tensor, targets: torch.Tensor, log_eps: float) -> torch.Tensor:
|
|
132
|
+
"""`-(y log(y_hat + eps) + (1-y) log(1 - y_hat + eps))`, exactly as in `build_model`.
|
|
133
|
+
|
|
134
|
+
Written out rather than delegated to `binary_cross_entropy_with_logits` because upstream adds
|
|
135
|
+
`log_eps` inside both logarithms, which changes the loss surface near saturation.
|
|
136
|
+
"""
|
|
137
|
+
probabilities = torch.sigmoid(logits)
|
|
138
|
+
return -(
|
|
139
|
+
targets * torch.log(probabilities + log_eps)
|
|
140
|
+
+ (1.0 - targets) * torch.log(1.0 - probabilities + log_eps)
|
|
141
|
+
).mean()
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _optimizer(model: Retain, config: RETAINConfig) -> torch.optim.Optimizer:
|
|
145
|
+
parameters = model.trainable_parameters()
|
|
146
|
+
if config.solver == "adam":
|
|
147
|
+
# `adam()` upstream parameterises b1/b2 as 1 - beta, so b1=0.1, b2=0.001 is the usual
|
|
148
|
+
# (0.9, 0.999), and its fix1/fix2 terms are standard bias correction.
|
|
149
|
+
return torch.optim.Adam(parameters, lr=0.0002, betas=(0.9, 0.999), eps=1e-8)
|
|
150
|
+
# The Theano `adadelta()` block is rho=0.95 with eps=1e-6 and an implicit unit learning rate.
|
|
151
|
+
return torch.optim.Adadelta(parameters, lr=1.0, rho=0.95, eps=1e-6)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass
|
|
155
|
+
class TrainingHistory:
|
|
156
|
+
"""A commit-safe record of how the fit actually went."""
|
|
157
|
+
|
|
158
|
+
epochs_run: int = 0
|
|
159
|
+
best_epoch: int = 0
|
|
160
|
+
best_validation_auc: float | None = None
|
|
161
|
+
train_losses: list[float] = field(default_factory=list)
|
|
162
|
+
validation_aucs: list[float | None] = field(default_factory=list)
|
|
163
|
+
n_train_rows: int = 0
|
|
164
|
+
n_validation_rows: int = 0
|
|
165
|
+
n_positive_train_rows: int = 0
|
|
166
|
+
n_positive_validation_rows: int = 0
|
|
167
|
+
n_empty_train_sequences: int = 0
|
|
168
|
+
n_empty_validation_sequences: int = 0
|
|
169
|
+
optimizer_steps: int = 0
|
|
170
|
+
selected_by: str = "validation_auc"
|
|
171
|
+
|
|
172
|
+
def summary(self) -> dict[str, Any]:
|
|
173
|
+
return {
|
|
174
|
+
"epochs_run": self.epochs_run,
|
|
175
|
+
"best_epoch": self.best_epoch,
|
|
176
|
+
"best_validation_auc": self.best_validation_auc,
|
|
177
|
+
"train_losses": [round(value, 6) for value in self.train_losses],
|
|
178
|
+
"validation_aucs": [None if v is None else round(v, 6) for v in self.validation_aucs],
|
|
179
|
+
"n_train_rows": self.n_train_rows,
|
|
180
|
+
"n_validation_rows": self.n_validation_rows,
|
|
181
|
+
"n_positive_train_rows": self.n_positive_train_rows,
|
|
182
|
+
"n_positive_validation_rows": self.n_positive_validation_rows,
|
|
183
|
+
"n_empty_train_sequences": self.n_empty_train_sequences,
|
|
184
|
+
"n_empty_validation_sequences": self.n_empty_validation_sequences,
|
|
185
|
+
"optimizer_steps": self.optimizer_steps,
|
|
186
|
+
"selected_by": self.selected_by,
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _length_sorted(batch: SequenceBatch) -> np.ndarray:
|
|
191
|
+
"""`len_argsort`: order rows by sequence length so a minibatch pads as little as possible.
|
|
192
|
+
|
|
193
|
+
Ties break on `(subject_id, prediction_time)` rather than on row position. Upstream's `len_argsort`
|
|
194
|
+
is a stable sort on a list whose order is fixed by a pickled file, so position is a stable tiebreak
|
|
195
|
+
there; here the rows come from a task index whose row order carries no meaning and is not guaranteed
|
|
196
|
+
to be reproduced. Leaving the tie on position would let two byte-different but content-identical label
|
|
197
|
+
files produce different minibatches, and so a different fit, at the same seed.
|
|
198
|
+
"""
|
|
199
|
+
return np.lexsort((batch.keys[:, 1], batch.keys[:, 0], batch.lengths))
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def score(
|
|
203
|
+
model: Retain, batch: SequenceBatch, config: RETAINConfig, device: torch.device
|
|
204
|
+
) -> np.ndarray:
|
|
205
|
+
"""Probabilities for every row of `batch`, in `batch`'s own row order."""
|
|
206
|
+
model.eval()
|
|
207
|
+
scores = np.zeros(batch.n_keys, dtype=np.float64)
|
|
208
|
+
order = _length_sorted(batch)
|
|
209
|
+
with torch.no_grad():
|
|
210
|
+
for start in range(0, order.size, config.batch_size):
|
|
211
|
+
rows = order[start : start + config.batch_size]
|
|
212
|
+
tensors = collate(batch, rows, config, device)
|
|
213
|
+
output = model(
|
|
214
|
+
tensors.codes,
|
|
215
|
+
tensors.offsets,
|
|
216
|
+
tensors.slots,
|
|
217
|
+
tensors.lengths,
|
|
218
|
+
tensors.gaps,
|
|
219
|
+
n_rows=tensors.n_rows,
|
|
220
|
+
n_visits=tensors.n_visits,
|
|
221
|
+
)
|
|
222
|
+
scores[rows] = torch.sigmoid(output.logits).detach().cpu().numpy().astype(np.float64)
|
|
223
|
+
return scores
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def train(
|
|
227
|
+
*,
|
|
228
|
+
vocabulary_size: int,
|
|
229
|
+
train_batch: SequenceBatch,
|
|
230
|
+
train_labels: np.ndarray,
|
|
231
|
+
validation_batch: SequenceBatch,
|
|
232
|
+
validation_labels: np.ndarray,
|
|
233
|
+
config: RETAINConfig,
|
|
234
|
+
device: torch.device,
|
|
235
|
+
seed: int,
|
|
236
|
+
) -> tuple[Retain, TrainingHistory]:
|
|
237
|
+
torch.manual_seed(seed)
|
|
238
|
+
np.random.seed(seed)
|
|
239
|
+
shuffler = random.Random(seed)
|
|
240
|
+
|
|
241
|
+
model = Retain(vocabulary_size, config).to(device)
|
|
242
|
+
optimizer = _optimizer(model, config)
|
|
243
|
+
targets = torch.from_numpy(train_labels.astype(np.float32)).to(device)
|
|
244
|
+
|
|
245
|
+
order = _length_sorted(train_batch)
|
|
246
|
+
n_batches = max(1, int(np.ceil(order.size / config.batch_size)))
|
|
247
|
+
history = TrainingHistory(
|
|
248
|
+
n_train_rows=train_batch.n_keys,
|
|
249
|
+
n_validation_rows=validation_batch.n_keys,
|
|
250
|
+
n_positive_train_rows=int(train_labels.sum()),
|
|
251
|
+
n_positive_validation_rows=int(validation_labels.sum()),
|
|
252
|
+
n_empty_train_sequences=train_batch.n_empty,
|
|
253
|
+
n_empty_validation_sequences=validation_batch.n_empty,
|
|
254
|
+
)
|
|
255
|
+
best_state = copy.deepcopy(model.state_dict())
|
|
256
|
+
|
|
257
|
+
for epoch in range(config.n_epochs):
|
|
258
|
+
model.train()
|
|
259
|
+
losses: list[float] = []
|
|
260
|
+
for index in shuffler.sample(range(n_batches), n_batches):
|
|
261
|
+
rows = order[index * config.batch_size : (index + 1) * config.batch_size]
|
|
262
|
+
if not rows.size:
|
|
263
|
+
continue
|
|
264
|
+
tensors = collate(train_batch, rows, config, device)
|
|
265
|
+
output = model(
|
|
266
|
+
tensors.codes,
|
|
267
|
+
tensors.offsets,
|
|
268
|
+
tensors.slots,
|
|
269
|
+
tensors.lengths,
|
|
270
|
+
tensors.gaps,
|
|
271
|
+
n_rows=tensors.n_rows,
|
|
272
|
+
n_visits=tensors.n_visits,
|
|
273
|
+
)
|
|
274
|
+
loss = _cross_entropy(output.logits, targets[rows], config.log_eps)
|
|
275
|
+
cost = loss + model.regularization()
|
|
276
|
+
optimizer.zero_grad(set_to_none=True)
|
|
277
|
+
cost.backward()
|
|
278
|
+
optimizer.step()
|
|
279
|
+
history.optimizer_steps += 1
|
|
280
|
+
losses.append(float(loss.detach().cpu()))
|
|
281
|
+
history.epochs_run = epoch + 1
|
|
282
|
+
history.train_losses.append(float(np.mean(losses)) if losses else float("nan"))
|
|
283
|
+
|
|
284
|
+
auc = None
|
|
285
|
+
if validation_batch.n_keys:
|
|
286
|
+
auc = roc_auc(validation_labels, score(model, validation_batch, config, device))
|
|
287
|
+
history.validation_aucs.append(auc)
|
|
288
|
+
improved = auc is not None and (
|
|
289
|
+
history.best_validation_auc is None or auc > history.best_validation_auc
|
|
290
|
+
)
|
|
291
|
+
if improved:
|
|
292
|
+
history.best_validation_auc = auc
|
|
293
|
+
history.best_epoch = epoch
|
|
294
|
+
best_state = copy.deepcopy(model.state_dict())
|
|
295
|
+
|
|
296
|
+
# Emitted every epoch, unconditionally, to stderr. A full-data fit runs for hours and this
|
|
297
|
+
# command writes its artifact and manifest only at the very end, so without this line a healthy
|
|
298
|
+
# run and a hung one are indistinguishable from outside and there is nothing on disk to inspect
|
|
299
|
+
# mid-run. `epochs_since_best` is the quantity that says whether the fit is still going anywhere.
|
|
300
|
+
print(
|
|
301
|
+
f"[retain] epoch {epoch + 1}/{config.n_epochs} "
|
|
302
|
+
f"train_loss={history.train_losses[-1]:.6f} "
|
|
303
|
+
f"validation_auc={'n/a' if auc is None else f'{auc:.6f}'} "
|
|
304
|
+
f"best_epoch={history.best_epoch + 1} "
|
|
305
|
+
f"epochs_since_best={epoch - history.best_epoch} "
|
|
306
|
+
f"steps={history.optimizer_steps}"
|
|
307
|
+
f"{' *' if improved else ''}",
|
|
308
|
+
file=sys.stderr,
|
|
309
|
+
flush=True,
|
|
310
|
+
)
|
|
311
|
+
|
|
312
|
+
if history.best_validation_auc is None:
|
|
313
|
+
# No usable validation signal: either no tuning rows, or one class only. Upstream would have
|
|
314
|
+
# saved nothing at all here; keeping the final epoch is the closest usable behaviour and is
|
|
315
|
+
# recorded so the report cannot claim a selection that never happened.
|
|
316
|
+
history.selected_by = "final_epoch"
|
|
317
|
+
history.best_epoch = history.epochs_run - 1
|
|
318
|
+
else:
|
|
319
|
+
model.load_state_dict(best_state)
|
|
320
|
+
return model, history
|