meds-algorithm-retain 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- meds_algorithm_retain-0.1.0.dist-info/METADATA +218 -0
- meds_algorithm_retain-0.1.0.dist-info/RECORD +16 -0
- meds_algorithm_retain-0.1.0.dist-info/WHEEL +4 -0
- meds_algorithm_retain-0.1.0.dist-info/entry_points.txt +2 -0
- meds_algorithm_retain-0.1.0.dist-info/licenses/LICENSE +21 -0
- retain_meds/__init__.py +6 -0
- retain_meds/__main__.py +65 -0
- retain_meds/_interpret.py +102 -0
- retain_meds/_model.py +171 -0
- retain_meds/_preprocess.py +270 -0
- retain_meds/_training.py +320 -0
- retain_meds/_windows.py +221 -0
- retain_meds/config.py +114 -0
- retain_meds/resources/algorithms/supervised/model.yaml +24 -0
- retain_meds/resources/runs/run.yaml +25 -0
- retain_meds/retain_meds.py +252 -0
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: meds-algorithm-retain
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: RETAIN (Choi et al., NeurIPS 2016) reverse-time attention model for MEDS, a supervised classifier behind the MEDS algorithm contract.
|
|
5
|
+
Project-URL: Repository, https://github.com/florian6973/meds-algorithm-retain
|
|
6
|
+
Author-email: Florent Pollet <flo1.raymond@gmail.com>
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
License-File: LICENSE
|
|
9
|
+
Requires-Python: >=3.11
|
|
10
|
+
Requires-Dist: meds-algorithm[data]==0.4.0a5
|
|
11
|
+
Requires-Dist: numpy>=1.26
|
|
12
|
+
Requires-Dist: pyarrow>=15
|
|
13
|
+
Requires-Dist: pyyaml>=6
|
|
14
|
+
Requires-Dist: torch>=2.2
|
|
15
|
+
Provides-Extra: orchestration
|
|
16
|
+
Requires-Dist: meds-algorithm[orchestration]==0.4.0a5; extra == 'orchestration'
|
|
17
|
+
Provides-Extra: test
|
|
18
|
+
Requires-Dist: meds-algorithm[conformance]==0.4.0a5; extra == 'test'
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
|
|
21
|
+
# RETAIN
|
|
22
|
+
|
|
23
|
+
RETAIN ([Choi et al., NeurIPS 2016](https://proceedings.neurips.cc/paper/2016/hash/231141b34c82aa95e48810a9d1b33a79-Abstract.html))
|
|
24
|
+
for datasets in the [Medical Event Data Standard](https://github.com/Medical-Event-Data-Standard/meds): a
|
|
25
|
+
reverse-time attention model that predicts a binary outcome from a subject's sequence of visits and
|
|
26
|
+
reports which codes drove each prediction.
|
|
27
|
+
|
|
28
|
+
- **What it runs:** `preprocess_data` (code vocabulary fitted on the training split), `train_supervised`
|
|
29
|
+
(the whole model, for one task) and `predict_supervised` (one probability per requested key).
|
|
30
|
+
- **Model:** a PyTorch reimplementation of the official Theano code
|
|
31
|
+
([`mp2893/retain`](https://github.com/mp2893/retain) @ `9fd39c46`), with the official defaults, checked
|
|
32
|
+
against a NumPy transcription of the official equations.
|
|
33
|
+
- **MEDS adaptation:** a visit is the set of codes in one time bucket (`visit_bucket_minutes`, default one
|
|
34
|
+
day), and each history is cut at its prediction time before visits are formed. No dataset predicates
|
|
35
|
+
are needed.
|
|
36
|
+
- **Status:** [IMPLEMENTATION_REPORT.md](IMPLEMENTATION_REPORT.md) lists what was verified, including the
|
|
37
|
+
temporal-leakage checks, and what was not.
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
pip install meds-algorithm-retain
|
|
41
|
+
retain-meds preprocess_data external_meds_dir=$MEDS_DIR output_data_dir=$RUN/data
|
|
42
|
+
retain-meds train_supervised input_data_dir=$RUN/data external_labels_dir=$LABELS_DIR \
|
|
43
|
+
output_supervised_algorithm_dir=$RUN/algorithm
|
|
44
|
+
retain-meds predict_supervised input_data_dir=$RUN/data input_supervised_algorithm_dir=$RUN/algorithm \
|
|
45
|
+
external_index_dir=$LABELS_DIR splits=[held_out] output_predictions_dir=$RUN/predictions
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Settings are the fields of `RETAINConfig` (`src/retain_meds/config.py`); pass a YAML or JSON file of
|
|
49
|
+
overrides to every step with `config_file=<path>`.
|
|
50
|
+
|
|
51
|
+
## Python API and named command
|
|
52
|
+
|
|
53
|
+
The implementation is `src/retain_meds/retain_meds.py`, with configuration in `config.py`.
|
|
54
|
+
Both concrete classes are public package exports:
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from retain_meds import RETAIN, RETAINConfig
|
|
58
|
+
|
|
59
|
+
algorithm = RETAIN(RETAINConfig())
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Run typed requests through `meds_algorithm.StepRunner.run(...)` so validation, measurements and
|
|
63
|
+
publication apply. Calling an implementation method directly bypasses that execution contract.
|
|
64
|
+
After installation, the algorithm's own `retain-meds` command runs the CLI:
|
|
65
|
+
|
|
66
|
+
```bash
|
|
67
|
+
retain-meds commands --profile supervised
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
RETAIN (Choi et al., NeurIPS 2016) reverse-time attention model for MEDS, a supervised classifier behind the MEDS algorithm contract.
|
|
71
|
+
|
|
72
|
+
This repository declares the **`supervised`** profiles under version 4 of the minimal MEDS algorithm
|
|
73
|
+
contract. The contract standardizes MEDS input, command meanings and arguments, artifact envelopes,
|
|
74
|
+
final predictions, measurements, MEDS-DEV packaging, and operational interfaces. The algorithm owns every
|
|
75
|
+
computational choice and every intermediate payload.
|
|
76
|
+
|
|
77
|
+
## Algorithm profile commands
|
|
78
|
+
|
|
79
|
+
- `supervised`: `preprocess_data → train_supervised → predict_supervised`
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
retain-meds profiles
|
|
83
|
+
retain-meds commands --profile supervised
|
|
84
|
+
retain-meds <step> --profile <profile> key=value ...
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Each profile has `algorithms/<profile>/model.yaml`. Multiple profiles share one Python package and one
|
|
88
|
+
registry; select one explicitly for each run. `STUB_PROFILES` tracks implementation status separately.
|
|
89
|
+
Read [Software contract](SPEC.md) for exact class responsibilities and step inputs.
|
|
90
|
+
|
|
91
|
+
Intermediate artifacts may be tensors, files, databases, external-system references, or any other
|
|
92
|
+
implementation-owned representation. Generic tests inspect only their `manifest.yaml` envelopes. They
|
|
93
|
+
must not assume batches, Lightning, PyTorch, Parquet intermediate data, or iteration order.
|
|
94
|
+
|
|
95
|
+
## Profile runner (prototype)
|
|
96
|
+
|
|
97
|
+
Install `uv sync --extra orchestration --extra test`, then fill the input paths and new output directory
|
|
98
|
+
in `configs/runs/run.yaml`. Algorithm settings belong under `algorithm`; core validates them using
|
|
99
|
+
this repository's concrete Config class.
|
|
100
|
+
|
|
101
|
+
```bash
|
|
102
|
+
uv run retain-meds run --dry-run
|
|
103
|
+
uv run retain-meds run execution=local
|
|
104
|
+
# Use a NEW output_dir and configure your cluster's resources before submission:
|
|
105
|
+
uv run retain-meds run execution=slurm output_dir=/runs/new-cluster-run
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
A dry run prints all resolved arguments, paths, configuration and resources without writing files or
|
|
109
|
+
submitting jobs. Local execution uses fresh subprocesses; Submitit submits separate dependent SLURM
|
|
110
|
+
jobs. See the [shared runner guide](https://meds-algorithm.readthedocs.io/en/latest/running/#profile-runner-prototype)
|
|
111
|
+
for stage subsets, artifact reuse, resource configuration and prototype limitations. The existing
|
|
112
|
+
per-step CLI and shell launcher remain available for comparison. This prototype does not implement
|
|
113
|
+
Hydra multirun, multi-node training or checkpoint/resume.
|
|
114
|
+
|
|
115
|
+
## Publishing this algorithm
|
|
116
|
+
|
|
117
|
+
Follow [the release guide](RELEASING.md) for package metadata, one-time PyPI Trusted Publisher setup,
|
|
118
|
+
installed-wheel checks, and version-tag publication. All profiles ship in the `meds-algorithm-retain`
|
|
119
|
+
distribution. Tests run in CI; publishing is an explicit workflow dispatch after implementation.
|
|
120
|
+
|
|
121
|
+
## Inputs and predicates
|
|
122
|
+
|
|
123
|
+
- `external_meds_dir` is the standard MEDS dataset boundary.
|
|
124
|
+
- `external_index_dir` supplies only `(subject_id, prediction_time)` to inference and scoring.
|
|
125
|
+
- `external_labels_dir` supplies permitted training targets or explicit in-context examples.
|
|
126
|
+
- `external_evaluation_labels_dir`, when supplied, is used only to attach truth after scores are fixed.
|
|
127
|
+
- `external_predicates_file` supplies dataset-specific predicates when useful. For MIMIC-IV, reuse the
|
|
128
|
+
predicates from MEDS-DEV. This repository decides whether and how to interpret them.
|
|
129
|
+
|
|
130
|
+
A command that consumes predicates records their absolute path and SHA-256 digest in its manifest.
|
|
131
|
+
Predicates do not imply a standard feature representation.
|
|
132
|
+
|
|
133
|
+
## Implementing the algorithm
|
|
134
|
+
|
|
135
|
+
Read [AGENTS.md](AGENTS.md), [SPEC.md](SPEC.md), and
|
|
136
|
+
[docs/IMPLEMENTATION_GUIDE.md](docs/IMPLEMENTATION_GUIDE.md) completely before changing implementation
|
|
137
|
+
files. The implementation source recorded when this repository was generated is:
|
|
138
|
+
|
|
139
|
+
```text
|
|
140
|
+
https://proceedings.neurips.cc/paper/2016/hash/231141b34c82aa95e48810a9d1b33a79-Abstract.html and https://github.com/mp2893/retain@9fd39c46e44b3fdae3a667a47fbdd5ffbd87bff5
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
The algorithm-owned surface is `src/retain_meds/retain_meds.py` (the typed algorithm), `src/retain_meds/config.py`
|
|
144
|
+
(its immutable configuration), `algorithms/<profile>/model.yaml`, and
|
|
145
|
+
`tests/e2e.py`. Implement the selected profile methods, define exact intermediate compatibility checks in
|
|
146
|
+
`validate_inputs`,
|
|
147
|
+
finalize MEDS-DEV command strings, and complete the report. Remove a profile from `STUB_PROFILES` only after it runs end to end. Complete every declared profile.
|
|
148
|
+
|
|
149
|
+
For each generated test and its purpose, read [the test guide](docs/TESTING.md).
|
|
150
|
+
|
|
151
|
+
## Configuration and execution ownership
|
|
152
|
+
|
|
153
|
+
Declare typed fields/defaults in `RETAINConfig`, with `validate()` for constraints. Pass
|
|
154
|
+
`config_file=/path/config.yaml` to a step for a JSON/YAML values mapping; unknown fields fail. Core saves
|
|
155
|
+
the complete resolved config and original typed request as `config.json` and `request.json` in each
|
|
156
|
+
published artifact. These JSON envelopes can be reloaded with the corresponding explicit classes.
|
|
157
|
+
|
|
158
|
+
Algorithm methods receive typed requests, write payloads into `request.output.directory` (staging), and
|
|
159
|
+
return `PayloadMetadata`. `create_repository().run(...)` constructs a fresh algorithm and delegates to
|
|
160
|
+
core's runner. It owns measurements, prediction validation, optional post-score truth attachment,
|
|
161
|
+
metadata and publication. Never publish from inside algorithm methods or write the reserved metadata files.
|
|
162
|
+
Reopen fitted artifacts on each step. Test config/request round trips and execution in a fresh process.
|
|
163
|
+
|
|
164
|
+
## Verification
|
|
165
|
+
|
|
166
|
+
```bash
|
|
167
|
+
uv run pytest -rs
|
|
168
|
+
uv run pytest -m meds_dev -rs
|
|
169
|
+
uv run pytest -m real_data -rs
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
- The default suite runs the local end-to-end adapter and contract tests.
|
|
173
|
+
- `meds_dev` registers `algorithms/<profile>/model.yaml`, installs this repository from `requirements.txt` in an isolated
|
|
174
|
+
environment, and executes MEDS-DEV's real command slots.
|
|
175
|
+
- `real_data` runs the full selected algorithm profile on the MEDS-DEV MIMIC-IV demo, the pinned
|
|
176
|
+
`mortality/in_icu/first_24h` task, and MEDS-DEV's
|
|
177
|
+
MIMIC predicates. Set `MEDS_DEMO_DIR` to reuse an existing demo and `MEDS_DEV_DIR` to reuse a checkout.
|
|
178
|
+
|
|
179
|
+
Automatic provisioning pins MEDS-DEV PR #330's tested commit, which has the `{predicates_path}`
|
|
180
|
+
passthrough (PR #325). Set `MEDS_DEV_REF` to use another
|
|
181
|
+
revision containing those capabilities. A checkout supplied through `MEDS_DEV_DIR` is checked and fails
|
|
182
|
+
clearly if it cannot populate them.
|
|
183
|
+
|
|
184
|
+
Both integration tiers explicitly `algorithm_stub`-skip until implementation is complete. A completed port
|
|
185
|
+
must have zero `algorithm_stub` skips and must account for every other skip using `-rs` output.
|
|
186
|
+
|
|
187
|
+
These commands are gates, not an unordered checklist. Follow the canonical staged workflow in
|
|
188
|
+
[docs/IMPLEMENTATION_GUIDE.md](docs/IMPLEMENTATION_GUIDE.md): local, MEDS-DEV, pre-demo report and human
|
|
189
|
+
review, MIMIC-IV demo, demo report and human review, authorized full-data run, then final report and GitHub
|
|
190
|
+
synchronization. Full-data cluster jobs run only from a clean, pushed commit and use the reviewed shared
|
|
191
|
+
task bundle for the selected dataset (for example, MIMIC-IV or NWICU in MEDS format), built upstream from
|
|
192
|
+
clinical ACES tasks or randomly sampled code-by-horizon tasks. The provided `real_data` test remains the MIMIC-IV demo regardless of that choice.
|
|
193
|
+
|
|
194
|
+
Final predictions must conform to MEDS evaluation, contain unique `(subject_id, prediction_time)` keys,
|
|
195
|
+
and exactly cover the requested evaluation index. Tests compare the actual key sets; self-reported manifest
|
|
196
|
+
counts are not accepted as evidence. Row order is never an association mechanism.
|
|
197
|
+
|
|
198
|
+
## Measurements
|
|
199
|
+
|
|
200
|
+
Every command manifest records monotonic wall time and sampled live process-tree memory. On Linux, peak
|
|
201
|
+
PSS from `/proc/*/smaps_rollup` is the primary comparison metric; aggregate RSS is retained as a
|
|
202
|
+
diagnostic. The measurement method and sampling interval are explicit. GPU measurements and SLURM
|
|
203
|
+
`MaxRSS`, when available, are separate metrics. Missing measurements are `null`; observed zero is `0`.
|
|
204
|
+
Time and memory are evidence, not portable test thresholds.
|
|
205
|
+
|
|
206
|
+
## Cluster execution and GitHub communication
|
|
207
|
+
|
|
208
|
+
```bash
|
|
209
|
+
$EDITOR slurm/config.sh
|
|
210
|
+
PROFILE=supervised ./slurm/submit.sh --dry-run
|
|
211
|
+
PROFILE=supervised ./slurm/submit.sh
|
|
212
|
+
scripts/github-sync.sh <run-id>
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
SLURM follows the selected algorithm profile and chains stages by successful dependencies. Structured evidence belongs
|
|
216
|
+
under `runs/<run-id>/`; raw logs, datasets, checkpoints, and credentials do not. GitHub synchronization
|
|
217
|
+
pushes a `runs/retain_meds/<run-id>` branch and optionally opens a draft PR. A failed push leaves the
|
|
218
|
+
result committed and recoverable locally.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
retain_meds/__init__.py,sha256=idlpW2U0iIuzFxMCuvdBbU4K865J04FPwG1RT0uzfhE,118
|
|
2
|
+
retain_meds/__main__.py,sha256=tuyYwTtKxYNAzcCM5EohAmGmESs2IlWcMB7oGcC_Cpw,2759
|
|
3
|
+
retain_meds/_interpret.py,sha256=2vkz-PgxNsnytm_IxJ4Y-cxy4gOebyy-G8ByRrrF6RU,4382
|
|
4
|
+
retain_meds/_model.py,sha256=DFnfwbcBZhUhgxxMRAdszx8zF8g3cYIcQzfZ8xCg9ms,8493
|
|
5
|
+
retain_meds/_preprocess.py,sha256=4uctV_fFii6YGxD2hJPHhQQy19pXkEhzinDp3KI9XXE,13166
|
|
6
|
+
retain_meds/_training.py,sha256=GVFM1yZK6egyvmxqZJlnaEJRF0kbHnEsYQIlwTfJe3g,13315
|
|
7
|
+
retain_meds/_windows.py,sha256=Ft7qrNrFYp2koi7ExVMbpYpzY1TELuWKPuWFWKIbIPk,10389
|
|
8
|
+
retain_meds/config.py,sha256=yq7GuCykazsR0JM1Ns3FV3eEuXXSh2a7C71WgYWqsIQ,6631
|
|
9
|
+
retain_meds/retain_meds.py,sha256=5x1Lb6UchIyIxbB40R0aZZjLnnCvE0eaGLzNhh85hrw,12103
|
|
10
|
+
retain_meds/resources/algorithms/supervised/model.yaml,sha256=Z7j-vExQcZVXi2wU-UffwUTlfBC9ZeRieNfp4rkC4tQ,1412
|
|
11
|
+
retain_meds/resources/runs/run.yaml,sha256=MTNs4ZNsEZwAw5yUODq9dev0xk5vhE1ANyMJ6EZk6I4,930
|
|
12
|
+
meds_algorithm_retain-0.1.0.dist-info/METADATA,sha256=jcC6I0laBFhVymNoSPYV7D3g1HMVFcFUUYO7M82NhXA,11774
|
|
13
|
+
meds_algorithm_retain-0.1.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
14
|
+
meds_algorithm_retain-0.1.0.dist-info/entry_points.txt,sha256=sFHziIow-gCAYqpMqg8Ge1t1gXIUIZJkt3kYW0G5_jA,58
|
|
15
|
+
meds_algorithm_retain-0.1.0.dist-info/licenses/LICENSE,sha256=FAUMbnQf4z0hgnQETTcoeY5gGXRb6MHYPg-4AmgGtMw,1071
|
|
16
|
+
meds_algorithm_retain-0.1.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Florent Pollet
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
retain_meds/__init__.py
ADDED
retain_meds/__main__.py
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""Small backend-neutral command dispatcher."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
import json
|
|
5
|
+
import sys
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from .retain_meds import create_repository
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _value(text: str):
|
|
12
|
+
try:
|
|
13
|
+
return json.loads(text)
|
|
14
|
+
except json.JSONDecodeError:
|
|
15
|
+
# MEDS-DEV and Hydra-style commands pass unquoted lists such as splits=[held_out].
|
|
16
|
+
if text.startswith("[") and text.endswith("]"):
|
|
17
|
+
return [_value(item.strip()) for item in text[1:-1].split(",") if item.strip()]
|
|
18
|
+
return text
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def main() -> None:
|
|
22
|
+
repository = create_repository()
|
|
23
|
+
if len(sys.argv) > 1 and sys.argv[1] == "run":
|
|
24
|
+
try:
|
|
25
|
+
from meds_algorithm.orchestration.cli import main as run_profile
|
|
26
|
+
except ImportError as error:
|
|
27
|
+
raise SystemExit('Install the runner with: pip install "meds-algorithm-retain[orchestration]" (or uv sync --extra orchestration in a checkout)') from error
|
|
28
|
+
arguments = sys.argv[2:]
|
|
29
|
+
explicit_directory = any(arg == "--config-dir" or arg.startswith("--config-dir=") for arg in arguments)
|
|
30
|
+
if not explicit_directory and not (Path.cwd() / "configs" / "runs").is_dir():
|
|
31
|
+
arguments = ["--config-dir", str(Path(__file__).parent / "resources" / "runs"), *arguments]
|
|
32
|
+
run_profile(repository, module="retain_meds", argv=arguments)
|
|
33
|
+
return
|
|
34
|
+
parser = argparse.ArgumentParser(description="Run one declared MEDS algorithm profile")
|
|
35
|
+
parser.add_argument("command", help="run (prototype), profiles, commands, or a step name")
|
|
36
|
+
parser.add_argument("--profile", choices=repository.profiles)
|
|
37
|
+
parser.add_argument("arguments", nargs="*", metavar="key=value")
|
|
38
|
+
args = parser.parse_intermixed_args()
|
|
39
|
+
if args.command == "profiles":
|
|
40
|
+
if args.arguments:
|
|
41
|
+
parser.error("profiles does not accept key=value arguments")
|
|
42
|
+
print("\n".join(repository.profiles))
|
|
43
|
+
return
|
|
44
|
+
try:
|
|
45
|
+
profile = repository.select_profile(args.profile)
|
|
46
|
+
if args.command == "commands":
|
|
47
|
+
if args.arguments:
|
|
48
|
+
parser.error("commands does not accept key=value arguments")
|
|
49
|
+
print("\n".join(repository.profile_commands(profile)))
|
|
50
|
+
return
|
|
51
|
+
arguments = {}
|
|
52
|
+
for item in args.arguments:
|
|
53
|
+
if "=" not in item:
|
|
54
|
+
parser.error(f"Expected key=value, got {item!r}")
|
|
55
|
+
key, value = item.split("=", 1)
|
|
56
|
+
if not key or key in arguments:
|
|
57
|
+
parser.error(f"Empty or duplicate argument {key!r}")
|
|
58
|
+
arguments[key] = _value(value)
|
|
59
|
+
repository.run(args.command, arguments, profile=profile)
|
|
60
|
+
except (ValueError, TypeError, FileNotFoundError) as error:
|
|
61
|
+
parser.error(str(error))
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
if __name__ == "__main__":
|
|
65
|
+
main()
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"""Per-code contributions, Eq. (5) of the paper and `test_retain.py` of the official repository.
|
|
2
|
+
|
|
3
|
+
omega(y, x_{j,k}) = alpha_j * w_output . (beta_j * W_emb[:, k]) * x_{j,k}
|
|
4
|
+
|
|
5
|
+
`x` is binary here, so the coefficient *is* the contribution, which is the case the paper calls out
|
|
6
|
+
explicitly. This is the interpretability claim that motivates RETAIN, so it is implemented rather than
|
|
7
|
+
described; it is off by default because it costs a second pass and writes one record per scored key.
|
|
8
|
+
|
|
9
|
+
The output is NDJSON on purpose. MEDS-DEV and the conformance suite read every `*.parquet` under the
|
|
10
|
+
predictions directory as predictions, so a second keyed parquet there would look like duplicate
|
|
11
|
+
prediction keys and fail the coverage check.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
import numpy as np
|
|
20
|
+
import torch
|
|
21
|
+
|
|
22
|
+
from .config import RETAINConfig
|
|
23
|
+
from ._model import Retain
|
|
24
|
+
from ._training import collate
|
|
25
|
+
from ._windows import SequenceBatch
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _timestamp(microseconds: int) -> str:
|
|
29
|
+
return np.datetime64(int(microseconds), "us").astype("datetime64[us]").item().isoformat()
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def write_contributions(
|
|
33
|
+
path: Path,
|
|
34
|
+
*,
|
|
35
|
+
model: Retain,
|
|
36
|
+
batch: SequenceBatch,
|
|
37
|
+
config: RETAINConfig,
|
|
38
|
+
device: torch.device,
|
|
39
|
+
code_strings: list[str],
|
|
40
|
+
top_k: int,
|
|
41
|
+
) -> int:
|
|
42
|
+
"""Write one NDJSON record per key and return the number of records written."""
|
|
43
|
+
model.eval()
|
|
44
|
+
written = 0
|
|
45
|
+
with Path(path).open("w") as handle:
|
|
46
|
+
for start in range(0, batch.n_keys, config.batch_size):
|
|
47
|
+
rows = np.arange(start, min(start + config.batch_size, batch.n_keys), dtype=np.int64)
|
|
48
|
+
tensors = collate(batch, rows, config, device)
|
|
49
|
+
with torch.no_grad():
|
|
50
|
+
output = model(
|
|
51
|
+
tensors.codes,
|
|
52
|
+
tensors.offsets,
|
|
53
|
+
tensors.slots,
|
|
54
|
+
tensors.lengths,
|
|
55
|
+
tensors.gaps,
|
|
56
|
+
n_rows=tensors.n_rows,
|
|
57
|
+
n_visits=tensors.n_visits,
|
|
58
|
+
return_attention=True,
|
|
59
|
+
)
|
|
60
|
+
probabilities = torch.sigmoid(output.logits).cpu().numpy()
|
|
61
|
+
alpha = output.alpha.cpu().numpy()
|
|
62
|
+
beta = output.beta.cpu().numpy()
|
|
63
|
+
weights = model.W_emb.detach()
|
|
64
|
+
w_output = model.w_output.weight.detach().squeeze(0)
|
|
65
|
+
|
|
66
|
+
for position, row in enumerate(rows.tolist()):
|
|
67
|
+
first, last = int(batch.row_ptr[row]), int(batch.row_ptr[row + 1])
|
|
68
|
+
entries = []
|
|
69
|
+
for visit in range(last - first):
|
|
70
|
+
index = first + visit
|
|
71
|
+
span = slice(int(batch.visit_ptr[index]), int(batch.visit_ptr[index + 1]))
|
|
72
|
+
codes = batch.codes[span].astype(np.int64)
|
|
73
|
+
if not codes.size:
|
|
74
|
+
continue
|
|
75
|
+
embeddings = weights[torch.from_numpy(codes).to(weights.device)]
|
|
76
|
+
scaled = embeddings * torch.from_numpy(beta[position, visit]).to(weights.device)
|
|
77
|
+
contributions = (scaled @ w_output).cpu().numpy() * float(alpha[position, visit])
|
|
78
|
+
for code, contribution in zip(codes.tolist(), contributions.tolist()):
|
|
79
|
+
entries.append(
|
|
80
|
+
{
|
|
81
|
+
"visit_index": visit,
|
|
82
|
+
"visit_time": _timestamp(batch.visit_time[index]),
|
|
83
|
+
"code": code_strings[code],
|
|
84
|
+
"alpha": float(alpha[position, visit]),
|
|
85
|
+
"contribution": float(contribution),
|
|
86
|
+
}
|
|
87
|
+
)
|
|
88
|
+
entries.sort(key=lambda entry: abs(entry["contribution"]), reverse=True)
|
|
89
|
+
handle.write(
|
|
90
|
+
json.dumps(
|
|
91
|
+
{
|
|
92
|
+
"subject_id": int(batch.keys[row, 0]),
|
|
93
|
+
"prediction_time": _timestamp(batch.keys[row, 1]),
|
|
94
|
+
"predicted_boolean_probability": float(probabilities[position]),
|
|
95
|
+
"n_visits": last - first,
|
|
96
|
+
"top_contributions": entries[:top_k],
|
|
97
|
+
}
|
|
98
|
+
)
|
|
99
|
+
+ "\n"
|
|
100
|
+
)
|
|
101
|
+
written += 1
|
|
102
|
+
return written
|
retain_meds/_model.py
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
"""RETAIN, transcribed from `mp2893/retain` @ 9fd39c46 (`retain.py`, `build_model`).
|
|
2
|
+
|
|
3
|
+
Steps 1-5 of the paper, with the official code as the tie-breaker wherever the paper leaves a choice open:
|
|
4
|
+
|
|
5
|
+
v_j = W_emb x_j Step 1, linear, no bias, no activation
|
|
6
|
+
g_i..g_1 = GRU_alpha(v_i..v_1) Step 2, reverse time
|
|
7
|
+
e_j = w_alpha . g_j + b_alpha ; alpha = softmax(e) over visits
|
|
8
|
+
h_i..h_1 = GRU_beta(v_i..v_1) Step 3, reverse time
|
|
9
|
+
beta_j = tanh(W_beta h_j + b_beta)
|
|
10
|
+
c_i = sum_j alpha_j beta_j * v_j Step 4
|
|
11
|
+
y = sigmoid(w_output . c_i + b_output) Step 5, binary case
|
|
12
|
+
|
|
13
|
+
Three transcription notes:
|
|
14
|
+
|
|
15
|
+
* `torch.nn.GRU` is gate-for-gate identical to the official `gru_layer`: both compute
|
|
16
|
+
`n = tanh(W_in x + r * (W_hn h + b_hn))` with the reset gate applied to the whole recurrent term, both
|
|
17
|
+
use `h' = (1 - z) n + z h`, and both order the packed gates `(r, z, n)`. No custom cell is needed.
|
|
18
|
+
One difference is real and is removed here rather than tolerated: Theano's `gru_layer` has a single
|
|
19
|
+
bias, added to `W x` only, while `nn.GRU` has a second recurrent bias whose `n` component sits *inside*
|
|
20
|
+
the reset-gate product as `r * b_hn`, a term the official cell does not have. `bias_hh` is therefore
|
|
21
|
+
pinned to zero and excluded from optimisation, which makes the two cells algebraically identical.
|
|
22
|
+
* The official code multiplies both GRU outputs by `0.5` before the attention heads. That factor is in
|
|
23
|
+
`retain.py` and again in `test_retain.py`, so it is deliberate upstream rather than a typo, and it is
|
|
24
|
+
kept here. It is not in the paper: on alpha it acts as a softmax temperature.
|
|
25
|
+
* Dropout is applied to the embedding *in place*, so the same dropped `v` feeds the attention GRUs and the
|
|
26
|
+
context sum of Step 4. `build_model` rebinds `emb` before using it in `attentionStep`.
|
|
27
|
+
|
|
28
|
+
Only the final timestep is computed. The official training graph builds `c_t` for every prefix but reads
|
|
29
|
+
`y_hat` at `lengths - 1`, because this code performs sequence classification; computing `c_T` alone is the
|
|
30
|
+
same value, and is what the official `test_retain.py` does.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
from dataclasses import dataclass
|
|
36
|
+
|
|
37
|
+
import torch
|
|
38
|
+
from torch import nn
|
|
39
|
+
from torch.nn import functional as F
|
|
40
|
+
from torch.nn.utils import rnn as rnn_utils
|
|
41
|
+
|
|
42
|
+
from .config import RETAINConfig
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class RetainOutput:
|
|
47
|
+
"""Logits plus the quantities Eq. (5) needs to attribute a score to individual codes."""
|
|
48
|
+
|
|
49
|
+
logits: torch.Tensor
|
|
50
|
+
alpha: torch.Tensor | None = None
|
|
51
|
+
beta: torch.Tensor | None = None
|
|
52
|
+
embeddings: torch.Tensor | None = None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def reverse_padded(sequence: torch.Tensor, lengths: torch.Tensor) -> torch.Tensor:
|
|
56
|
+
"""Reverse each row's first `length` positions and leave its padding untouched."""
|
|
57
|
+
batch, total = sequence.shape[0], sequence.shape[1]
|
|
58
|
+
positions = torch.arange(total, device=sequence.device).unsqueeze(0)
|
|
59
|
+
limits = lengths.to(sequence.device).unsqueeze(1)
|
|
60
|
+
flipped = (limits - 1 - positions).clamp(min=0)
|
|
61
|
+
index = torch.where(positions < limits, flipped, positions)
|
|
62
|
+
return sequence.gather(1, index.unsqueeze(-1).expand(-1, -1, sequence.shape[2]))
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class Retain(nn.Module):
|
|
66
|
+
def __init__(self, vocabulary_size: int, config: RETAINConfig) -> None:
|
|
67
|
+
super().__init__()
|
|
68
|
+
self.vocabulary_size = int(vocabulary_size)
|
|
69
|
+
self.config = config
|
|
70
|
+
embed = config.embed_size
|
|
71
|
+
self.W_emb = nn.Parameter(torch.empty(self.vocabulary_size, embed))
|
|
72
|
+
self.gru_alpha = nn.GRU(config.gru_input_size, config.alpha_hidden_dim_size, batch_first=True)
|
|
73
|
+
self.gru_beta = nn.GRU(config.gru_input_size, config.beta_hidden_dim_size, batch_first=True)
|
|
74
|
+
self.w_alpha = nn.Linear(config.alpha_hidden_dim_size, 1)
|
|
75
|
+
self.W_beta = nn.Linear(config.beta_hidden_dim_size, embed)
|
|
76
|
+
self.w_output = nn.Linear(embed, 1)
|
|
77
|
+
self.embedding_dropout = nn.Dropout(p=1.0 - config.keep_prob_emb)
|
|
78
|
+
self.context_dropout = nn.Dropout(p=1.0 - config.keep_prob_context)
|
|
79
|
+
self.reset_parameters()
|
|
80
|
+
|
|
81
|
+
def reset_parameters(self) -> None:
|
|
82
|
+
"""`get_random_weight` is uniform(-0.1, 0.1); every bias in `init_params` starts at zero."""
|
|
83
|
+
for name, parameter in self.named_parameters():
|
|
84
|
+
if "bias" in name:
|
|
85
|
+
nn.init.zeros_(parameter)
|
|
86
|
+
else:
|
|
87
|
+
nn.init.uniform_(parameter, -0.1, 0.1)
|
|
88
|
+
for gru in (self.gru_alpha, self.gru_beta):
|
|
89
|
+
gru.bias_hh_l0.requires_grad_(False)
|
|
90
|
+
|
|
91
|
+
def trainable_parameters(self) -> list[nn.Parameter]:
|
|
92
|
+
"""Everything the official `tparams` optimises, which excludes the pinned `bias_hh`."""
|
|
93
|
+
return [parameter for parameter in self.parameters() if parameter.requires_grad]
|
|
94
|
+
|
|
95
|
+
def n_parameters(self) -> int:
|
|
96
|
+
return sum(parameter.numel() for parameter in self.parameters())
|
|
97
|
+
|
|
98
|
+
def forward(
|
|
99
|
+
self,
|
|
100
|
+
codes: torch.Tensor,
|
|
101
|
+
offsets: torch.Tensor,
|
|
102
|
+
slots: torch.Tensor,
|
|
103
|
+
lengths: torch.Tensor,
|
|
104
|
+
gaps: torch.Tensor | None = None,
|
|
105
|
+
*,
|
|
106
|
+
n_rows: int,
|
|
107
|
+
n_visits: int,
|
|
108
|
+
return_attention: bool = False,
|
|
109
|
+
) -> RetainOutput:
|
|
110
|
+
"""Score `n_rows` sequences padded to `n_visits` visits.
|
|
111
|
+
|
|
112
|
+
`codes`/`offsets` are an `EmbeddingBag` view of the batch: one bag per real visit, holding that
|
|
113
|
+
visit's vocabulary indices. `x_j` is binary, so summing the rows of `W_emb` named by the bag *is*
|
|
114
|
+
`W_emb x_j`, without ever materialising the `|C|`-wide one-hot matrix. `slots` places each bag at
|
|
115
|
+
its `row * n_visits + position` cell of the padded grid.
|
|
116
|
+
"""
|
|
117
|
+
device = self.W_emb.device
|
|
118
|
+
embed = self.config.embed_size
|
|
119
|
+
flat = torch.zeros(n_rows * n_visits, embed, device=device, dtype=self.W_emb.dtype)
|
|
120
|
+
if codes.numel():
|
|
121
|
+
bags = F.embedding_bag(
|
|
122
|
+
codes, self.W_emb, offsets=offsets, mode="sum", include_last_offset=True
|
|
123
|
+
)
|
|
124
|
+
flat = flat.index_copy(0, slots, bags)
|
|
125
|
+
visits = self.embedding_dropout(flat.view(n_rows, n_visits, embed))
|
|
126
|
+
|
|
127
|
+
recurrent_input = visits
|
|
128
|
+
if self.config.use_time:
|
|
129
|
+
if gaps is None:
|
|
130
|
+
raise ValueError("use_time is set but no visit gaps were supplied")
|
|
131
|
+
durations = torch.zeros(n_rows * n_visits, 1, device=device, dtype=visits.dtype)
|
|
132
|
+
if codes.numel():
|
|
133
|
+
durations = durations.index_copy(0, slots, gaps.to(visits.dtype).unsqueeze(1))
|
|
134
|
+
durations = durations.view(n_rows, n_visits, 1)
|
|
135
|
+
if self.config.use_log_time:
|
|
136
|
+
durations = torch.log1p(durations)
|
|
137
|
+
recurrent_input = torch.cat([visits, durations], dim=2)
|
|
138
|
+
|
|
139
|
+
reversed_input = reverse_padded(recurrent_input, lengths)
|
|
140
|
+
packed = rnn_utils.pack_padded_sequence(
|
|
141
|
+
reversed_input, lengths.cpu(), batch_first=True, enforce_sorted=False
|
|
142
|
+
)
|
|
143
|
+
alpha_states, _ = self.gru_alpha(packed)
|
|
144
|
+
beta_states, _ = self.gru_beta(packed)
|
|
145
|
+
alpha_states, _ = rnn_utils.pad_packed_sequence(
|
|
146
|
+
alpha_states, batch_first=True, total_length=n_visits
|
|
147
|
+
)
|
|
148
|
+
beta_states, _ = rnn_utils.pad_packed_sequence(
|
|
149
|
+
beta_states, batch_first=True, total_length=n_visits
|
|
150
|
+
)
|
|
151
|
+
alpha_states = reverse_padded(alpha_states, lengths) * 0.5
|
|
152
|
+
beta_states = reverse_padded(beta_states, lengths) * 0.5
|
|
153
|
+
|
|
154
|
+
mask = torch.arange(n_visits, device=device).unsqueeze(0) < lengths.to(device).unsqueeze(1)
|
|
155
|
+
logits_alpha = self.w_alpha(alpha_states).squeeze(-1)
|
|
156
|
+
alpha = torch.softmax(logits_alpha.masked_fill(~mask, float("-inf")), dim=1)
|
|
157
|
+
beta = torch.tanh(self.W_beta(beta_states))
|
|
158
|
+
|
|
159
|
+
context = (alpha.unsqueeze(-1) * beta * visits).sum(dim=1)
|
|
160
|
+
logits = self.w_output(self.context_dropout(context)).squeeze(-1)
|
|
161
|
+
if not return_attention:
|
|
162
|
+
return RetainOutput(logits=logits)
|
|
163
|
+
return RetainOutput(logits=logits, alpha=alpha, beta=beta, embeddings=visits)
|
|
164
|
+
|
|
165
|
+
def regularization(self) -> torch.Tensor:
|
|
166
|
+
"""The four L2 penalties of `build_model`, on weights only, never on biases."""
|
|
167
|
+
config = self.config
|
|
168
|
+
penalty = config.L2_output * (self.w_output.weight**2).sum()
|
|
169
|
+
penalty = penalty + config.L2_alpha * (self.w_alpha.weight**2).sum()
|
|
170
|
+
penalty = penalty + config.L2_beta * (self.W_beta.weight**2).sum()
|
|
171
|
+
return penalty + config.L2_emb * (self.W_emb**2).sum()
|