coldread-tfm 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- coldread/__init__.py +5 -0
- coldread/context/__init__.py +55 -0
- coldread/context/adaptive_inference.py +372 -0
- coldread/context/attention_retrieval.py +287 -0
- coldread/context/posterior_matching.py +111 -0
- coldread/context/row_assembly.py +168 -0
- coldread/context/variational_rows.py +937 -0
- coldread/core/__init__.py +55 -0
- coldread/core/config.py +1205 -0
- coldread/core/output.py +256 -0
- coldread/core/pipeline.py +152 -0
- coldread/core/runtime.py +152 -0
- coldread/estimators.py +208 -0
- coldread/model/__init__.py +54 -0
- coldread/model/attention.py +701 -0
- coldread/model/bar_distribution.py +591 -0
- coldread/model/blocks.py +314 -0
- coldread/model/coldread_model.py +1542 -0
- coldread/model/decoders.py +636 -0
- coldread/model/feature_grouping.py +198 -0
- coldread/model/moe.py +1237 -0
- coldread/model/progressive_residual.py +329 -0
- coldread/model/stage4_deep_attention.py +188 -0
- coldread/model/stage4_deep_regression_distribution.py +67 -0
- coldread/model/stage4_label_head.py +53 -0
- coldread/model/stage4_predictive_attention.py +227 -0
- coldread/preprocessing/__init__.py +66 -0
- coldread/preprocessing/context_scaler.py +463 -0
- coldread/preprocessing/router.py +714 -0
- coldread/preprocessing/torch_stats.py +547 -0
- coldread/preprocessing/views.py +523 -0
- coldread/serialization.py +79 -0
- coldread_tfm-0.1.0.dist-info/METADATA +57 -0
- coldread_tfm-0.1.0.dist-info/RECORD +37 -0
- coldread_tfm-0.1.0.dist-info/WHEEL +5 -0
- coldread_tfm-0.1.0.dist-info/licenses/LICENSE +202 -0
- coldread_tfm-0.1.0.dist-info/top_level.txt +1 -0
coldread/__init__.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Context compression utilities for ColdRead.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from coldread.context.adaptive_inference import (
|
|
6
|
+
AdaptiveInducingPrediction,
|
|
7
|
+
adaptive_inducing_predict,
|
|
8
|
+
)
|
|
9
|
+
from coldread.context.attention_retrieval import (
|
|
10
|
+
AttentionRetrievalPrediction,
|
|
11
|
+
attention_retrieval_predict,
|
|
12
|
+
select_context_indices,
|
|
13
|
+
)
|
|
14
|
+
from coldread.context.posterior_matching import (
|
|
15
|
+
PosteriorMatchKind,
|
|
16
|
+
PosteriorMatchLoss,
|
|
17
|
+
classification_posterior_kl,
|
|
18
|
+
posterior_match_loss,
|
|
19
|
+
regression_posterior_mse,
|
|
20
|
+
)
|
|
21
|
+
from coldread.context.row_assembly import (
|
|
22
|
+
assemble_adaptive_inducing_context_rows,
|
|
23
|
+
validate_context_forward_mode,
|
|
24
|
+
)
|
|
25
|
+
from coldread.context.variational_rows import (
|
|
26
|
+
AdaptiveAttentionVariationalCompressor,
|
|
27
|
+
AdaptiveMeanPoolVariationalCompressor,
|
|
28
|
+
BayesByBackpropLinear,
|
|
29
|
+
CompressorWritableVariationalRows,
|
|
30
|
+
ContextForwardMode,
|
|
31
|
+
VariationalCompressorOutput,
|
|
32
|
+
VariationalRowGateKind,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
__all__ = [
|
|
36
|
+
"AdaptiveMeanPoolVariationalCompressor",
|
|
37
|
+
"AttentionRetrievalPrediction",
|
|
38
|
+
"AdaptiveAttentionVariationalCompressor",
|
|
39
|
+
"BayesByBackpropLinear",
|
|
40
|
+
"CompressorWritableVariationalRows",
|
|
41
|
+
"ContextForwardMode",
|
|
42
|
+
"PosteriorMatchKind",
|
|
43
|
+
"PosteriorMatchLoss",
|
|
44
|
+
"VariationalCompressorOutput",
|
|
45
|
+
"VariationalRowGateKind",
|
|
46
|
+
"assemble_adaptive_inducing_context_rows",
|
|
47
|
+
"classification_posterior_kl",
|
|
48
|
+
"posterior_match_loss",
|
|
49
|
+
"regression_posterior_mse",
|
|
50
|
+
"validate_context_forward_mode",
|
|
51
|
+
"AdaptiveInducingPrediction",
|
|
52
|
+
"adaptive_inducing_predict",
|
|
53
|
+
"attention_retrieval_predict",
|
|
54
|
+
"select_context_indices",
|
|
55
|
+
]
|
|
@@ -0,0 +1,372 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Adaptive inducing-row inference for ColdRead.
|
|
3
|
+
|
|
4
|
+
This module chooses how many synthetic variational rows to use during prediction.
|
|
5
|
+
|
|
6
|
+
The compressor always writes rows at forward time. This helper optionally runs
|
|
7
|
+
several inducing-mode forwards with increasing active-prefix budgets and stops
|
|
8
|
+
when the compressed predictive distribution stabilizes.
|
|
9
|
+
|
|
10
|
+
There are two useful modes:
|
|
11
|
+
|
|
12
|
+
fast adaptive mode:
|
|
13
|
+
compare inducing predictions across increasing row counts
|
|
14
|
+
|
|
15
|
+
diagnostic posterior-matching mode:
|
|
16
|
+
also compare inducing predictions to the full-context prediction
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from typing import Literal
|
|
23
|
+
|
|
24
|
+
import torch
|
|
25
|
+
import torch.nn.functional as F
|
|
26
|
+
|
|
27
|
+
from coldread.context.posterior_matching import posterior_match_loss
|
|
28
|
+
from coldread.core.output import ColdReadOutput
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
AdaptiveStopReason = Literal[
|
|
32
|
+
"prediction_stabilized",
|
|
33
|
+
"posterior_matched",
|
|
34
|
+
"reached_max_rows",
|
|
35
|
+
]
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class AdaptiveInducingPrediction:
|
|
40
|
+
"""
|
|
41
|
+
Result from adaptive inducing inference.
|
|
42
|
+
|
|
43
|
+
Attributes:
|
|
44
|
+
output:
|
|
45
|
+
Final ColdReadOutput from inducing mode.
|
|
46
|
+
|
|
47
|
+
selected_row_count:
|
|
48
|
+
Number of variational inducing rows used in the final pass.
|
|
49
|
+
|
|
50
|
+
stop_reason:
|
|
51
|
+
Why the adaptive loop stopped.
|
|
52
|
+
|
|
53
|
+
tried_row_counts:
|
|
54
|
+
Tensor containing tested row counts.
|
|
55
|
+
|
|
56
|
+
stability_scores:
|
|
57
|
+
Tensor of distances between consecutive inducing predictions.
|
|
58
|
+
The first entry is NaN because there is no previous prediction.
|
|
59
|
+
|
|
60
|
+
posterior_match_scores:
|
|
61
|
+
Optional tensor of distances to full-context prediction.
|
|
62
|
+
Filled with NaN when full-context comparison is disabled.
|
|
63
|
+
|
|
64
|
+
full_output:
|
|
65
|
+
Optional full-context ColdReadOutput used for diagnostic posterior
|
|
66
|
+
matching. None in fast mode.
|
|
67
|
+
"""
|
|
68
|
+
|
|
69
|
+
output: ColdReadOutput
|
|
70
|
+
selected_row_count: int
|
|
71
|
+
stop_reason: AdaptiveStopReason
|
|
72
|
+
tried_row_counts: torch.Tensor
|
|
73
|
+
stability_scores: torch.Tensor
|
|
74
|
+
posterior_match_scores: torch.Tensor
|
|
75
|
+
full_output: ColdReadOutput | None = None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _classification_prediction_distance(
|
|
79
|
+
previous_logits: torch.Tensor,
|
|
80
|
+
current_logits: torch.Tensor,
|
|
81
|
+
*,
|
|
82
|
+
temperature: float,
|
|
83
|
+
) -> torch.Tensor:
|
|
84
|
+
"""
|
|
85
|
+
Symmetric KL-like distance between two classification predictions.
|
|
86
|
+
"""
|
|
87
|
+
previous_probs = F.softmax(previous_logits / temperature, dim=-1)
|
|
88
|
+
current_probs = F.softmax(current_logits / temperature, dim=-1)
|
|
89
|
+
|
|
90
|
+
previous_log_probs = torch.log(previous_probs.clamp_min(1.0e-8))
|
|
91
|
+
current_log_probs = torch.log(current_probs.clamp_min(1.0e-8))
|
|
92
|
+
|
|
93
|
+
kl_prev_cur = F.kl_div(
|
|
94
|
+
current_log_probs,
|
|
95
|
+
previous_probs,
|
|
96
|
+
reduction="batchmean",
|
|
97
|
+
)
|
|
98
|
+
kl_cur_prev = F.kl_div(
|
|
99
|
+
previous_log_probs,
|
|
100
|
+
current_probs,
|
|
101
|
+
reduction="batchmean",
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
return 0.5 * (kl_prev_cur + kl_cur_prev) * (temperature**2)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _regression_prediction_distance(
|
|
108
|
+
previous_prediction: torch.Tensor,
|
|
109
|
+
current_prediction: torch.Tensor,
|
|
110
|
+
) -> torch.Tensor:
|
|
111
|
+
"""
|
|
112
|
+
MSE distance between two regression predictions.
|
|
113
|
+
"""
|
|
114
|
+
return F.mse_loss(current_prediction, previous_prediction)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _prediction_distance(
|
|
118
|
+
*,
|
|
119
|
+
previous: torch.Tensor,
|
|
120
|
+
current: torch.Tensor,
|
|
121
|
+
task: str,
|
|
122
|
+
temperature: float,
|
|
123
|
+
) -> torch.Tensor:
|
|
124
|
+
"""
|
|
125
|
+
Compute prediction distance for adaptive stopping.
|
|
126
|
+
"""
|
|
127
|
+
if task == "classification":
|
|
128
|
+
return _classification_prediction_distance(
|
|
129
|
+
previous,
|
|
130
|
+
current,
|
|
131
|
+
temperature=temperature,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
if task == "regression":
|
|
135
|
+
return _regression_prediction_distance(previous, current)
|
|
136
|
+
|
|
137
|
+
raise ValueError(f"Unknown task: {task!r}.")
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _make_row_counts(
|
|
141
|
+
*,
|
|
142
|
+
min_rows: int,
|
|
143
|
+
max_rows: int,
|
|
144
|
+
step_rows: int,
|
|
145
|
+
device: torch.device,
|
|
146
|
+
) -> list[int]:
|
|
147
|
+
"""
|
|
148
|
+
Build row-count schedule.
|
|
149
|
+
"""
|
|
150
|
+
if min_rows <= 0:
|
|
151
|
+
raise ValueError(f"min_rows must be positive, got {min_rows}.")
|
|
152
|
+
|
|
153
|
+
if max_rows < min_rows:
|
|
154
|
+
raise ValueError(
|
|
155
|
+
f"max_rows must be >= min_rows. Got min_rows={min_rows}, "
|
|
156
|
+
f"max_rows={max_rows}."
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
if step_rows <= 0:
|
|
160
|
+
raise ValueError(f"step_rows must be positive, got {step_rows}.")
|
|
161
|
+
|
|
162
|
+
row_counts = list(range(min_rows, max_rows + 1, step_rows))
|
|
163
|
+
|
|
164
|
+
if row_counts[-1] != max_rows:
|
|
165
|
+
row_counts.append(max_rows)
|
|
166
|
+
|
|
167
|
+
return row_counts
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
@torch.no_grad()
|
|
171
|
+
def adaptive_inducing_predict(
|
|
172
|
+
*,
|
|
173
|
+
model,
|
|
174
|
+
x_context: torch.Tensor,
|
|
175
|
+
y_context: torch.Tensor,
|
|
176
|
+
x_query: torch.Tensor,
|
|
177
|
+
task: Literal["classification", "regression"],
|
|
178
|
+
n_classes: int | None = None,
|
|
179
|
+
min_rows: int | None = None,
|
|
180
|
+
max_rows: int | None = None,
|
|
181
|
+
step_rows: int = 1,
|
|
182
|
+
stability_tolerance: float = 1.0e-3,
|
|
183
|
+
posterior_match_tolerance: float | None = None,
|
|
184
|
+
compare_to_full: bool = False,
|
|
185
|
+
temperature: float = 2.0,
|
|
186
|
+
) -> AdaptiveInducingPrediction:
|
|
187
|
+
"""
|
|
188
|
+
Predict with adaptive variational inducing rows.
|
|
189
|
+
|
|
190
|
+
Args:
|
|
191
|
+
model:
|
|
192
|
+
ColdReadModel-like object supporting:
|
|
193
|
+
context_forward_mode="inducing"
|
|
194
|
+
forced_inducing_row_count=...
|
|
195
|
+
|
|
196
|
+
x_context, y_context, x_query:
|
|
197
|
+
Context/query batch.
|
|
198
|
+
|
|
199
|
+
task:
|
|
200
|
+
"classification" or "regression".
|
|
201
|
+
|
|
202
|
+
n_classes:
|
|
203
|
+
Realized class count for classification.
|
|
204
|
+
|
|
205
|
+
min_rows:
|
|
206
|
+
Smallest inducing-row prefix to try. Defaults to
|
|
207
|
+
model.config.min_variational_context_rows.
|
|
208
|
+
|
|
209
|
+
max_rows:
|
|
210
|
+
Largest inducing-row prefix to try. Defaults to
|
|
211
|
+
model.config.num_variational_context_rows.
|
|
212
|
+
|
|
213
|
+
step_rows:
|
|
214
|
+
Row-count increment.
|
|
215
|
+
|
|
216
|
+
stability_tolerance:
|
|
217
|
+
Stop when consecutive inducing predictions differ by this much or less.
|
|
218
|
+
|
|
219
|
+
posterior_match_tolerance:
|
|
220
|
+
Optional diagnostic threshold against the full-context prediction.
|
|
221
|
+
|
|
222
|
+
compare_to_full:
|
|
223
|
+
If True, compute full-context prediction once and compare each
|
|
224
|
+
inducing prediction to it. This is slower and intended for research
|
|
225
|
+
diagnostics, not fast runtime.
|
|
226
|
+
|
|
227
|
+
temperature:
|
|
228
|
+
Temperature used for classification distances.
|
|
229
|
+
|
|
230
|
+
Returns:
|
|
231
|
+
AdaptiveInducingPrediction.
|
|
232
|
+
"""
|
|
233
|
+
if task not in {"classification", "regression"}:
|
|
234
|
+
raise ValueError(f"task must be 'classification' or 'regression', got {task!r}.")
|
|
235
|
+
|
|
236
|
+
if stability_tolerance < 0:
|
|
237
|
+
raise ValueError(
|
|
238
|
+
f"stability_tolerance must be non-negative, got {stability_tolerance}."
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
if posterior_match_tolerance is not None and posterior_match_tolerance < 0:
|
|
242
|
+
raise ValueError(
|
|
243
|
+
"posterior_match_tolerance must be non-negative when provided, got "
|
|
244
|
+
f"{posterior_match_tolerance}."
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
resolved_min_rows = (
|
|
248
|
+
int(model.config.min_variational_context_rows)
|
|
249
|
+
if min_rows is None
|
|
250
|
+
else int(min_rows)
|
|
251
|
+
)
|
|
252
|
+
resolved_max_rows = (
|
|
253
|
+
int(model.config.num_variational_context_rows)
|
|
254
|
+
if max_rows is None
|
|
255
|
+
else int(max_rows)
|
|
256
|
+
)
|
|
257
|
+
|
|
258
|
+
row_counts = _make_row_counts(
|
|
259
|
+
min_rows=resolved_min_rows,
|
|
260
|
+
max_rows=resolved_max_rows,
|
|
261
|
+
step_rows=step_rows,
|
|
262
|
+
device=x_context.device,
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
full_output = None
|
|
266
|
+
|
|
267
|
+
if compare_to_full:
|
|
268
|
+
full_output = model(
|
|
269
|
+
x_context=x_context,
|
|
270
|
+
y_context=y_context,
|
|
271
|
+
x_query=x_query,
|
|
272
|
+
task=task,
|
|
273
|
+
n_classes=n_classes,
|
|
274
|
+
context_forward_mode="full",
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
previous_predictions: torch.Tensor | None = None
|
|
278
|
+
final_output: ColdReadOutput | None = None
|
|
279
|
+
final_row_count = row_counts[-1]
|
|
280
|
+
stop_reason: AdaptiveStopReason = "reached_max_rows"
|
|
281
|
+
|
|
282
|
+
stability_scores: list[torch.Tensor] = []
|
|
283
|
+
posterior_scores: list[torch.Tensor] = []
|
|
284
|
+
|
|
285
|
+
for row_count in row_counts:
|
|
286
|
+
output = model(
|
|
287
|
+
x_context=x_context,
|
|
288
|
+
y_context=y_context,
|
|
289
|
+
x_query=x_query,
|
|
290
|
+
task=task,
|
|
291
|
+
n_classes=n_classes,
|
|
292
|
+
context_forward_mode="inducing",
|
|
293
|
+
forced_inducing_row_count=row_count,
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
final_output = output
|
|
297
|
+
final_row_count = row_count
|
|
298
|
+
|
|
299
|
+
if previous_predictions is None:
|
|
300
|
+
stability_score = torch.full(
|
|
301
|
+
(),
|
|
302
|
+
float("nan"),
|
|
303
|
+
device=output.predictions.device,
|
|
304
|
+
)
|
|
305
|
+
else:
|
|
306
|
+
stability_score = _prediction_distance(
|
|
307
|
+
previous=previous_predictions,
|
|
308
|
+
current=output.predictions,
|
|
309
|
+
task=task,
|
|
310
|
+
temperature=temperature,
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
stability_scores.append(stability_score)
|
|
314
|
+
|
|
315
|
+
if full_output is not None:
|
|
316
|
+
match_kind = (
|
|
317
|
+
"classification_kl"
|
|
318
|
+
if task == "classification"
|
|
319
|
+
else "regression_mse"
|
|
320
|
+
)
|
|
321
|
+
posterior_score = posterior_match_loss(
|
|
322
|
+
full_output=full_output.predictions,
|
|
323
|
+
compressed_output=output.predictions,
|
|
324
|
+
kind=match_kind,
|
|
325
|
+
temperature=temperature,
|
|
326
|
+
).loss
|
|
327
|
+
else:
|
|
328
|
+
posterior_score = torch.full(
|
|
329
|
+
(),
|
|
330
|
+
float("nan"),
|
|
331
|
+
device=output.predictions.device,
|
|
332
|
+
)
|
|
333
|
+
|
|
334
|
+
posterior_scores.append(posterior_score)
|
|
335
|
+
|
|
336
|
+
if (
|
|
337
|
+
posterior_match_tolerance is not None
|
|
338
|
+
and full_output is not None
|
|
339
|
+
and torch.isfinite(posterior_score)
|
|
340
|
+
and float(posterior_score.item()) <= posterior_match_tolerance
|
|
341
|
+
):
|
|
342
|
+
stop_reason = "posterior_matched"
|
|
343
|
+
break
|
|
344
|
+
|
|
345
|
+
if (
|
|
346
|
+
previous_predictions is not None
|
|
347
|
+
and torch.isfinite(stability_score)
|
|
348
|
+
and float(stability_score.item()) <= stability_tolerance
|
|
349
|
+
):
|
|
350
|
+
stop_reason = "prediction_stabilized"
|
|
351
|
+
break
|
|
352
|
+
|
|
353
|
+
previous_predictions = output.predictions.detach()
|
|
354
|
+
|
|
355
|
+
if final_output is None:
|
|
356
|
+
raise RuntimeError("adaptive_inducing_predict did not run any forward passes.")
|
|
357
|
+
|
|
358
|
+
tried_row_counts = torch.tensor(
|
|
359
|
+
row_counts[: len(stability_scores)],
|
|
360
|
+
device=final_output.predictions.device,
|
|
361
|
+
dtype=torch.long,
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
return AdaptiveInducingPrediction(
|
|
365
|
+
output=final_output,
|
|
366
|
+
selected_row_count=int(final_row_count),
|
|
367
|
+
stop_reason=stop_reason,
|
|
368
|
+
tried_row_counts=tried_row_counts,
|
|
369
|
+
stability_scores=torch.stack(stability_scores),
|
|
370
|
+
posterior_match_scores=torch.stack(posterior_scores),
|
|
371
|
+
full_output=full_output,
|
|
372
|
+
)
|