coldread-tfm 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. coldread/__init__.py +5 -0
  2. coldread/context/__init__.py +55 -0
  3. coldread/context/adaptive_inference.py +372 -0
  4. coldread/context/attention_retrieval.py +287 -0
  5. coldread/context/posterior_matching.py +111 -0
  6. coldread/context/row_assembly.py +168 -0
  7. coldread/context/variational_rows.py +937 -0
  8. coldread/core/__init__.py +55 -0
  9. coldread/core/config.py +1205 -0
  10. coldread/core/output.py +256 -0
  11. coldread/core/pipeline.py +152 -0
  12. coldread/core/runtime.py +152 -0
  13. coldread/estimators.py +208 -0
  14. coldread/model/__init__.py +54 -0
  15. coldread/model/attention.py +701 -0
  16. coldread/model/bar_distribution.py +591 -0
  17. coldread/model/blocks.py +314 -0
  18. coldread/model/coldread_model.py +1542 -0
  19. coldread/model/decoders.py +636 -0
  20. coldread/model/feature_grouping.py +198 -0
  21. coldread/model/moe.py +1237 -0
  22. coldread/model/progressive_residual.py +329 -0
  23. coldread/model/stage4_deep_attention.py +188 -0
  24. coldread/model/stage4_deep_regression_distribution.py +67 -0
  25. coldread/model/stage4_label_head.py +53 -0
  26. coldread/model/stage4_predictive_attention.py +227 -0
  27. coldread/preprocessing/__init__.py +66 -0
  28. coldread/preprocessing/context_scaler.py +463 -0
  29. coldread/preprocessing/router.py +714 -0
  30. coldread/preprocessing/torch_stats.py +547 -0
  31. coldread/preprocessing/views.py +523 -0
  32. coldread/serialization.py +79 -0
  33. coldread_tfm-0.1.0.dist-info/METADATA +57 -0
  34. coldread_tfm-0.1.0.dist-info/RECORD +37 -0
  35. coldread_tfm-0.1.0.dist-info/WHEEL +5 -0
  36. coldread_tfm-0.1.0.dist-info/licenses/LICENSE +202 -0
  37. coldread_tfm-0.1.0.dist-info/top_level.txt +1 -0
coldread/__init__.py ADDED
@@ -0,0 +1,5 @@
1
+ """ColdRead tabular foundation model: inference-only public interface."""
2
+ from coldread.estimators import ColdReadClassifier, ColdReadRegressor
3
+
4
+ __version__ = "0.1.0"
5
+ __all__ = ["ColdReadClassifier", "ColdReadRegressor", "__version__"]
@@ -0,0 +1,55 @@
1
+ """
2
+ Context compression utilities for ColdRead.
3
+ """
4
+
5
+ from coldread.context.adaptive_inference import (
6
+ AdaptiveInducingPrediction,
7
+ adaptive_inducing_predict,
8
+ )
9
+ from coldread.context.attention_retrieval import (
10
+ AttentionRetrievalPrediction,
11
+ attention_retrieval_predict,
12
+ select_context_indices,
13
+ )
14
+ from coldread.context.posterior_matching import (
15
+ PosteriorMatchKind,
16
+ PosteriorMatchLoss,
17
+ classification_posterior_kl,
18
+ posterior_match_loss,
19
+ regression_posterior_mse,
20
+ )
21
+ from coldread.context.row_assembly import (
22
+ assemble_adaptive_inducing_context_rows,
23
+ validate_context_forward_mode,
24
+ )
25
+ from coldread.context.variational_rows import (
26
+ AdaptiveAttentionVariationalCompressor,
27
+ AdaptiveMeanPoolVariationalCompressor,
28
+ BayesByBackpropLinear,
29
+ CompressorWritableVariationalRows,
30
+ ContextForwardMode,
31
+ VariationalCompressorOutput,
32
+ VariationalRowGateKind,
33
+ )
34
+
35
+ __all__ = [
36
+ "AdaptiveMeanPoolVariationalCompressor",
37
+ "AttentionRetrievalPrediction",
38
+ "AdaptiveAttentionVariationalCompressor",
39
+ "BayesByBackpropLinear",
40
+ "CompressorWritableVariationalRows",
41
+ "ContextForwardMode",
42
+ "PosteriorMatchKind",
43
+ "PosteriorMatchLoss",
44
+ "VariationalCompressorOutput",
45
+ "VariationalRowGateKind",
46
+ "assemble_adaptive_inducing_context_rows",
47
+ "classification_posterior_kl",
48
+ "posterior_match_loss",
49
+ "regression_posterior_mse",
50
+ "validate_context_forward_mode",
51
+ "AdaptiveInducingPrediction",
52
+ "adaptive_inducing_predict",
53
+ "attention_retrieval_predict",
54
+ "select_context_indices",
55
+ ]
@@ -0,0 +1,372 @@
1
+ """
2
+ Adaptive inducing-row inference for ColdRead.
3
+
4
+ This module chooses how many synthetic variational rows to use during prediction.
5
+
6
+ The compressor always writes rows at forward time. This helper optionally runs
7
+ several inducing-mode forwards with increasing active-prefix budgets and stops
8
+ when the compressed predictive distribution stabilizes.
9
+
10
+ There are two useful modes:
11
+
12
+ fast adaptive mode:
13
+ compare inducing predictions across increasing row counts
14
+
15
+ diagnostic posterior-matching mode:
16
+ also compare inducing predictions to the full-context prediction
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ from dataclasses import dataclass
22
+ from typing import Literal
23
+
24
+ import torch
25
+ import torch.nn.functional as F
26
+
27
+ from coldread.context.posterior_matching import posterior_match_loss
28
+ from coldread.core.output import ColdReadOutput
29
+
30
+
31
+ AdaptiveStopReason = Literal[
32
+ "prediction_stabilized",
33
+ "posterior_matched",
34
+ "reached_max_rows",
35
+ ]
36
+
37
+
38
+ @dataclass(frozen=True)
39
+ class AdaptiveInducingPrediction:
40
+ """
41
+ Result from adaptive inducing inference.
42
+
43
+ Attributes:
44
+ output:
45
+ Final ColdReadOutput from inducing mode.
46
+
47
+ selected_row_count:
48
+ Number of variational inducing rows used in the final pass.
49
+
50
+ stop_reason:
51
+ Why the adaptive loop stopped.
52
+
53
+ tried_row_counts:
54
+ Tensor containing tested row counts.
55
+
56
+ stability_scores:
57
+ Tensor of distances between consecutive inducing predictions.
58
+ The first entry is NaN because there is no previous prediction.
59
+
60
+ posterior_match_scores:
61
+ Optional tensor of distances to full-context prediction.
62
+ Filled with NaN when full-context comparison is disabled.
63
+
64
+ full_output:
65
+ Optional full-context ColdReadOutput used for diagnostic posterior
66
+ matching. None in fast mode.
67
+ """
68
+
69
+ output: ColdReadOutput
70
+ selected_row_count: int
71
+ stop_reason: AdaptiveStopReason
72
+ tried_row_counts: torch.Tensor
73
+ stability_scores: torch.Tensor
74
+ posterior_match_scores: torch.Tensor
75
+ full_output: ColdReadOutput | None = None
76
+
77
+
78
+ def _classification_prediction_distance(
79
+ previous_logits: torch.Tensor,
80
+ current_logits: torch.Tensor,
81
+ *,
82
+ temperature: float,
83
+ ) -> torch.Tensor:
84
+ """
85
+ Symmetric KL-like distance between two classification predictions.
86
+ """
87
+ previous_probs = F.softmax(previous_logits / temperature, dim=-1)
88
+ current_probs = F.softmax(current_logits / temperature, dim=-1)
89
+
90
+ previous_log_probs = torch.log(previous_probs.clamp_min(1.0e-8))
91
+ current_log_probs = torch.log(current_probs.clamp_min(1.0e-8))
92
+
93
+ kl_prev_cur = F.kl_div(
94
+ current_log_probs,
95
+ previous_probs,
96
+ reduction="batchmean",
97
+ )
98
+ kl_cur_prev = F.kl_div(
99
+ previous_log_probs,
100
+ current_probs,
101
+ reduction="batchmean",
102
+ )
103
+
104
+ return 0.5 * (kl_prev_cur + kl_cur_prev) * (temperature**2)
105
+
106
+
107
+ def _regression_prediction_distance(
108
+ previous_prediction: torch.Tensor,
109
+ current_prediction: torch.Tensor,
110
+ ) -> torch.Tensor:
111
+ """
112
+ MSE distance between two regression predictions.
113
+ """
114
+ return F.mse_loss(current_prediction, previous_prediction)
115
+
116
+
117
+ def _prediction_distance(
118
+ *,
119
+ previous: torch.Tensor,
120
+ current: torch.Tensor,
121
+ task: str,
122
+ temperature: float,
123
+ ) -> torch.Tensor:
124
+ """
125
+ Compute prediction distance for adaptive stopping.
126
+ """
127
+ if task == "classification":
128
+ return _classification_prediction_distance(
129
+ previous,
130
+ current,
131
+ temperature=temperature,
132
+ )
133
+
134
+ if task == "regression":
135
+ return _regression_prediction_distance(previous, current)
136
+
137
+ raise ValueError(f"Unknown task: {task!r}.")
138
+
139
+
140
+ def _make_row_counts(
141
+ *,
142
+ min_rows: int,
143
+ max_rows: int,
144
+ step_rows: int,
145
+ device: torch.device,
146
+ ) -> list[int]:
147
+ """
148
+ Build row-count schedule.
149
+ """
150
+ if min_rows <= 0:
151
+ raise ValueError(f"min_rows must be positive, got {min_rows}.")
152
+
153
+ if max_rows < min_rows:
154
+ raise ValueError(
155
+ f"max_rows must be >= min_rows. Got min_rows={min_rows}, "
156
+ f"max_rows={max_rows}."
157
+ )
158
+
159
+ if step_rows <= 0:
160
+ raise ValueError(f"step_rows must be positive, got {step_rows}.")
161
+
162
+ row_counts = list(range(min_rows, max_rows + 1, step_rows))
163
+
164
+ if row_counts[-1] != max_rows:
165
+ row_counts.append(max_rows)
166
+
167
+ return row_counts
168
+
169
+
170
+ @torch.no_grad()
171
+ def adaptive_inducing_predict(
172
+ *,
173
+ model,
174
+ x_context: torch.Tensor,
175
+ y_context: torch.Tensor,
176
+ x_query: torch.Tensor,
177
+ task: Literal["classification", "regression"],
178
+ n_classes: int | None = None,
179
+ min_rows: int | None = None,
180
+ max_rows: int | None = None,
181
+ step_rows: int = 1,
182
+ stability_tolerance: float = 1.0e-3,
183
+ posterior_match_tolerance: float | None = None,
184
+ compare_to_full: bool = False,
185
+ temperature: float = 2.0,
186
+ ) -> AdaptiveInducingPrediction:
187
+ """
188
+ Predict with adaptive variational inducing rows.
189
+
190
+ Args:
191
+ model:
192
+ ColdReadModel-like object supporting:
193
+ context_forward_mode="inducing"
194
+ forced_inducing_row_count=...
195
+
196
+ x_context, y_context, x_query:
197
+ Context/query batch.
198
+
199
+ task:
200
+ "classification" or "regression".
201
+
202
+ n_classes:
203
+ Realized class count for classification.
204
+
205
+ min_rows:
206
+ Smallest inducing-row prefix to try. Defaults to
207
+ model.config.min_variational_context_rows.
208
+
209
+ max_rows:
210
+ Largest inducing-row prefix to try. Defaults to
211
+ model.config.num_variational_context_rows.
212
+
213
+ step_rows:
214
+ Row-count increment.
215
+
216
+ stability_tolerance:
217
+ Stop when consecutive inducing predictions differ by this much or less.
218
+
219
+ posterior_match_tolerance:
220
+ Optional diagnostic threshold against the full-context prediction.
221
+
222
+ compare_to_full:
223
+ If True, compute full-context prediction once and compare each
224
+ inducing prediction to it. This is slower and intended for research
225
+ diagnostics, not fast runtime.
226
+
227
+ temperature:
228
+ Temperature used for classification distances.
229
+
230
+ Returns:
231
+ AdaptiveInducingPrediction.
232
+ """
233
+ if task not in {"classification", "regression"}:
234
+ raise ValueError(f"task must be 'classification' or 'regression', got {task!r}.")
235
+
236
+ if stability_tolerance < 0:
237
+ raise ValueError(
238
+ f"stability_tolerance must be non-negative, got {stability_tolerance}."
239
+ )
240
+
241
+ if posterior_match_tolerance is not None and posterior_match_tolerance < 0:
242
+ raise ValueError(
243
+ "posterior_match_tolerance must be non-negative when provided, got "
244
+ f"{posterior_match_tolerance}."
245
+ )
246
+
247
+ resolved_min_rows = (
248
+ int(model.config.min_variational_context_rows)
249
+ if min_rows is None
250
+ else int(min_rows)
251
+ )
252
+ resolved_max_rows = (
253
+ int(model.config.num_variational_context_rows)
254
+ if max_rows is None
255
+ else int(max_rows)
256
+ )
257
+
258
+ row_counts = _make_row_counts(
259
+ min_rows=resolved_min_rows,
260
+ max_rows=resolved_max_rows,
261
+ step_rows=step_rows,
262
+ device=x_context.device,
263
+ )
264
+
265
+ full_output = None
266
+
267
+ if compare_to_full:
268
+ full_output = model(
269
+ x_context=x_context,
270
+ y_context=y_context,
271
+ x_query=x_query,
272
+ task=task,
273
+ n_classes=n_classes,
274
+ context_forward_mode="full",
275
+ )
276
+
277
+ previous_predictions: torch.Tensor | None = None
278
+ final_output: ColdReadOutput | None = None
279
+ final_row_count = row_counts[-1]
280
+ stop_reason: AdaptiveStopReason = "reached_max_rows"
281
+
282
+ stability_scores: list[torch.Tensor] = []
283
+ posterior_scores: list[torch.Tensor] = []
284
+
285
+ for row_count in row_counts:
286
+ output = model(
287
+ x_context=x_context,
288
+ y_context=y_context,
289
+ x_query=x_query,
290
+ task=task,
291
+ n_classes=n_classes,
292
+ context_forward_mode="inducing",
293
+ forced_inducing_row_count=row_count,
294
+ )
295
+
296
+ final_output = output
297
+ final_row_count = row_count
298
+
299
+ if previous_predictions is None:
300
+ stability_score = torch.full(
301
+ (),
302
+ float("nan"),
303
+ device=output.predictions.device,
304
+ )
305
+ else:
306
+ stability_score = _prediction_distance(
307
+ previous=previous_predictions,
308
+ current=output.predictions,
309
+ task=task,
310
+ temperature=temperature,
311
+ )
312
+
313
+ stability_scores.append(stability_score)
314
+
315
+ if full_output is not None:
316
+ match_kind = (
317
+ "classification_kl"
318
+ if task == "classification"
319
+ else "regression_mse"
320
+ )
321
+ posterior_score = posterior_match_loss(
322
+ full_output=full_output.predictions,
323
+ compressed_output=output.predictions,
324
+ kind=match_kind,
325
+ temperature=temperature,
326
+ ).loss
327
+ else:
328
+ posterior_score = torch.full(
329
+ (),
330
+ float("nan"),
331
+ device=output.predictions.device,
332
+ )
333
+
334
+ posterior_scores.append(posterior_score)
335
+
336
+ if (
337
+ posterior_match_tolerance is not None
338
+ and full_output is not None
339
+ and torch.isfinite(posterior_score)
340
+ and float(posterior_score.item()) <= posterior_match_tolerance
341
+ ):
342
+ stop_reason = "posterior_matched"
343
+ break
344
+
345
+ if (
346
+ previous_predictions is not None
347
+ and torch.isfinite(stability_score)
348
+ and float(stability_score.item()) <= stability_tolerance
349
+ ):
350
+ stop_reason = "prediction_stabilized"
351
+ break
352
+
353
+ previous_predictions = output.predictions.detach()
354
+
355
+ if final_output is None:
356
+ raise RuntimeError("adaptive_inducing_predict did not run any forward passes.")
357
+
358
+ tried_row_counts = torch.tensor(
359
+ row_counts[: len(stability_scores)],
360
+ device=final_output.predictions.device,
361
+ dtype=torch.long,
362
+ )
363
+
364
+ return AdaptiveInducingPrediction(
365
+ output=final_output,
366
+ selected_row_count=int(final_row_count),
367
+ stop_reason=stop_reason,
368
+ tried_row_counts=tried_row_counts,
369
+ stability_scores=torch.stack(stability_scores),
370
+ posterior_match_scores=torch.stack(posterior_scores),
371
+ full_output=full_output,
372
+ )