structured-data-models 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (152) hide show
  1. sdm/__init__.py +58 -0
  2. sdm/_constants.py +6 -0
  3. sdm/_inference.py +42 -0
  4. sdm/_kernels/__init__.py +10 -0
  5. sdm/_kernels/rmsnorm_cast.py +92 -0
  6. sdm/_kernels/segment_multi_reduce.py +141 -0
  7. sdm/_kernels/triton/__init__.py +7 -0
  8. sdm/_kernels/triton/rmsnorm_cast.py +136 -0
  9. sdm/_kernels/triton/segment_multi_reduce.py +173 -0
  10. sdm/_memory.py +34 -0
  11. sdm/_warnings.py +29 -0
  12. sdm/cache.py +163 -0
  13. sdm/ensemble.py +501 -0
  14. sdm/evaluation/__init__.py +11 -0
  15. sdm/evaluation/classification.py +205 -0
  16. sdm/explain/__init__.py +12 -0
  17. sdm/explain/base.py +123 -0
  18. sdm/explain/gradient.py +112 -0
  19. sdm/models/__init__.py +20 -0
  20. sdm/models/_huggingface.py +58 -0
  21. sdm/models/base.py +966 -0
  22. sdm/models/callback.py +106 -0
  23. sdm/models/ecoc.py +197 -0
  24. sdm/models/kumo/__init__.py +13 -0
  25. sdm/models/kumo/relational/NOTICE +5 -0
  26. sdm/models/kumo/relational/__init__.py +11 -0
  27. sdm/models/kumo/relational/graph.py +101 -0
  28. sdm/models/kumo/relational/invariant_gnn.py +122 -0
  29. sdm/models/kumo/relational/model.py +602 -0
  30. sdm/models/kumo/relational/recipe.py +18 -0
  31. sdm/models/kumo/relational/task.py +113 -0
  32. sdm/models/kumo/tabular/__init__.py +17 -0
  33. sdm/models/kumo/tabular/batching.py +139 -0
  34. sdm/models/kumo/tabular/block.py +100 -0
  35. sdm/models/kumo/tabular/cell_embedding.py +85 -0
  36. sdm/models/kumo/tabular/icl.py +136 -0
  37. sdm/models/kumo/tabular/model.py +299 -0
  38. sdm/models/kumo/tabular/recipe.py +71 -0
  39. sdm/models/kumo/tabular/row_embedding.py +214 -0
  40. sdm/models/tabfm/LICENSE +209 -0
  41. sdm/models/tabfm/__init__.py +11 -0
  42. sdm/models/tabfm/block.py +94 -0
  43. sdm/models/tabfm/cell_embedding.py +219 -0
  44. sdm/models/tabfm/ckpt.py +242 -0
  45. sdm/models/tabfm/icl.py +107 -0
  46. sdm/models/tabfm/model.py +261 -0
  47. sdm/models/tabfm/recipe.py +46 -0
  48. sdm/models/tabfm/row_embedding.py +241 -0
  49. sdm/models/tabiclv2/LICENSE +33 -0
  50. sdm/models/tabiclv2/__init__.py +11 -0
  51. sdm/models/tabiclv2/block.py +81 -0
  52. sdm/models/tabiclv2/ckpt.py +134 -0
  53. sdm/models/tabiclv2/icl.py +420 -0
  54. sdm/models/tabiclv2/model.py +293 -0
  55. sdm/models/tabiclv2/recipe.py +46 -0
  56. sdm/models/tabiclv2/row_embedding.py +254 -0
  57. sdm/nn/__init__.py +28 -0
  58. sdm/nn/_buffer.py +91 -0
  59. sdm/nn/attention.py +825 -0
  60. sdm/nn/glu.py +78 -0
  61. sdm/nn/linear.py +46 -0
  62. sdm/nn/rms_norm.py +59 -0
  63. sdm/nn/rope.py +107 -0
  64. sdm/nn/scaling.py +240 -0
  65. sdm/nn/set_transformer.py +162 -0
  66. sdm/nn/softplus.py +65 -0
  67. sdm/processing/__init__.py +96 -0
  68. sdm/processing/base.py +254 -0
  69. sdm/processing/categorical/__init__.py +16 -0
  70. sdm/processing/categorical/_categorical.py +60 -0
  71. sdm/processing/categorical/align.py +540 -0
  72. sdm/processing/categorical/count.py +121 -0
  73. sdm/processing/categorical/impute.py +107 -0
  74. sdm/processing/categorical/shuffle.py +204 -0
  75. sdm/processing/common/__init__.py +32 -0
  76. sdm/processing/common/callable.py +29 -0
  77. sdm/processing/common/choice.py +185 -0
  78. sdm/processing/common/drop.py +28 -0
  79. sdm/processing/common/ensemble.py +187 -0
  80. sdm/processing/common/identity.py +18 -0
  81. sdm/processing/common/select.py +107 -0
  82. sdm/processing/common/sequential.py +158 -0
  83. sdm/processing/common/shuffle.py +195 -0
  84. sdm/processing/common/stype.py +199 -0
  85. sdm/processing/common/table.py +120 -0
  86. sdm/processing/common/task.py +135 -0
  87. sdm/processing/common/to_numerical.py +60 -0
  88. sdm/processing/datetime/__init__.py +8 -0
  89. sdm/processing/datetime/calendar.py +196 -0
  90. sdm/processing/ensemble.py +203 -0
  91. sdm/processing/execution.py +418 -0
  92. sdm/processing/numerical/__init__.py +38 -0
  93. sdm/processing/numerical/_stats.py +29 -0
  94. sdm/processing/numerical/cast.py +33 -0
  95. sdm/processing/numerical/clip.py +48 -0
  96. sdm/processing/numerical/clip_soft.py +54 -0
  97. sdm/processing/numerical/constant.py +235 -0
  98. sdm/processing/numerical/flip_sign.py +78 -0
  99. sdm/processing/numerical/impute.py +52 -0
  100. sdm/processing/numerical/pca.py +62 -0
  101. sdm/processing/numerical/power.py +325 -0
  102. sdm/processing/numerical/quantile.py +306 -0
  103. sdm/processing/numerical/quantile_clip.py +63 -0
  104. sdm/processing/numerical/random_projection.py +86 -0
  105. sdm/processing/numerical/rank_gaussian.py +168 -0
  106. sdm/processing/numerical/robust_scale.py +88 -0
  107. sdm/processing/numerical/sigma_clip.py +123 -0
  108. sdm/processing/numerical/standardize.py +71 -0
  109. sdm/processing/output/__init__.py +14 -0
  110. sdm/processing/output/reduce.py +86 -0
  111. sdm/processing/output/softmax.py +48 -0
  112. sdm/processing/output/sort.py +16 -0
  113. sdm/processing/recipe.py +154 -0
  114. sdm/processing/text/__init__.py +12 -0
  115. sdm/processing/text/sentence_transformer.py +451 -0
  116. sdm/processing/text/tfidf.py +478 -0
  117. sdm/relational/__init__.py +16 -0
  118. sdm/relational/backend/__init__.py +12 -0
  119. sdm/relational/backend/_cugraph.py +546 -0
  120. sdm/relational/backend/_pyg_lib.py +175 -0
  121. sdm/relational/data.py +393 -0
  122. sdm/relational/join.py +173 -0
  123. sdm/relational/sampler.py +259 -0
  124. sdm/relational/task.py +465 -0
  125. sdm/stype.py +396 -0
  126. sdm/task.py +53 -0
  127. sdm/tensor/__init__.py +20 -0
  128. sdm/tensor/categorical.py +953 -0
  129. sdm/tensor/columnar.py +1127 -0
  130. sdm/tensor/io/__init__.py +22 -0
  131. sdm/tensor/io/arrow.py +120 -0
  132. sdm/tensor/io/cudf.py +47 -0
  133. sdm/tensor/mixin.py +124 -0
  134. sdm/tensor/nullable.py +963 -0
  135. sdm/tensor/string.py +526 -0
  136. sdm/tensor/table.py +2035 -0
  137. sdm/tensor/var_len.py +1486 -0
  138. sdm/testing/__init__.py +14 -0
  139. sdm/testing/decorators.py +61 -0
  140. sdm/testing/memory.py +92 -0
  141. structured_data_models-0.2.0.dist-info/METADATA +142 -0
  142. structured_data_models-0.2.0.dist-info/RECORD +152 -0
  143. structured_data_models-0.2.0.dist-info/WHEEL +5 -0
  144. structured_data_models-0.2.0.dist-info/licenses/LICENSE +204 -0
  145. structured_data_models-0.2.0.dist-info/licenses/THIRD_PARTY_LICENSES.md +40 -0
  146. structured_data_models-0.2.0.dist-info/licenses/sdm/models/kumo/relational/NOTICE +5 -0
  147. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabfm/LICENSE +209 -0
  148. structured_data_models-0.2.0.dist-info/licenses/sdm/models/tabiclv2/LICENSE +33 -0
  149. structured_data_models-0.2.0.dist-info/licenses/third_party/contributor-covenant/LICENSE +20 -0
  150. structured_data_models-0.2.0.dist-info/licenses/third_party/pytorch/LICENSE +84 -0
  151. structured_data_models-0.2.0.dist-info/licenses/third_party/tabpfn-extensions/LICENSE +202 -0
  152. structured_data_models-0.2.0.dist-info/top_level.txt +1 -0
sdm/models/base.py ADDED
@@ -0,0 +1,966 @@
1
+ # SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2
+ # SPDX-License-Identifier: Apache-2.0
3
+
4
+ import abc
5
+ import copy
6
+ from collections.abc import Hashable, Iterable, Mapping, Sequence
7
+ from typing import Any, ClassVar, cast
8
+
9
+ import torch
10
+ from torch import Tensor
11
+
12
+ from sdm import (
13
+ EnsembleTable,
14
+ Recipe,
15
+ RelatedTables,
16
+ Stype,
17
+ StypeLike,
18
+ TableTensor,
19
+ Task,
20
+ TaskLike,
21
+ )
22
+ from sdm._inference import inference_mode
23
+ from sdm._warnings import warn_once
24
+ from sdm.cache import Cache
25
+ from sdm.models.callback import Callback
26
+ from sdm.processing.execution import (
27
+ MemberContext,
28
+ MemberQuery,
29
+ RecipeExecution,
30
+ )
31
+ from sdm.relational.task import RelatedTablesSchema
32
+ from sdm.tensor.table import TableSchema
33
+
34
+
35
+ class ICLModel(torch.nn.Module, abc.ABC):
36
+ r"""Base model for in-context foundation models on structured data.
37
+
38
+ :class:`ICLModel` defines the public interface shared among in-context
39
+ foundation models on structured data.
40
+ It enriches models by unified pre-processing and post-processing routines,
41
+ key/value caching, and ensembling.
42
+
43
+ Args:
44
+ task: The tasks to initialize. If ``None``, all tasks supported by this
45
+ model are initialized.
46
+ """
47
+
48
+ #: Semantic types supported for input columns in this model.
49
+ supported_feature_stypes: ClassVar[frozenset[Stype]]
50
+
51
+ #: Semantic types supported for target columns in this model.
52
+ supported_target_stypes: ClassVar[frozenset[Stype]]
53
+
54
+ #: Prediction tasks supported in this model.
55
+ supported_tasks: ClassVar[frozenset[Task]]
56
+
57
+ #: Whether this model supports multi-target predictions.
58
+ supports_multi_target: ClassVar[bool]
59
+
60
+ #: Whether this model supports additional related context.
61
+ supports_related_tables: ClassVar[bool]
62
+
63
+ def __init_subclass__(cls, **kwargs: Any) -> None:
64
+ super().__init_subclass__(**kwargs)
65
+
66
+ if hasattr(cls, "supported_target_stypes"):
67
+ cls.supported_tasks = frozenset(
68
+ Task.from_stype(stype) for stype in cls.supported_target_stypes
69
+ )
70
+
71
+ def __init__(
72
+ self,
73
+ task: TaskLike | Iterable[TaskLike] | None = None,
74
+ ) -> None:
75
+ super().__init__()
76
+
77
+ if task is None:
78
+ self.tasks = self.supported_tasks
79
+ elif isinstance(task, str):
80
+ self.tasks = frozenset({Task(task)})
81
+ else:
82
+ self.tasks = frozenset({Task(t) for t in task})
83
+
84
+ if not self.tasks.issubset(self.supported_tasks):
85
+ invalid = ", ".join(
86
+ f"{str(task)!r}" for task in self.tasks - self.supported_tasks
87
+ )
88
+ raise ValueError(
89
+ f"{self.__class__.__name__!r} received unsupported tasks "
90
+ f"{invalid}"
91
+ )
92
+
93
+ self._cache: Cache | None = None
94
+ self._transfer_streams: dict[torch.device, torch.cuda.Stream] = {}
95
+
96
+ def forward(
97
+ self,
98
+ x_context: Tensor | TableTensor | EnsembleTable, # [..., R_context, D]
99
+ y_context: Tensor | TableTensor | EnsembleTable, # [..., R_context, 1]
100
+ x_query: Tensor | TableTensor | EnsembleTable, # [..., R_query, D]
101
+ related_context_tables: RelatedTables | None = None,
102
+ related_query_tables: RelatedTables | None = None,
103
+ *,
104
+ recipe: Recipe | None = None,
105
+ num_estimators: int | None = None,
106
+ estimator_batch_size: int | None = 1,
107
+ callbacks: Sequence[Callback] | None = None,
108
+ generator: torch.Generator | None = None,
109
+ **kwargs: Any,
110
+ ) -> TableTensor: # Recipe-defined output shape.
111
+ r"""The in-context learning forward pass.
112
+
113
+ Args:
114
+ x_context: The feature tensor of in-context examples with shape
115
+ ``[..., R_context, D]`` with ``R_context`` rows and ``D``
116
+ columns.
117
+ y_context: The targets of in-context examples with shape
118
+ ``[..., R_context, 1]``.
119
+ x_query: The feature tensor of query examples with shape
120
+ ``[..., R_query, D]`` with ``R_query`` rows and ``D`` columns.
121
+ related_context_tables: Related context for in-context examples.
122
+ related_query_tables: Related context for query examples.
123
+ recipe: The custom recipe for pre- and post-processing.
124
+ num_estimators: The number of estimators ``E`` for ensembling.
125
+ If ``None``, the leading dimension of higher-rank inputs is
126
+ used as the estimator dimension, allowing input data to be
127
+ customized per estimator (*e.g.*, different in-context examples
128
+ per estimator).
129
+ estimator_batch_size: Maximum number of consecutive estimators run
130
+ through the model in one call. ``1`` (default) runs estimators
131
+ one by one; ``None`` batches as many as possible. Estimators
132
+ whose preprocessed tables differ in shape or target class
133
+ set, or that come with related tables, run in separate
134
+ calls. Device memory grows with the batch size.
135
+ callbacks: Callbacks applied in sequence to this model call.
136
+ generator: Pseudorandom number generator used for sampling during
137
+ pre-processing and model execution.
138
+ kwargs: Additional keyword arguments passed to the model.
139
+
140
+ Returns:
141
+ The processed prediction after applying ``recipe.output`` to the
142
+ stacked estimator outputs with shape ``[E, ..., R_query, *]``.
143
+ """
144
+ callbacks = () if callbacks is None else callbacks
145
+ requires_grad = self.training
146
+ requires_grad |= any(callback.requires_grad for callback in callbacks)
147
+
148
+ if (related_context_tables is None) != (related_query_tables is None):
149
+ raise ValueError(
150
+ "Expected 'related_context_tables' and 'related_query_tables' "
151
+ "to be provided together"
152
+ )
153
+
154
+ if related_query_tables is not None:
155
+ assert related_context_tables is not None
156
+ related_query_tables = related_query_tables.select_tables(
157
+ tables=related_context_tables.tables
158
+ )
159
+
160
+ recipe_execution = RecipeExecution(
161
+ self.default_recipe() if recipe is None else copy.deepcopy(recipe)
162
+ )
163
+ with (
164
+ torch.amp.autocast(x_query.device.type, enabled=False),
165
+ inference_mode("no_grad" if requires_grad else "inference"),
166
+ ):
167
+ contexts = recipe_execution.fit_transform(
168
+ x=x_context,
169
+ y=y_context,
170
+ related_tables=related_context_tables,
171
+ num_members=num_estimators,
172
+ generator=generator,
173
+ )
174
+ with (
175
+ torch.amp.autocast(x_query.device.type, enabled=False),
176
+ inference_mode("no_grad" if requires_grad else "inference"),
177
+ ):
178
+ queries = recipe_execution.transform(
179
+ x=x_query,
180
+ related_tables=related_query_tables,
181
+ )
182
+
183
+ outs = self._forward_members(
184
+ contexts=contexts,
185
+ queries=queries,
186
+ estimator_batch_size=estimator_batch_size,
187
+ callbacks=callbacks,
188
+ generator=generator,
189
+ **kwargs,
190
+ )
191
+
192
+ # Regression: invert target before stacking estimator outputs.
193
+ if contexts[0].y.numerical.size(-1) > 0:
194
+ with (
195
+ torch.amp.autocast(x_query.device.type, enabled=False),
196
+ inference_mode("grad" if requires_grad else "inference"),
197
+ ):
198
+ outs = list(recipe_execution.inverse_transform_target(outs))
199
+
200
+ with (
201
+ torch.amp.autocast(x_query.device.type, enabled=False),
202
+ inference_mode("grad" if requires_grad else "inference"),
203
+ ):
204
+ return recipe_execution.transform_output(outs)
205
+
206
+ def fit(
207
+ self,
208
+ x: Tensor | TableTensor | EnsembleTable, # [..., R, D]
209
+ y: Tensor | TableTensor | EnsembleTable, # [..., R, 1]
210
+ related_tables: RelatedTables | None = None,
211
+ *,
212
+ recipe: Recipe | None = None,
213
+ num_estimators: int | None = None,
214
+ estimator_batch_size: int | None = 1,
215
+ callbacks: Sequence[Callback] | None = None,
216
+ generator: torch.Generator | None = None,
217
+ **kwargs: Any,
218
+ ) -> None:
219
+ r"""Fit and cache in-context examples.
220
+
221
+ Repeated calls to :meth:`predict` can then reuse the same in-context
222
+ examples while only providing new query examples.
223
+
224
+ Args:
225
+ x: The feature tensor of in-context examples with shape
226
+ ``[..., R, D]`` with ``R`` rows and ``C`` columns.
227
+ y: The targets of in-context examples with shape
228
+ ``[..., R, 1]``.
229
+ related_tables: Related context for in-context examples.
230
+ recipe: The custom recipe for pre- and post-processing.
231
+ num_estimators: The number of estimators ``E`` for ensembling.
232
+ If ``None``, the leading dimension of higher-rank inputs is
233
+ used as the estimator dimension, allowing input data to be
234
+ customized per estimator (*e.g.*, different in-context examples
235
+ per estimator).
236
+ estimator_batch_size: Maximum number of consecutive estimators run
237
+ through the model in one call. ``1`` (default) runs estimators
238
+ one by one; ``None`` batches as many as possible. Estimators
239
+ whose preprocessed tables differ in shape or target class
240
+ set, or that come with related tables, run in separate
241
+ calls. Device memory grows with the batch size. Estimators
242
+ fitted together are predicted together.
243
+ callbacks: Callbacks applied in sequence to this model call.
244
+ generator: Pseudorandom number generator used for sampling during
245
+ pre-processing and model execution.
246
+ kwargs: Additional keyword arguments passed to the model.
247
+ """
248
+ callbacks = () if callbacks is None else callbacks
249
+
250
+ self.clear()
251
+
252
+ recipe_execution = RecipeExecution(
253
+ self.default_recipe() if recipe is None else copy.deepcopy(recipe)
254
+ )
255
+ with (
256
+ torch.amp.autocast(x.device.type, enabled=False),
257
+ inference_mode("no_grad"),
258
+ ):
259
+ contexts = recipe_execution.fit_transform(
260
+ x=x,
261
+ y=y,
262
+ related_tables=related_tables,
263
+ num_members=num_estimators,
264
+ generator=generator,
265
+ )
266
+
267
+ contexts = [
268
+ self._prepare_context(context, callbacks) for context in contexts
269
+ ]
270
+ class_values = _class_values(contexts, estimator_batch_size)
271
+ batches = _batch_slices(
272
+ contexts=contexts,
273
+ queries=None,
274
+ class_values=class_values,
275
+ estimator_batch_size=estimator_batch_size,
276
+ )
277
+ cache = Cache(
278
+ recipe_execution=recipe_execution,
279
+ kwargs=kwargs,
280
+ num_batches=len(batches),
281
+ )
282
+ for i, batch in enumerate(batches):
283
+ with inference_mode("no_grad"):
284
+ context = _stack_context(contexts[batch])
285
+ categorical_mask = _categorical_mask(contexts[batch])
286
+ batch_cache = Cache(
287
+ x_schemas=tuple(
288
+ context.x.schema for context in contexts[batch]
289
+ ),
290
+ y_schema=context.y.schema,
291
+ related_tables_schema=context.related_tables.schema
292
+ if context.related_tables is not None
293
+ else None,
294
+ classes=(
295
+ context.y.categorical.categories[0]
296
+ if context.y.categorical.size(-1) > 0
297
+ else None
298
+ ),
299
+ class_values=class_values[batch],
300
+ categorical_mask=categorical_mask,
301
+ )
302
+ self._forward(
303
+ x_context=context.x,
304
+ y_context=context.y,
305
+ x_query=None,
306
+ related_context_tables=context.related_tables,
307
+ related_query_tables=None,
308
+ cache=batch_cache,
309
+ generator=generator,
310
+ categorical_mask=categorical_mask,
311
+ **kwargs,
312
+ )
313
+
314
+ if x.is_cuda and len(contexts) > 1:
315
+ try: # Copy to pinned CPU memory:
316
+ batch_cache = batch_cache._apply_tensor(
317
+ lambda tensor: torch.ops.aten._to_copy.default(
318
+ tensor,
319
+ device="cpu",
320
+ pin_memory=True,
321
+ non_blocking=True, # Required for `pin_memory`.
322
+ )
323
+ )
324
+ finally:
325
+ torch.cuda.current_stream(x.device).synchronize()
326
+
327
+ cache[i] = batch_cache
328
+
329
+ self._cache = cache.freeze()
330
+
331
+ def predict(
332
+ self,
333
+ x: Tensor | TableTensor | EnsembleTable, # [..., R, D]
334
+ related_tables: RelatedTables | None = None,
335
+ *,
336
+ callbacks: Sequence[Callback] | None = None,
337
+ ) -> TableTensor: # Recipe-defined output shape.
338
+ r"""Predict unseen query examples.
339
+
340
+ .. note::
341
+
342
+ This method requires a prior call to :meth:`fit`.
343
+
344
+ Args:
345
+ x: The feature tensor of query examples with shape
346
+ ``[..., R, D]`` with ``R`` rows and ``D`` columns.
347
+ related_tables: Related context for query examples.
348
+ callbacks: Callbacks applied in sequence to this model call.
349
+
350
+ Returns:
351
+ The processed prediction after applying ``recipe.output`` to the
352
+ stacked estimator outputs with shape ``[E, ..., R, *]``.
353
+ """
354
+ if self.training:
355
+ raise RuntimeError(
356
+ f"{self.__class__.__name__!r}.predict() does not support "
357
+ "gradient-based training through a fitted context cache. "
358
+ "To fix, call `model.eval()`."
359
+ )
360
+
361
+ callbacks = () if callbacks is None else callbacks
362
+ requires_grad = any(callback.requires_grad for callback in callbacks)
363
+
364
+ if self._cache is None:
365
+ raise RuntimeError(
366
+ f"{self.__class__.__name__!r} not yet fitted. Make sure to "
367
+ f"call '{self.__class__.__name__}.fit()' before."
368
+ )
369
+
370
+ if related_tables is not None:
371
+ if cast(Cache, self._cache[0])["related_tables_schema"] is None:
372
+ raise ValueError(
373
+ "Expected related tables to be provided together"
374
+ )
375
+ related_tables = related_tables.select_tables(
376
+ tables=cast(
377
+ RelatedTablesSchema,
378
+ cast(Cache, self._cache[0])["related_tables_schema"],
379
+ ).tables,
380
+ )
381
+
382
+ recipe_execution = cast(
383
+ RecipeExecution,
384
+ self._cache["recipe_execution"],
385
+ )
386
+ num_batches = cast(int, self._cache["num_batches"])
387
+ caches = [cast(Cache, self._cache[i]) for i in range(num_batches)]
388
+ next_cache = caches[0]
389
+
390
+ compute_stream: torch.cuda.Stream | None = None
391
+ transfer_stream: torch.cuda.Stream | None = None
392
+ try:
393
+ if x.is_cuda:
394
+ compute_stream = torch.cuda.current_stream(x.device)
395
+ if x.device not in self._transfer_streams:
396
+ transfer_stream = torch.cuda.Stream(x.device)
397
+ self._transfer_streams[x.device] = transfer_stream
398
+ else:
399
+ transfer_stream = self._transfer_streams[x.device]
400
+ with torch.cuda.stream(transfer_stream):
401
+ next_cache = next_cache.to(x.device, non_blocking=True)
402
+
403
+ with (
404
+ torch.amp.autocast(x.device.type, enabled=False),
405
+ inference_mode("no_grad" if requires_grad else "inference"),
406
+ ):
407
+ queries = recipe_execution.transform(x, related_tables)
408
+
409
+ if x.is_cuda:
410
+ assert compute_stream is not None
411
+ assert transfer_stream is not None
412
+ compute_stream.wait_stream(transfer_stream)
413
+
414
+ outs: list[TableTensor] = []
415
+ start = 0
416
+ for i in range(len(caches)):
417
+ cache, next_cache = next_cache, None
418
+ assert cache is not None
419
+
420
+ x_schemas = cast(tuple[TableSchema, ...], cache["x_schemas"])
421
+ batch_queries = [
422
+ self._prepare_query(
423
+ query=query,
424
+ x_schema=x_schema,
425
+ related_tables_schema=cast(
426
+ RelatedTablesSchema | None,
427
+ cache["related_tables_schema"],
428
+ ),
429
+ callbacks=callbacks,
430
+ )
431
+ for query, x_schema in zip(
432
+ queries[start : start + len(x_schemas)],
433
+ x_schemas,
434
+ strict=True,
435
+ )
436
+ ]
437
+ start += len(x_schemas)
438
+
439
+ if i + 1 < len(caches):
440
+ next_cache = caches[i + 1]
441
+ if x.is_cuda and next_cache is not None:
442
+ assert transfer_stream is not None
443
+ with torch.cuda.stream(transfer_stream):
444
+ next_cache = next_cache.to(x.device, non_blocking=True)
445
+
446
+ outs += self._forward_batch(
447
+ contexts=None,
448
+ queries=batch_queries,
449
+ cache=cache,
450
+ categorical_mask=cast(Tensor, cache["categorical_mask"]),
451
+ class_values=cast(
452
+ Sequence[tuple[Any, ...] | None] | None,
453
+ cache["class_values"],
454
+ ),
455
+ callbacks=callbacks,
456
+ requires_grad=requires_grad,
457
+ generator=None,
458
+ **cast(dict[str, Any], self._cache["kwargs"]),
459
+ )
460
+
461
+ if x.is_cuda:
462
+ assert compute_stream is not None
463
+ for tensor in cache._tensors():
464
+ tensor.record_stream(compute_stream)
465
+
466
+ if x.is_cuda and next_cache is not None:
467
+ assert compute_stream is not None
468
+ assert transfer_stream is not None
469
+ compute_stream.wait_stream(transfer_stream)
470
+
471
+ except BaseException:
472
+ if transfer_stream is not None:
473
+ transfer_stream.synchronize()
474
+ raise
475
+
476
+ # Regression: invert target before stacking estimator outputs.
477
+ if cast(Cache, self._cache[0])["classes"] is None:
478
+ with (
479
+ torch.amp.autocast(x.device.type, enabled=False),
480
+ inference_mode("grad" if requires_grad else "inference"),
481
+ ):
482
+ outs = list(recipe_execution.inverse_transform_target(outs))
483
+
484
+ with (
485
+ torch.amp.autocast(x.device.type, enabled=False),
486
+ inference_mode("grad" if requires_grad else "inference"),
487
+ ):
488
+ return recipe_execution.transform_output(outs)
489
+
490
+ def clear(self) -> None:
491
+ r"""Clear cached context state created by :meth:`fit`."""
492
+ self._cache = None
493
+
494
+ def __getstate__(self) -> dict[str, object]:
495
+ for stream in self._transfer_streams.values():
496
+ stream.synchronize()
497
+ state = super().__getstate__()
498
+ state.pop("_transfer_streams", None)
499
+ return state
500
+
501
+ def __setstate__(self, state: dict[str, object]) -> None:
502
+ super().__setstate__(state)
503
+ self._transfer_streams = {}
504
+
505
+ def __repr__(self) -> str:
506
+ device = next(self.parameters()).device
507
+ device_repr = f"device={device}" if device.type != "cpu" else ""
508
+ return f"{self.__class__.__name__}({device_repr})"
509
+
510
+ # Abstract Methods ########################################################
511
+
512
+ @abc.abstractmethod
513
+ def _forward(
514
+ self,
515
+ x_context: TableTensor | None, # [..., R_context, D]
516
+ y_context: TableTensor | None, # [..., R_context, Y]
517
+ x_query: TableTensor | None, # [..., R_query, D]
518
+ related_context_tables: RelatedTables[TableTensor] | None,
519
+ related_query_tables: RelatedTables[TableTensor] | None,
520
+ cache: Cache | None,
521
+ generator: torch.Generator | None,
522
+ *,
523
+ categorical_mask: Tensor,
524
+ **kwargs: Any,
525
+ ) -> TableTensor: # [..., R_query, *]
526
+ r"""Run the model on preprocessed tables of one estimator batch.
527
+
528
+ Tables carry shape ``[E, ..., R, D]`` when ``E > 1`` estimators run
529
+ together and ``[..., R, D]`` otherwise. Column names and category
530
+ vocabularies are those of the first estimator; per-estimator column
531
+ and class order is not observable from the tables.
532
+
533
+ Args:
534
+ x_context: The feature tensor of in-context examples, or ``None``
535
+ when replaying a cache.
536
+ y_context: The targets of in-context examples, or ``None`` when
537
+ replaying a cache.
538
+ x_query: The feature tensor of query examples, or ``None`` when
539
+ recording a cache.
540
+ related_context_tables: Related context for in-context examples.
541
+ related_query_tables: Related context for query examples.
542
+ cache: The cache to record into or replay from, or ``None``.
543
+ Recorded tensors are replayed with queries of the same
544
+ estimator batch.
545
+ generator: Pseudorandom number generator for model execution.
546
+ categorical_mask: Boolean ``[C]`` or ``[E, 1, ..., C]`` tensor
547
+ broadcastable to ``x.numerical.size()[:-2] + (C,)`` marking
548
+ numerical feature columns that were categorical before
549
+ preprocessing. Passed on recording, replaying and uncached
550
+ calls alike.
551
+ kwargs: Additional keyword arguments passed by the caller.
552
+
553
+ Returns:
554
+ Predictions of shape ``[..., R_query, *]``. Classification outputs
555
+ hold exactly one column per class in the order of
556
+ ``y_context.categorical.categories[0]`` (``cache["classes"]`` on
557
+ replay), each named by its class value; :class:`ICLModel` relabels
558
+ them per estimator when stacked.
559
+ """
560
+
561
+ @classmethod
562
+ @abc.abstractmethod
563
+ def default_recipe(cls) -> Recipe:
564
+ r"""Return the default processing recipe for this model."""
565
+
566
+ # Helpers #################################################################
567
+
568
+ def _forward_members(
569
+ self,
570
+ contexts: Sequence[MemberContext],
571
+ queries: Sequence[MemberQuery],
572
+ *,
573
+ estimator_batch_size: int | None = 1,
574
+ callbacks: Sequence[Callback] | None = None,
575
+ generator: torch.Generator | None = None,
576
+ **kwargs: Any,
577
+ ) -> list[TableTensor]:
578
+ r"""Run recipe-transformed members that live on the model device.
579
+
580
+ Returns one output per member before target inversion and
581
+ ``recipe.output``.
582
+ """
583
+ callbacks = () if callbacks is None else callbacks
584
+ requires_grad = self.training
585
+ requires_grad |= any(callback.requires_grad for callback in callbacks)
586
+
587
+ if estimator_batch_size == 1 and len(contexts) > 1:
588
+ outs: list[TableTensor] = []
589
+ for context, query in zip(contexts, queries, strict=True):
590
+ outs.extend(
591
+ self._forward_members(
592
+ contexts=(context,),
593
+ queries=(query,),
594
+ estimator_batch_size=1,
595
+ callbacks=callbacks,
596
+ generator=generator,
597
+ **kwargs,
598
+ )
599
+ )
600
+ return outs
601
+
602
+ contexts = [
603
+ self._prepare_context(context, callbacks) for context in contexts
604
+ ]
605
+ queries = [
606
+ self._prepare_query(
607
+ query=query,
608
+ x_schema=context.x.schema,
609
+ related_tables_schema=context.related_tables.schema
610
+ if context.related_tables is not None
611
+ else None,
612
+ callbacks=callbacks,
613
+ )
614
+ for context, query in zip(contexts, queries, strict=True)
615
+ ]
616
+ class_values = _class_values(contexts, estimator_batch_size)
617
+ outs: list[TableTensor] = []
618
+ for batch in _batch_slices(
619
+ contexts=contexts,
620
+ queries=queries,
621
+ class_values=class_values,
622
+ estimator_batch_size=estimator_batch_size,
623
+ ):
624
+ outs += self._forward_batch(
625
+ contexts=contexts[batch],
626
+ queries=queries[batch],
627
+ cache=None,
628
+ categorical_mask=None,
629
+ class_values=class_values[batch],
630
+ callbacks=callbacks,
631
+ requires_grad=requires_grad,
632
+ generator=generator,
633
+ **kwargs,
634
+ )
635
+ return outs
636
+
637
+ def _prepare_context(
638
+ self,
639
+ context: MemberContext,
640
+ callbacks: Sequence[Callback],
641
+ ) -> MemberContext:
642
+ for callback in callbacks:
643
+ x, y, related_tables = callback.on_context_preprocessing_end(
644
+ self,
645
+ context.x,
646
+ context.y,
647
+ context.related_tables,
648
+ )
649
+ context = context._replace(x=x, y=y, related_tables=related_tables)
650
+ self._validate_context(
651
+ x=context.x,
652
+ y=context.y,
653
+ related_tables=context.related_tables,
654
+ )
655
+ return context
656
+
657
+ def _prepare_query(
658
+ self,
659
+ query: MemberQuery,
660
+ x_schema: TableSchema,
661
+ related_tables_schema: RelatedTablesSchema | None,
662
+ callbacks: Sequence[Callback],
663
+ ) -> MemberQuery:
664
+ for callback in callbacks:
665
+ query = MemberQuery(
666
+ *callback.on_query_preprocessing_end(self, *query)
667
+ )
668
+ self._validate_query(
669
+ x_context=x_schema,
670
+ x_query=query.x,
671
+ related_context_tables=related_tables_schema,
672
+ related_query_tables=query.related_tables,
673
+ )
674
+ return query
675
+
676
+ def _forward_batch(
677
+ self,
678
+ contexts: Sequence[MemberContext] | None,
679
+ queries: Sequence[MemberQuery],
680
+ *,
681
+ cache: Cache | None,
682
+ categorical_mask: Tensor | None,
683
+ class_values: Sequence[tuple[Any, ...] | None] | None,
684
+ callbacks: Sequence[Callback],
685
+ requires_grad: bool,
686
+ generator: torch.Generator | None,
687
+ **kwargs: Any,
688
+ ) -> list[TableTensor]:
689
+ # Stacking inside the autograd region keeps callback-captured leaves
690
+ # attached to the graph.
691
+ with inference_mode("grad" if requires_grad else "inference"):
692
+ context = None if contexts is None else _stack_context(contexts)
693
+ if categorical_mask is None:
694
+ assert contexts is not None
695
+ categorical_mask = _categorical_mask(contexts)
696
+ query = _stack_query(queries)
697
+ out = self._forward(
698
+ x_context=None if context is None else context.x,
699
+ y_context=None if context is None else context.y,
700
+ x_query=query.x,
701
+ related_context_tables=(
702
+ None if context is None else context.related_tables
703
+ ),
704
+ related_query_tables=query.related_tables,
705
+ cache=cache,
706
+ generator=generator,
707
+ categorical_mask=categorical_mask,
708
+ **kwargs,
709
+ )
710
+ outs = _unstack(out, class_values, len(queries))
711
+ for i in range(len(outs)):
712
+ for callback in callbacks:
713
+ outs[i] = callback.on_model_forward_end(self, outs[i])
714
+ return [cast(TableTensor, out.to(query.x.dtype)) for out in outs]
715
+
716
+ def _validate_context(
717
+ self,
718
+ x: TableTensor,
719
+ y: TableTensor,
720
+ related_tables: RelatedTables[TableTensor] | None,
721
+ ) -> None:
722
+
723
+ if not self.supports_multi_target and y.size(-1) != 1:
724
+ raise ValueError(
725
+ f"Expected target to have exactly one column "
726
+ f"(got {y.size(-1)})"
727
+ )
728
+ if x.size()[:-1] != y.size()[:-1]:
729
+ raise ValueError(
730
+ f"Expected features and targets to have matching row "
731
+ f"dimensions (got {tuple(x.size()[:-1])} and "
732
+ f"{tuple(y.size()[:-1])})"
733
+ )
734
+ invalid = x.active_stypes - self.supported_feature_stypes - {Stype.id}
735
+ if len(invalid) > 0:
736
+ stypes = ", ".join(f"{str(stype)!r}" for stype in invalid)
737
+ warn_once(
738
+ key="model-unsupported-feature-stypes",
739
+ message=(
740
+ f"{self.__class__.__name__!r} received unsupported "
741
+ f"feature stypes {stypes}. Columns with unsupported "
742
+ f"feature stypes will not be consumed by the model."
743
+ ),
744
+ )
745
+ invalid = y.active_stypes - self.supported_target_stypes
746
+ if len(invalid) > 0:
747
+ stypes = ", ".join(f"{str(stype)!r}" for stype in invalid)
748
+ raise ValueError(
749
+ f"{self.__class__.__name__!r} received unsupported target "
750
+ f"stypes {stypes}"
751
+ )
752
+ invalid = y.active_stypes - {task.stype for task in self.tasks}
753
+ if len(invalid) > 0:
754
+ tasks = ", ".join(
755
+ f"{str(Task.from_stype(stype))!r}" for stype in invalid
756
+ )
757
+ raise ValueError(
758
+ f"{self.__class__.__name__!r} is not initialized for tasks "
759
+ f"{tasks}"
760
+ )
761
+
762
+ if related_tables is not None:
763
+ if not self.supports_related_tables:
764
+ raise ValueError(
765
+ f"{self.__class__.__name__!r} does not support related "
766
+ f"tables"
767
+ )
768
+ for table_name, table in related_tables.tables.items():
769
+ invalid = table.active_stypes - self.supported_feature_stypes
770
+ invalid = invalid - {Stype.id}
771
+ if len(invalid) > 0:
772
+ stypes = ", ".join(f"{str(stype)!r}" for stype in invalid)
773
+ warn_once(
774
+ key="model-unsupported-feature-stypes",
775
+ message=(
776
+ f"{self.__class__.__name__!r} received "
777
+ f"unsupported feature stypes {stypes} in related "
778
+ f"table {table_name!r}. Columns with unsupported "
779
+ f"feature stypes will not be consumed by the "
780
+ f"model."
781
+ ),
782
+ )
783
+
784
+ def _validate_query(
785
+ self,
786
+ x_context: TableSchema,
787
+ x_query: TableTensor,
788
+ related_context_tables: RelatedTablesSchema | None,
789
+ related_query_tables: RelatedTables[TableTensor] | None,
790
+ ) -> None:
791
+
792
+ if x_context != x_query.schema:
793
+ raise ValueError(
794
+ "Expected context and query features to share the same schema"
795
+ )
796
+
797
+ if (related_context_tables is None) != (related_query_tables is None):
798
+ raise ValueError("Expected related tables to be provided together")
799
+
800
+ if related_context_tables is not None:
801
+ assert related_query_tables is not None
802
+ if not related_query_tables.schema.is_subset_of(
803
+ related_context_tables
804
+ ):
805
+ raise ValueError(
806
+ "Expected related context and query tables to share the "
807
+ "same schema"
808
+ )
809
+
810
+
811
+ def _stack(tables: Sequence[TableTensor]) -> TableTensor:
812
+ ref = tables[0]
813
+ if len(tables) == 1:
814
+ return ref
815
+ # torch.stack aligns columns by name, which would undo per-estimator column
816
+ # shuffles; renaming to the first member's names stacks blocks by position.
817
+ columns = cast(Mapping[StypeLike, Sequence[str]], ref.columns)
818
+ renamed: list[Tensor] = [
819
+ ref,
820
+ *(
821
+ table.__class__(columns=columns, **dict(table.items()))
822
+ for table in tables[1:]
823
+ ),
824
+ ]
825
+ return cast(TableTensor, torch.stack(renamed))
826
+
827
+
828
+ def _stack_context(members: Sequence[MemberContext]) -> MemberContext:
829
+ if len(members) == 1:
830
+ return members[0]
831
+ return MemberContext(
832
+ x=_stack([member.x for member in members]),
833
+ y=_stack([member.y for member in members]),
834
+ related_tables=None,
835
+ input_stypes=members[0].input_stypes,
836
+ )
837
+
838
+
839
+ def _stack_query(members: Sequence[MemberQuery]) -> MemberQuery:
840
+ if len(members) == 1:
841
+ return members[0]
842
+ return MemberQuery(
843
+ x=_stack([member.x for member in members]), related_tables=None
844
+ )
845
+
846
+
847
+ def _batch_slices(
848
+ contexts: Sequence[MemberContext],
849
+ queries: Sequence[MemberQuery] | None,
850
+ class_values: Sequence[tuple[Any, ...] | None],
851
+ estimator_batch_size: int | None,
852
+ ) -> list[slice]:
853
+ # Consecutive estimators that can go through one `_forward` together,
854
+ # split when shapes/dtypes or the target class set change, or the
855
+ # batch is full.
856
+ related = any(context.related_tables is not None for context in contexts)
857
+ if queries is not None:
858
+ related = related or any(
859
+ query.related_tables is not None for query in queries
860
+ )
861
+ if related:
862
+ return [slice(i, i + 1) for i in range(len(contexts))]
863
+
864
+ batches: list[slice] = []
865
+ start = 0
866
+ key: Hashable = None
867
+ # Stack when table shapes/dtypes match and the target class set matches.
868
+ for i, context in enumerate(contexts):
869
+ query = None if queries is None else queries[i]
870
+ tables = (
871
+ [context.x, context.y]
872
+ if query is None
873
+ else [context.x, context.y, query.x]
874
+ )
875
+ classes = class_values[i]
876
+ member_key = (
877
+ tuple(
878
+ (
879
+ tuple(
880
+ (stype, block.size(), block.dtype)
881
+ for stype, block in table.items()
882
+ ),
883
+ tuple(c.numel() for c in table.categorical.categories),
884
+ )
885
+ for table in tables
886
+ ),
887
+ None if classes is None else frozenset(classes),
888
+ )
889
+ if i > start and (
890
+ member_key != key
891
+ or (
892
+ estimator_batch_size is not None
893
+ and i - start == estimator_batch_size
894
+ )
895
+ ):
896
+ batches.append(slice(start, i))
897
+ start = i
898
+ key = member_key
899
+ batches.append(slice(start, len(contexts)))
900
+ return batches
901
+
902
+
903
+ def _categorical_mask(members: Sequence[MemberContext]) -> Tensor:
904
+ x = members[0].x
905
+ mask = torch.tensor(
906
+ [
907
+ [
908
+ member.input_stypes.get(column) == Stype.categorical
909
+ for column in member.x.columns[Stype.numerical]
910
+ ]
911
+ for member in members
912
+ ],
913
+ dtype=torch.bool,
914
+ device=x.device,
915
+ ) # [E, C]
916
+ if len(members) == 1:
917
+ return mask[0]
918
+ # Insert the member's batch dimensions so the mask broadcasts over them:
919
+ return mask.view(len(members), *(1,) * (x.dim() - 2), -1) # [E, 1, ..., C]
920
+
921
+
922
+ def _class_values(
923
+ contexts: Sequence[MemberContext],
924
+ estimator_batch_size: int | None,
925
+ ) -> list[tuple[Any, ...] | None]:
926
+ # Class labels per estimator, used to group and relabel stacked batches.
927
+ # Unused when each estimator already has its own ``_forward``.
928
+ if (
929
+ estimator_batch_size == 1
930
+ or contexts[0].related_tables is not None
931
+ or contexts[0].y.categorical.size(-1) == 0
932
+ ):
933
+ return [None] * len(contexts)
934
+ return [
935
+ tuple(context.y.categorical.categories[0].tolist())
936
+ for context in contexts
937
+ ]
938
+
939
+
940
+ def _unstack(
941
+ out: TableTensor,
942
+ class_values: Sequence[tuple[Any, ...] | None] | None,
943
+ num_members: int,
944
+ ) -> list[TableTensor]:
945
+ outs = (
946
+ [out]
947
+ if num_members == 1
948
+ else list(cast(tuple[TableTensor, ...], out.unbind(0)))
949
+ )
950
+ if class_values is None or len(class_values) == 1:
951
+ return outs
952
+ first = class_values[0]
953
+ if first is None:
954
+ return outs
955
+ # The model labels columns in the first member's class order; member `e`'s
956
+ # column `j` holds class `class_values[e][j]`.
957
+ labels = out.columns[Stype.numerical]
958
+ index = {value: i for i, value in enumerate(first)}
959
+ stacked = cast(Sequence[tuple[Any, ...]], class_values)
960
+ return [
961
+ TableTensor(
962
+ columns={Stype.numerical: [labels[index[v]] for v in classes]},
963
+ numerical=member_out.numerical,
964
+ )
965
+ for member_out, classes in zip(outs, stacked, strict=True)
966
+ ]